Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
7a7ffe65 HX |
2 | /* |
3 | * Symmetric key cipher operations. | |
4 | * | |
5 | * Generic encrypt/decrypt wrapper for ciphers, handles operations across | |
6 | * multiple page boundaries by using temporary blocks. In user context, | |
7 | * the kernel is given a chance to schedule us once per page. | |
8 | * | |
9 | * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au> | |
7a7ffe65 HX |
10 | */ |
11 | ||
b286d8b1 | 12 | #include <crypto/internal/aead.h> |
0eb76ba2 | 13 | #include <crypto/internal/cipher.h> |
7a7ffe65 | 14 | #include <crypto/internal/skcipher.h> |
b286d8b1 | 15 | #include <crypto/scatterwalk.h> |
7a7ffe65 | 16 | #include <linux/bug.h> |
4e6c3df4 | 17 | #include <linux/cryptouser.h> |
d8c34b94 | 18 | #include <linux/compiler.h> |
b286d8b1 | 19 | #include <linux/list.h> |
7a7ffe65 | 20 | #include <linux/module.h> |
4e6c3df4 HX |
21 | #include <linux/rtnetlink.h> |
22 | #include <linux/seq_file.h> | |
23 | #include <net/netlink.h> | |
7a7ffe65 HX |
24 | |
25 | #include "internal.h" | |
26 | ||
b286d8b1 HX |
27 | enum { |
28 | SKCIPHER_WALK_PHYS = 1 << 0, | |
29 | SKCIPHER_WALK_SLOW = 1 << 1, | |
30 | SKCIPHER_WALK_COPY = 1 << 2, | |
31 | SKCIPHER_WALK_DIFF = 1 << 3, | |
32 | SKCIPHER_WALK_SLEEP = 1 << 4, | |
33 | }; | |
34 | ||
35 | struct skcipher_walk_buffer { | |
36 | struct list_head entry; | |
37 | struct scatter_walk dst; | |
38 | unsigned int len; | |
39 | u8 *data; | |
40 | u8 buffer[]; | |
41 | }; | |
42 | ||
43 | static int skcipher_walk_next(struct skcipher_walk *walk); | |
44 | ||
b286d8b1 HX |
45 | static inline void skcipher_map_src(struct skcipher_walk *walk) |
46 | { | |
d07bd950 | 47 | walk->src.virt.addr = scatterwalk_map(&walk->in); |
b286d8b1 HX |
48 | } |
49 | ||
50 | static inline void skcipher_map_dst(struct skcipher_walk *walk) | |
51 | { | |
d07bd950 | 52 | walk->dst.virt.addr = scatterwalk_map(&walk->out); |
b286d8b1 HX |
53 | } |
54 | ||
55 | static inline void skcipher_unmap_src(struct skcipher_walk *walk) | |
56 | { | |
d07bd950 | 57 | scatterwalk_unmap(walk->src.virt.addr); |
b286d8b1 HX |
58 | } |
59 | ||
60 | static inline void skcipher_unmap_dst(struct skcipher_walk *walk) | |
61 | { | |
d07bd950 | 62 | scatterwalk_unmap(walk->dst.virt.addr); |
b286d8b1 HX |
63 | } |
64 | ||
65 | static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) | |
66 | { | |
67 | return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; | |
68 | } | |
69 | ||
70 | /* Get a spot of the specified length that does not straddle a page. | |
71 | * The caller needs to ensure that there is enough space for this operation. | |
72 | */ | |
73 | static inline u8 *skcipher_get_spot(u8 *start, unsigned int len) | |
74 | { | |
75 | u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); | |
76 | ||
77 | return max(start, end_page); | |
78 | } | |
79 | ||
0ba3c026 | 80 | static int skcipher_done_slow(struct skcipher_walk *walk, unsigned int bsize) |
b286d8b1 HX |
81 | { |
82 | u8 *addr; | |
83 | ||
84 | addr = (u8 *)ALIGN((unsigned long)walk->buffer, walk->alignmask + 1); | |
85 | addr = skcipher_get_spot(addr, bsize); | |
86 | scatterwalk_copychunks(addr, &walk->out, bsize, | |
87 | (walk->flags & SKCIPHER_WALK_PHYS) ? 2 : 1); | |
0ba3c026 | 88 | return 0; |
b286d8b1 HX |
89 | } |
90 | ||
91 | int skcipher_walk_done(struct skcipher_walk *walk, int err) | |
92 | { | |
0ba3c026 HX |
93 | unsigned int n = walk->nbytes; |
94 | unsigned int nbytes = 0; | |
8088d3dd | 95 | |
0ba3c026 | 96 | if (!n) |
8088d3dd EB |
97 | goto finish; |
98 | ||
0ba3c026 HX |
99 | if (likely(err >= 0)) { |
100 | n -= err; | |
101 | nbytes = walk->total - n; | |
102 | } | |
8088d3dd EB |
103 | |
104 | if (likely(!(walk->flags & (SKCIPHER_WALK_PHYS | | |
105 | SKCIPHER_WALK_SLOW | | |
106 | SKCIPHER_WALK_COPY | | |
107 | SKCIPHER_WALK_DIFF)))) { | |
b286d8b1 HX |
108 | unmap_src: |
109 | skcipher_unmap_src(walk); | |
110 | } else if (walk->flags & SKCIPHER_WALK_DIFF) { | |
111 | skcipher_unmap_dst(walk); | |
112 | goto unmap_src; | |
113 | } else if (walk->flags & SKCIPHER_WALK_COPY) { | |
114 | skcipher_map_dst(walk); | |
115 | memcpy(walk->dst.virt.addr, walk->page, n); | |
116 | skcipher_unmap_dst(walk); | |
117 | } else if (unlikely(walk->flags & SKCIPHER_WALK_SLOW)) { | |
0ba3c026 | 118 | if (err > 0) { |
dcaca01a EB |
119 | /* |
120 | * Didn't process all bytes. Either the algorithm is | |
121 | * broken, or this was the last step and it turned out | |
122 | * the message wasn't evenly divisible into blocks but | |
123 | * the algorithm requires it. | |
124 | */ | |
b286d8b1 | 125 | err = -EINVAL; |
0ba3c026 HX |
126 | nbytes = 0; |
127 | } else | |
128 | n = skcipher_done_slow(walk, n); | |
b286d8b1 HX |
129 | } |
130 | ||
0ba3c026 HX |
131 | if (err > 0) |
132 | err = 0; | |
133 | ||
134 | walk->total = nbytes; | |
135 | walk->nbytes = 0; | |
136 | ||
b286d8b1 HX |
137 | scatterwalk_advance(&walk->in, n); |
138 | scatterwalk_advance(&walk->out, n); | |
0ba3c026 HX |
139 | scatterwalk_done(&walk->in, 0, nbytes); |
140 | scatterwalk_done(&walk->out, 1, nbytes); | |
b286d8b1 | 141 | |
0ba3c026 | 142 | if (nbytes) { |
b286d8b1 HX |
143 | crypto_yield(walk->flags & SKCIPHER_WALK_SLEEP ? |
144 | CRYPTO_TFM_REQ_MAY_SLEEP : 0); | |
145 | return skcipher_walk_next(walk); | |
146 | } | |
147 | ||
0ba3c026 | 148 | finish: |
b286d8b1 HX |
149 | /* Short-circuit for the common/fast path. */ |
150 | if (!((unsigned long)walk->buffer | (unsigned long)walk->page)) | |
151 | goto out; | |
152 | ||
153 | if (walk->flags & SKCIPHER_WALK_PHYS) | |
154 | goto out; | |
155 | ||
156 | if (walk->iv != walk->oiv) | |
157 | memcpy(walk->oiv, walk->iv, walk->ivsize); | |
158 | if (walk->buffer != walk->page) | |
159 | kfree(walk->buffer); | |
160 | if (walk->page) | |
161 | free_page((unsigned long)walk->page); | |
162 | ||
163 | out: | |
164 | return err; | |
165 | } | |
166 | EXPORT_SYMBOL_GPL(skcipher_walk_done); | |
167 | ||
168 | void skcipher_walk_complete(struct skcipher_walk *walk, int err) | |
169 | { | |
170 | struct skcipher_walk_buffer *p, *tmp; | |
171 | ||
172 | list_for_each_entry_safe(p, tmp, &walk->buffers, entry) { | |
173 | u8 *data; | |
174 | ||
175 | if (err) | |
176 | goto done; | |
177 | ||
178 | data = p->data; | |
179 | if (!data) { | |
180 | data = PTR_ALIGN(&p->buffer[0], walk->alignmask + 1); | |
c821f6ab | 181 | data = skcipher_get_spot(data, walk->stride); |
b286d8b1 HX |
182 | } |
183 | ||
184 | scatterwalk_copychunks(data, &p->dst, p->len, 1); | |
185 | ||
c821f6ab | 186 | if (offset_in_page(p->data) + p->len + walk->stride > |
b286d8b1 HX |
187 | PAGE_SIZE) |
188 | free_page((unsigned long)p->data); | |
189 | ||
190 | done: | |
191 | list_del(&p->entry); | |
192 | kfree(p); | |
193 | } | |
194 | ||
195 | if (!err && walk->iv != walk->oiv) | |
196 | memcpy(walk->oiv, walk->iv, walk->ivsize); | |
197 | if (walk->buffer != walk->page) | |
198 | kfree(walk->buffer); | |
199 | if (walk->page) | |
200 | free_page((unsigned long)walk->page); | |
201 | } | |
202 | EXPORT_SYMBOL_GPL(skcipher_walk_complete); | |
203 | ||
204 | static void skcipher_queue_write(struct skcipher_walk *walk, | |
205 | struct skcipher_walk_buffer *p) | |
206 | { | |
207 | p->dst = walk->out; | |
208 | list_add_tail(&p->entry, &walk->buffers); | |
209 | } | |
210 | ||
211 | static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) | |
212 | { | |
213 | bool phys = walk->flags & SKCIPHER_WALK_PHYS; | |
214 | unsigned alignmask = walk->alignmask; | |
215 | struct skcipher_walk_buffer *p; | |
216 | unsigned a; | |
217 | unsigned n; | |
218 | u8 *buffer; | |
219 | void *v; | |
220 | ||
221 | if (!phys) { | |
18e615ad AB |
222 | if (!walk->buffer) |
223 | walk->buffer = walk->page; | |
224 | buffer = walk->buffer; | |
b286d8b1 HX |
225 | if (buffer) |
226 | goto ok; | |
227 | } | |
228 | ||
229 | /* Start with the minimum alignment of kmalloc. */ | |
230 | a = crypto_tfm_ctx_alignment() - 1; | |
231 | n = bsize; | |
232 | ||
233 | if (phys) { | |
234 | /* Calculate the minimum alignment of p->buffer. */ | |
235 | a &= (sizeof(*p) ^ (sizeof(*p) - 1)) >> 1; | |
236 | n += sizeof(*p); | |
237 | } | |
238 | ||
239 | /* Minimum size to align p->buffer by alignmask. */ | |
240 | n += alignmask & ~a; | |
241 | ||
242 | /* Minimum size to ensure p->buffer does not straddle a page. */ | |
243 | n += (bsize - 1) & ~(alignmask | a); | |
244 | ||
245 | v = kzalloc(n, skcipher_walk_gfp(walk)); | |
246 | if (!v) | |
247 | return skcipher_walk_done(walk, -ENOMEM); | |
248 | ||
249 | if (phys) { | |
250 | p = v; | |
251 | p->len = bsize; | |
252 | skcipher_queue_write(walk, p); | |
253 | buffer = p->buffer; | |
254 | } else { | |
255 | walk->buffer = v; | |
256 | buffer = v; | |
257 | } | |
258 | ||
259 | ok: | |
260 | walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1); | |
261 | walk->dst.virt.addr = skcipher_get_spot(walk->dst.virt.addr, bsize); | |
262 | walk->src.virt.addr = walk->dst.virt.addr; | |
263 | ||
264 | scatterwalk_copychunks(walk->src.virt.addr, &walk->in, bsize, 0); | |
265 | ||
266 | walk->nbytes = bsize; | |
267 | walk->flags |= SKCIPHER_WALK_SLOW; | |
268 | ||
269 | return 0; | |
270 | } | |
271 | ||
272 | static int skcipher_next_copy(struct skcipher_walk *walk) | |
273 | { | |
274 | struct skcipher_walk_buffer *p; | |
275 | u8 *tmp = walk->page; | |
276 | ||
277 | skcipher_map_src(walk); | |
278 | memcpy(tmp, walk->src.virt.addr, walk->nbytes); | |
279 | skcipher_unmap_src(walk); | |
280 | ||
281 | walk->src.virt.addr = tmp; | |
282 | walk->dst.virt.addr = tmp; | |
283 | ||
284 | if (!(walk->flags & SKCIPHER_WALK_PHYS)) | |
285 | return 0; | |
286 | ||
287 | p = kmalloc(sizeof(*p), skcipher_walk_gfp(walk)); | |
288 | if (!p) | |
289 | return -ENOMEM; | |
290 | ||
291 | p->data = walk->page; | |
292 | p->len = walk->nbytes; | |
293 | skcipher_queue_write(walk, p); | |
294 | ||
c821f6ab | 295 | if (offset_in_page(walk->page) + walk->nbytes + walk->stride > |
b286d8b1 HX |
296 | PAGE_SIZE) |
297 | walk->page = NULL; | |
298 | else | |
299 | walk->page += walk->nbytes; | |
300 | ||
301 | return 0; | |
302 | } | |
303 | ||
304 | static int skcipher_next_fast(struct skcipher_walk *walk) | |
305 | { | |
306 | unsigned long diff; | |
307 | ||
308 | walk->src.phys.page = scatterwalk_page(&walk->in); | |
309 | walk->src.phys.offset = offset_in_page(walk->in.offset); | |
310 | walk->dst.phys.page = scatterwalk_page(&walk->out); | |
311 | walk->dst.phys.offset = offset_in_page(walk->out.offset); | |
312 | ||
313 | if (walk->flags & SKCIPHER_WALK_PHYS) | |
314 | return 0; | |
315 | ||
316 | diff = walk->src.phys.offset - walk->dst.phys.offset; | |
317 | diff |= walk->src.virt.page - walk->dst.virt.page; | |
318 | ||
319 | skcipher_map_src(walk); | |
320 | walk->dst.virt.addr = walk->src.virt.addr; | |
321 | ||
322 | if (diff) { | |
323 | walk->flags |= SKCIPHER_WALK_DIFF; | |
324 | skcipher_map_dst(walk); | |
325 | } | |
326 | ||
327 | return 0; | |
328 | } | |
329 | ||
330 | static int skcipher_walk_next(struct skcipher_walk *walk) | |
331 | { | |
332 | unsigned int bsize; | |
333 | unsigned int n; | |
334 | int err; | |
335 | ||
336 | walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | | |
337 | SKCIPHER_WALK_DIFF); | |
338 | ||
339 | n = walk->total; | |
c821f6ab | 340 | bsize = min(walk->stride, max(n, walk->blocksize)); |
b286d8b1 HX |
341 | n = scatterwalk_clamp(&walk->in, n); |
342 | n = scatterwalk_clamp(&walk->out, n); | |
343 | ||
344 | if (unlikely(n < bsize)) { | |
345 | if (unlikely(walk->total < walk->blocksize)) | |
346 | return skcipher_walk_done(walk, -EINVAL); | |
347 | ||
348 | slow_path: | |
349 | err = skcipher_next_slow(walk, bsize); | |
350 | goto set_phys_lowmem; | |
351 | } | |
352 | ||
353 | if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) { | |
354 | if (!walk->page) { | |
355 | gfp_t gfp = skcipher_walk_gfp(walk); | |
356 | ||
357 | walk->page = (void *)__get_free_page(gfp); | |
358 | if (!walk->page) | |
359 | goto slow_path; | |
360 | } | |
361 | ||
362 | walk->nbytes = min_t(unsigned, n, | |
363 | PAGE_SIZE - offset_in_page(walk->page)); | |
364 | walk->flags |= SKCIPHER_WALK_COPY; | |
365 | err = skcipher_next_copy(walk); | |
366 | goto set_phys_lowmem; | |
367 | } | |
368 | ||
369 | walk->nbytes = n; | |
370 | ||
371 | return skcipher_next_fast(walk); | |
372 | ||
373 | set_phys_lowmem: | |
374 | if (!err && (walk->flags & SKCIPHER_WALK_PHYS)) { | |
375 | walk->src.phys.page = virt_to_page(walk->src.virt.addr); | |
376 | walk->dst.phys.page = virt_to_page(walk->dst.virt.addr); | |
377 | walk->src.phys.offset &= PAGE_SIZE - 1; | |
378 | walk->dst.phys.offset &= PAGE_SIZE - 1; | |
379 | } | |
380 | return err; | |
381 | } | |
b286d8b1 HX |
382 | |
383 | static int skcipher_copy_iv(struct skcipher_walk *walk) | |
384 | { | |
385 | unsigned a = crypto_tfm_ctx_alignment() - 1; | |
386 | unsigned alignmask = walk->alignmask; | |
387 | unsigned ivsize = walk->ivsize; | |
c821f6ab | 388 | unsigned bs = walk->stride; |
b286d8b1 HX |
389 | unsigned aligned_bs; |
390 | unsigned size; | |
391 | u8 *iv; | |
392 | ||
0567fc9e | 393 | aligned_bs = ALIGN(bs, alignmask + 1); |
b286d8b1 HX |
394 | |
395 | /* Minimum size to align buffer by alignmask. */ | |
396 | size = alignmask & ~a; | |
397 | ||
398 | if (walk->flags & SKCIPHER_WALK_PHYS) | |
399 | size += ivsize; | |
400 | else { | |
401 | size += aligned_bs + ivsize; | |
402 | ||
403 | /* Minimum size to ensure buffer does not straddle a page. */ | |
404 | size += (bs - 1) & ~(alignmask | a); | |
405 | } | |
406 | ||
407 | walk->buffer = kmalloc(size, skcipher_walk_gfp(walk)); | |
408 | if (!walk->buffer) | |
409 | return -ENOMEM; | |
410 | ||
411 | iv = PTR_ALIGN(walk->buffer, alignmask + 1); | |
412 | iv = skcipher_get_spot(iv, bs) + aligned_bs; | |
413 | ||
414 | walk->iv = memcpy(iv, walk->iv, walk->ivsize); | |
415 | return 0; | |
416 | } | |
417 | ||
418 | static int skcipher_walk_first(struct skcipher_walk *walk) | |
419 | { | |
abfc7fad | 420 | if (WARN_ON_ONCE(in_hardirq())) |
b286d8b1 HX |
421 | return -EDEADLK; |
422 | ||
b286d8b1 HX |
423 | walk->buffer = NULL; |
424 | if (unlikely(((unsigned long)walk->iv & walk->alignmask))) { | |
425 | int err = skcipher_copy_iv(walk); | |
426 | if (err) | |
427 | return err; | |
428 | } | |
429 | ||
430 | walk->page = NULL; | |
b286d8b1 HX |
431 | |
432 | return skcipher_walk_next(walk); | |
433 | } | |
434 | ||
435 | static int skcipher_walk_skcipher(struct skcipher_walk *walk, | |
436 | struct skcipher_request *req) | |
437 | { | |
438 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
439 | ||
0cabf2af HX |
440 | walk->total = req->cryptlen; |
441 | walk->nbytes = 0; | |
2b4f27c3 EB |
442 | walk->iv = req->iv; |
443 | walk->oiv = req->iv; | |
0cabf2af HX |
444 | |
445 | if (unlikely(!walk->total)) | |
446 | return 0; | |
447 | ||
b286d8b1 HX |
448 | scatterwalk_start(&walk->in, req->src); |
449 | scatterwalk_start(&walk->out, req->dst); | |
450 | ||
b286d8b1 HX |
451 | walk->flags &= ~SKCIPHER_WALK_SLEEP; |
452 | walk->flags |= req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? | |
453 | SKCIPHER_WALK_SLEEP : 0; | |
454 | ||
455 | walk->blocksize = crypto_skcipher_blocksize(tfm); | |
c821f6ab | 456 | walk->stride = crypto_skcipher_walksize(tfm); |
b286d8b1 HX |
457 | walk->ivsize = crypto_skcipher_ivsize(tfm); |
458 | walk->alignmask = crypto_skcipher_alignmask(tfm); | |
459 | ||
460 | return skcipher_walk_first(walk); | |
461 | } | |
462 | ||
463 | int skcipher_walk_virt(struct skcipher_walk *walk, | |
464 | struct skcipher_request *req, bool atomic) | |
465 | { | |
466 | int err; | |
467 | ||
bb648291 EB |
468 | might_sleep_if(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); |
469 | ||
b286d8b1 HX |
470 | walk->flags &= ~SKCIPHER_WALK_PHYS; |
471 | ||
472 | err = skcipher_walk_skcipher(walk, req); | |
473 | ||
474 | walk->flags &= atomic ? ~SKCIPHER_WALK_SLEEP : ~0; | |
475 | ||
476 | return err; | |
477 | } | |
478 | EXPORT_SYMBOL_GPL(skcipher_walk_virt); | |
479 | ||
b286d8b1 HX |
480 | int skcipher_walk_async(struct skcipher_walk *walk, |
481 | struct skcipher_request *req) | |
482 | { | |
483 | walk->flags |= SKCIPHER_WALK_PHYS; | |
484 | ||
485 | INIT_LIST_HEAD(&walk->buffers); | |
486 | ||
487 | return skcipher_walk_skcipher(walk, req); | |
488 | } | |
489 | EXPORT_SYMBOL_GPL(skcipher_walk_async); | |
490 | ||
34bc085c HX |
491 | static int skcipher_walk_aead_common(struct skcipher_walk *walk, |
492 | struct aead_request *req, bool atomic) | |
b286d8b1 HX |
493 | { |
494 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
495 | int err; | |
496 | ||
0cabf2af | 497 | walk->nbytes = 0; |
2b4f27c3 EB |
498 | walk->iv = req->iv; |
499 | walk->oiv = req->iv; | |
0cabf2af HX |
500 | |
501 | if (unlikely(!walk->total)) | |
502 | return 0; | |
503 | ||
3cbf61fb AB |
504 | walk->flags &= ~SKCIPHER_WALK_PHYS; |
505 | ||
b286d8b1 HX |
506 | scatterwalk_start(&walk->in, req->src); |
507 | scatterwalk_start(&walk->out, req->dst); | |
508 | ||
509 | scatterwalk_copychunks(NULL, &walk->in, req->assoclen, 2); | |
510 | scatterwalk_copychunks(NULL, &walk->out, req->assoclen, 2); | |
511 | ||
c14ca838 OM |
512 | scatterwalk_done(&walk->in, 0, walk->total); |
513 | scatterwalk_done(&walk->out, 0, walk->total); | |
514 | ||
b286d8b1 HX |
515 | if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) |
516 | walk->flags |= SKCIPHER_WALK_SLEEP; | |
517 | else | |
518 | walk->flags &= ~SKCIPHER_WALK_SLEEP; | |
519 | ||
520 | walk->blocksize = crypto_aead_blocksize(tfm); | |
c821f6ab | 521 | walk->stride = crypto_aead_chunksize(tfm); |
b286d8b1 HX |
522 | walk->ivsize = crypto_aead_ivsize(tfm); |
523 | walk->alignmask = crypto_aead_alignmask(tfm); | |
524 | ||
525 | err = skcipher_walk_first(walk); | |
526 | ||
527 | if (atomic) | |
528 | walk->flags &= ~SKCIPHER_WALK_SLEEP; | |
529 | ||
530 | return err; | |
531 | } | |
34bc085c | 532 | |
34bc085c HX |
533 | int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, |
534 | struct aead_request *req, bool atomic) | |
535 | { | |
536 | walk->total = req->cryptlen; | |
537 | ||
538 | return skcipher_walk_aead_common(walk, req, atomic); | |
539 | } | |
540 | EXPORT_SYMBOL_GPL(skcipher_walk_aead_encrypt); | |
541 | ||
542 | int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, | |
543 | struct aead_request *req, bool atomic) | |
544 | { | |
545 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
546 | ||
547 | walk->total = req->cryptlen - crypto_aead_authsize(tfm); | |
548 | ||
549 | return skcipher_walk_aead_common(walk, req, atomic); | |
550 | } | |
551 | EXPORT_SYMBOL_GPL(skcipher_walk_aead_decrypt); | |
552 | ||
b1f6b4bf EB |
553 | static void skcipher_set_needkey(struct crypto_skcipher *tfm) |
554 | { | |
9ac0d136 | 555 | if (crypto_skcipher_max_keysize(tfm) != 0) |
b1f6b4bf EB |
556 | crypto_skcipher_set_flags(tfm, CRYPTO_TFM_NEED_KEY); |
557 | } | |
558 | ||
9933e113 HX |
559 | static int skcipher_setkey_unaligned(struct crypto_skcipher *tfm, |
560 | const u8 *key, unsigned int keylen) | |
561 | { | |
562 | unsigned long alignmask = crypto_skcipher_alignmask(tfm); | |
563 | struct skcipher_alg *cipher = crypto_skcipher_alg(tfm); | |
564 | u8 *buffer, *alignbuffer; | |
565 | unsigned long absize; | |
566 | int ret; | |
567 | ||
568 | absize = keylen + alignmask; | |
569 | buffer = kmalloc(absize, GFP_ATOMIC); | |
570 | if (!buffer) | |
571 | return -ENOMEM; | |
572 | ||
573 | alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); | |
574 | memcpy(alignbuffer, key, keylen); | |
575 | ret = cipher->setkey(tfm, alignbuffer, keylen); | |
453431a5 | 576 | kfree_sensitive(buffer); |
9933e113 HX |
577 | return ret; |
578 | } | |
579 | ||
15252d94 | 580 | int crypto_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, |
9933e113 HX |
581 | unsigned int keylen) |
582 | { | |
583 | struct skcipher_alg *cipher = crypto_skcipher_alg(tfm); | |
584 | unsigned long alignmask = crypto_skcipher_alignmask(tfm); | |
f8d33fac | 585 | int err; |
9933e113 | 586 | |
674f368a | 587 | if (keylen < cipher->min_keysize || keylen > cipher->max_keysize) |
9933e113 | 588 | return -EINVAL; |
9933e113 HX |
589 | |
590 | if ((unsigned long)key & alignmask) | |
f8d33fac EB |
591 | err = skcipher_setkey_unaligned(tfm, key, keylen); |
592 | else | |
593 | err = cipher->setkey(tfm, key, keylen); | |
594 | ||
b1f6b4bf EB |
595 | if (unlikely(err)) { |
596 | skcipher_set_needkey(tfm); | |
f8d33fac | 597 | return err; |
b1f6b4bf | 598 | } |
9933e113 | 599 | |
f8d33fac EB |
600 | crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY); |
601 | return 0; | |
9933e113 | 602 | } |
15252d94 | 603 | EXPORT_SYMBOL_GPL(crypto_skcipher_setkey); |
9933e113 | 604 | |
81bcbb1e EB |
605 | int crypto_skcipher_encrypt(struct skcipher_request *req) |
606 | { | |
607 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
608 | struct crypto_alg *alg = tfm->base.__crt_alg; | |
609 | unsigned int cryptlen = req->cryptlen; | |
610 | int ret; | |
611 | ||
612 | crypto_stats_get(alg); | |
613 | if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) | |
614 | ret = -ENOKEY; | |
615 | else | |
848755e3 | 616 | ret = crypto_skcipher_alg(tfm)->encrypt(req); |
81bcbb1e EB |
617 | crypto_stats_skcipher_encrypt(cryptlen, ret, alg); |
618 | return ret; | |
619 | } | |
620 | EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt); | |
621 | ||
622 | int crypto_skcipher_decrypt(struct skcipher_request *req) | |
623 | { | |
624 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
625 | struct crypto_alg *alg = tfm->base.__crt_alg; | |
626 | unsigned int cryptlen = req->cryptlen; | |
627 | int ret; | |
628 | ||
629 | crypto_stats_get(alg); | |
630 | if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) | |
631 | ret = -ENOKEY; | |
632 | else | |
7e1c1099 | 633 | ret = crypto_skcipher_alg(tfm)->decrypt(req); |
81bcbb1e EB |
634 | crypto_stats_skcipher_decrypt(cryptlen, ret, alg); |
635 | return ret; | |
636 | } | |
637 | EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt); | |
638 | ||
4e6c3df4 HX |
639 | static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) |
640 | { | |
641 | struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); | |
642 | struct skcipher_alg *alg = crypto_skcipher_alg(skcipher); | |
643 | ||
644 | alg->exit(skcipher); | |
645 | } | |
646 | ||
7a7ffe65 HX |
647 | static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm) |
648 | { | |
4e6c3df4 HX |
649 | struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); |
650 | struct skcipher_alg *alg = crypto_skcipher_alg(skcipher); | |
651 | ||
b1f6b4bf | 652 | skcipher_set_needkey(skcipher); |
f8d33fac | 653 | |
4e6c3df4 HX |
654 | if (alg->exit) |
655 | skcipher->base.exit = crypto_skcipher_exit_tfm; | |
7a7ffe65 | 656 | |
4e6c3df4 HX |
657 | if (alg->init) |
658 | return alg->init(skcipher); | |
659 | ||
660 | return 0; | |
661 | } | |
662 | ||
663 | static void crypto_skcipher_free_instance(struct crypto_instance *inst) | |
664 | { | |
665 | struct skcipher_instance *skcipher = | |
666 | container_of(inst, struct skcipher_instance, s.base); | |
667 | ||
668 | skcipher->free(skcipher); | |
669 | } | |
670 | ||
671 | static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) | |
d8c34b94 | 672 | __maybe_unused; |
4e6c3df4 HX |
673 | static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) |
674 | { | |
675 | struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg, | |
676 | base); | |
677 | ||
678 | seq_printf(m, "type : skcipher\n"); | |
679 | seq_printf(m, "async : %s\n", | |
680 | alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no"); | |
681 | seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); | |
682 | seq_printf(m, "min keysize : %u\n", skcipher->min_keysize); | |
683 | seq_printf(m, "max keysize : %u\n", skcipher->max_keysize); | |
684 | seq_printf(m, "ivsize : %u\n", skcipher->ivsize); | |
685 | seq_printf(m, "chunksize : %u\n", skcipher->chunksize); | |
c821f6ab | 686 | seq_printf(m, "walksize : %u\n", skcipher->walksize); |
7a7ffe65 HX |
687 | } |
688 | ||
4e6c3df4 HX |
689 | #ifdef CONFIG_NET |
690 | static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg) | |
691 | { | |
692 | struct crypto_report_blkcipher rblkcipher; | |
693 | struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg, | |
694 | base); | |
695 | ||
37db69e0 EB |
696 | memset(&rblkcipher, 0, sizeof(rblkcipher)); |
697 | ||
698 | strscpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type)); | |
699 | strscpy(rblkcipher.geniv, "<none>", sizeof(rblkcipher.geniv)); | |
4e6c3df4 HX |
700 | |
701 | rblkcipher.blocksize = alg->cra_blocksize; | |
702 | rblkcipher.min_keysize = skcipher->min_keysize; | |
703 | rblkcipher.max_keysize = skcipher->max_keysize; | |
704 | rblkcipher.ivsize = skcipher->ivsize; | |
705 | ||
37db69e0 EB |
706 | return nla_put(skb, CRYPTOCFGA_REPORT_BLKCIPHER, |
707 | sizeof(rblkcipher), &rblkcipher); | |
4e6c3df4 HX |
708 | } |
709 | #else | |
710 | static int crypto_skcipher_report(struct sk_buff *skb, struct crypto_alg *alg) | |
711 | { | |
712 | return -ENOSYS; | |
713 | } | |
714 | #endif | |
715 | ||
53253064 | 716 | static const struct crypto_type crypto_skcipher_type = { |
89873b44 | 717 | .extsize = crypto_alg_extsize, |
7a7ffe65 | 718 | .init_tfm = crypto_skcipher_init_tfm, |
4e6c3df4 HX |
719 | .free = crypto_skcipher_free_instance, |
720 | #ifdef CONFIG_PROC_FS | |
721 | .show = crypto_skcipher_show, | |
722 | #endif | |
723 | .report = crypto_skcipher_report, | |
7a7ffe65 | 724 | .maskclear = ~CRYPTO_ALG_TYPE_MASK, |
c65058b7 | 725 | .maskset = CRYPTO_ALG_TYPE_MASK, |
4e6c3df4 | 726 | .type = CRYPTO_ALG_TYPE_SKCIPHER, |
7a7ffe65 HX |
727 | .tfmsize = offsetof(struct crypto_skcipher, base), |
728 | }; | |
729 | ||
3a01d0ee | 730 | int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, |
b9f76ddd EB |
731 | struct crypto_instance *inst, |
732 | const char *name, u32 type, u32 mask) | |
4e6c3df4 | 733 | { |
53253064 | 734 | spawn->base.frontend = &crypto_skcipher_type; |
de95c957 | 735 | return crypto_grab_spawn(&spawn->base, inst, name, type, mask); |
4e6c3df4 | 736 | } |
3a01d0ee | 737 | EXPORT_SYMBOL_GPL(crypto_grab_skcipher); |
4e6c3df4 | 738 | |
7a7ffe65 HX |
739 | struct crypto_skcipher *crypto_alloc_skcipher(const char *alg_name, |
740 | u32 type, u32 mask) | |
741 | { | |
53253064 | 742 | return crypto_alloc_tfm(alg_name, &crypto_skcipher_type, type, mask); |
7a7ffe65 HX |
743 | } |
744 | EXPORT_SYMBOL_GPL(crypto_alloc_skcipher); | |
745 | ||
b350bee5 KC |
746 | struct crypto_sync_skcipher *crypto_alloc_sync_skcipher( |
747 | const char *alg_name, u32 type, u32 mask) | |
748 | { | |
749 | struct crypto_skcipher *tfm; | |
750 | ||
751 | /* Only sync algorithms allowed. */ | |
e6cb02bd | 752 | mask |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE; |
b350bee5 | 753 | |
53253064 | 754 | tfm = crypto_alloc_tfm(alg_name, &crypto_skcipher_type, type, mask); |
b350bee5 KC |
755 | |
756 | /* | |
757 | * Make sure we do not allocate something that might get used with | |
758 | * an on-stack request: check the request size. | |
759 | */ | |
760 | if (!IS_ERR(tfm) && WARN_ON(crypto_skcipher_reqsize(tfm) > | |
761 | MAX_SYNC_SKCIPHER_REQSIZE)) { | |
762 | crypto_free_skcipher(tfm); | |
763 | return ERR_PTR(-EINVAL); | |
764 | } | |
765 | ||
766 | return (struct crypto_sync_skcipher *)tfm; | |
767 | } | |
768 | EXPORT_SYMBOL_GPL(crypto_alloc_sync_skcipher); | |
769 | ||
d3ca75a8 | 770 | int crypto_has_skcipher(const char *alg_name, u32 type, u32 mask) |
4e6c3df4 | 771 | { |
53253064 | 772 | return crypto_type_has_alg(alg_name, &crypto_skcipher_type, type, mask); |
4e6c3df4 | 773 | } |
d3ca75a8 | 774 | EXPORT_SYMBOL_GPL(crypto_has_skcipher); |
4e6c3df4 HX |
775 | |
776 | static int skcipher_prepare_alg(struct skcipher_alg *alg) | |
777 | { | |
778 | struct crypto_alg *base = &alg->base; | |
779 | ||
c821f6ab AB |
780 | if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8 || |
781 | alg->walksize > PAGE_SIZE / 8) | |
4e6c3df4 HX |
782 | return -EINVAL; |
783 | ||
784 | if (!alg->chunksize) | |
785 | alg->chunksize = base->cra_blocksize; | |
c821f6ab AB |
786 | if (!alg->walksize) |
787 | alg->walksize = alg->chunksize; | |
4e6c3df4 | 788 | |
53253064 | 789 | base->cra_type = &crypto_skcipher_type; |
4e6c3df4 HX |
790 | base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; |
791 | base->cra_flags |= CRYPTO_ALG_TYPE_SKCIPHER; | |
792 | ||
793 | return 0; | |
794 | } | |
795 | ||
796 | int crypto_register_skcipher(struct skcipher_alg *alg) | |
797 | { | |
798 | struct crypto_alg *base = &alg->base; | |
799 | int err; | |
800 | ||
801 | err = skcipher_prepare_alg(alg); | |
802 | if (err) | |
803 | return err; | |
804 | ||
805 | return crypto_register_alg(base); | |
806 | } | |
807 | EXPORT_SYMBOL_GPL(crypto_register_skcipher); | |
808 | ||
809 | void crypto_unregister_skcipher(struct skcipher_alg *alg) | |
810 | { | |
811 | crypto_unregister_alg(&alg->base); | |
812 | } | |
813 | EXPORT_SYMBOL_GPL(crypto_unregister_skcipher); | |
814 | ||
815 | int crypto_register_skciphers(struct skcipher_alg *algs, int count) | |
816 | { | |
817 | int i, ret; | |
818 | ||
819 | for (i = 0; i < count; i++) { | |
820 | ret = crypto_register_skcipher(&algs[i]); | |
821 | if (ret) | |
822 | goto err; | |
823 | } | |
824 | ||
825 | return 0; | |
826 | ||
827 | err: | |
828 | for (--i; i >= 0; --i) | |
829 | crypto_unregister_skcipher(&algs[i]); | |
830 | ||
831 | return ret; | |
832 | } | |
833 | EXPORT_SYMBOL_GPL(crypto_register_skciphers); | |
834 | ||
835 | void crypto_unregister_skciphers(struct skcipher_alg *algs, int count) | |
836 | { | |
837 | int i; | |
838 | ||
839 | for (i = count - 1; i >= 0; --i) | |
840 | crypto_unregister_skcipher(&algs[i]); | |
841 | } | |
842 | EXPORT_SYMBOL_GPL(crypto_unregister_skciphers); | |
843 | ||
844 | int skcipher_register_instance(struct crypto_template *tmpl, | |
845 | struct skcipher_instance *inst) | |
846 | { | |
847 | int err; | |
848 | ||
d4fdc2df EB |
849 | if (WARN_ON(!inst->free)) |
850 | return -EINVAL; | |
851 | ||
4e6c3df4 HX |
852 | err = skcipher_prepare_alg(&inst->alg); |
853 | if (err) | |
854 | return err; | |
855 | ||
856 | return crypto_register_instance(tmpl, skcipher_crypto_instance(inst)); | |
857 | } | |
858 | EXPORT_SYMBOL_GPL(skcipher_register_instance); | |
859 | ||
0872da16 EB |
860 | static int skcipher_setkey_simple(struct crypto_skcipher *tfm, const u8 *key, |
861 | unsigned int keylen) | |
862 | { | |
863 | struct crypto_cipher *cipher = skcipher_cipher_simple(tfm); | |
0872da16 EB |
864 | |
865 | crypto_cipher_clear_flags(cipher, CRYPTO_TFM_REQ_MASK); | |
866 | crypto_cipher_set_flags(cipher, crypto_skcipher_get_flags(tfm) & | |
867 | CRYPTO_TFM_REQ_MASK); | |
af5034e8 | 868 | return crypto_cipher_setkey(cipher, key, keylen); |
0872da16 EB |
869 | } |
870 | ||
871 | static int skcipher_init_tfm_simple(struct crypto_skcipher *tfm) | |
872 | { | |
873 | struct skcipher_instance *inst = skcipher_alg_instance(tfm); | |
d5ed3b65 | 874 | struct crypto_cipher_spawn *spawn = skcipher_instance_ctx(inst); |
0872da16 EB |
875 | struct skcipher_ctx_simple *ctx = crypto_skcipher_ctx(tfm); |
876 | struct crypto_cipher *cipher; | |
877 | ||
878 | cipher = crypto_spawn_cipher(spawn); | |
879 | if (IS_ERR(cipher)) | |
880 | return PTR_ERR(cipher); | |
881 | ||
882 | ctx->cipher = cipher; | |
883 | return 0; | |
884 | } | |
885 | ||
886 | static void skcipher_exit_tfm_simple(struct crypto_skcipher *tfm) | |
887 | { | |
888 | struct skcipher_ctx_simple *ctx = crypto_skcipher_ctx(tfm); | |
889 | ||
890 | crypto_free_cipher(ctx->cipher); | |
891 | } | |
892 | ||
893 | static void skcipher_free_instance_simple(struct skcipher_instance *inst) | |
894 | { | |
aacd5b4c | 895 | crypto_drop_cipher(skcipher_instance_ctx(inst)); |
0872da16 EB |
896 | kfree(inst); |
897 | } | |
898 | ||
899 | /** | |
900 | * skcipher_alloc_instance_simple - allocate instance of simple block cipher mode | |
901 | * | |
902 | * Allocate an skcipher_instance for a simple block cipher mode of operation, | |
903 | * e.g. cbc or ecb. The instance context will have just a single crypto_spawn, | |
904 | * that for the underlying cipher. The {min,max}_keysize, ivsize, blocksize, | |
905 | * alignmask, and priority are set from the underlying cipher but can be | |
906 | * overridden if needed. The tfm context defaults to skcipher_ctx_simple, and | |
907 | * default ->setkey(), ->init(), and ->exit() methods are installed. | |
908 | * | |
909 | * @tmpl: the template being instantiated | |
910 | * @tb: the template parameters | |
0872da16 EB |
911 | * |
912 | * Return: a pointer to the new instance, or an ERR_PTR(). The caller still | |
913 | * needs to register the instance. | |
914 | */ | |
b3c16bfc HX |
915 | struct skcipher_instance *skcipher_alloc_instance_simple( |
916 | struct crypto_template *tmpl, struct rtattr **tb) | |
0872da16 | 917 | { |
0872da16 | 918 | u32 mask; |
aacd5b4c EB |
919 | struct skcipher_instance *inst; |
920 | struct crypto_cipher_spawn *spawn; | |
921 | struct crypto_alg *cipher_alg; | |
0872da16 EB |
922 | int err; |
923 | ||
7bcb2c99 EB |
924 | err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask); |
925 | if (err) | |
926 | return ERR_PTR(err); | |
0872da16 EB |
927 | |
928 | inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); | |
aacd5b4c EB |
929 | if (!inst) |
930 | return ERR_PTR(-ENOMEM); | |
0872da16 EB |
931 | spawn = skcipher_instance_ctx(inst); |
932 | ||
aacd5b4c EB |
933 | err = crypto_grab_cipher(spawn, skcipher_crypto_instance(inst), |
934 | crypto_attr_alg_name(tb[1]), 0, mask); | |
0872da16 EB |
935 | if (err) |
936 | goto err_free_inst; | |
aacd5b4c | 937 | cipher_alg = crypto_spawn_cipher_alg(spawn); |
0872da16 | 938 | |
aacd5b4c EB |
939 | err = crypto_inst_setname(skcipher_crypto_instance(inst), tmpl->name, |
940 | cipher_alg); | |
0872da16 EB |
941 | if (err) |
942 | goto err_free_inst; | |
aacd5b4c | 943 | |
0872da16 EB |
944 | inst->free = skcipher_free_instance_simple; |
945 | ||
946 | /* Default algorithm properties, can be overridden */ | |
947 | inst->alg.base.cra_blocksize = cipher_alg->cra_blocksize; | |
948 | inst->alg.base.cra_alignmask = cipher_alg->cra_alignmask; | |
949 | inst->alg.base.cra_priority = cipher_alg->cra_priority; | |
950 | inst->alg.min_keysize = cipher_alg->cra_cipher.cia_min_keysize; | |
951 | inst->alg.max_keysize = cipher_alg->cra_cipher.cia_max_keysize; | |
952 | inst->alg.ivsize = cipher_alg->cra_blocksize; | |
953 | ||
954 | /* Use skcipher_ctx_simple by default, can be overridden */ | |
955 | inst->alg.base.cra_ctxsize = sizeof(struct skcipher_ctx_simple); | |
956 | inst->alg.setkey = skcipher_setkey_simple; | |
957 | inst->alg.init = skcipher_init_tfm_simple; | |
958 | inst->alg.exit = skcipher_exit_tfm_simple; | |
959 | ||
0872da16 EB |
960 | return inst; |
961 | ||
962 | err_free_inst: | |
aacd5b4c | 963 | skcipher_free_instance_simple(inst); |
0872da16 EB |
964 | return ERR_PTR(err); |
965 | } | |
966 | EXPORT_SYMBOL_GPL(skcipher_alloc_instance_simple); | |
967 | ||
7a7ffe65 HX |
968 | MODULE_LICENSE("GPL"); |
969 | MODULE_DESCRIPTION("Symmetric key cipher type"); | |
0eb76ba2 | 970 | MODULE_IMPORT_NS(CRYPTO_INTERNAL); |