Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
124b53d0 HX |
2 | /* |
3 | * Software async crypto daemon. | |
4 | * | |
5 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
6 | * | |
298c926c AH |
7 | * Added AEAD support to cryptd. |
8 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
9 | * Adrian Hoban <adrian.hoban@intel.com> | |
10 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
11 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
12 | * Copyright (c) 2010, Intel Corporation. | |
124b53d0 HX |
13 | */ |
14 | ||
18e33e6d | 15 | #include <crypto/internal/hash.h> |
298c926c | 16 | #include <crypto/internal/aead.h> |
4e0958d1 | 17 | #include <crypto/internal/skcipher.h> |
1cac2cbc | 18 | #include <crypto/cryptd.h> |
43b970fa | 19 | #include <linux/refcount.h> |
124b53d0 HX |
20 | #include <linux/err.h> |
21 | #include <linux/init.h> | |
22 | #include <linux/kernel.h> | |
124b53d0 HX |
23 | #include <linux/list.h> |
24 | #include <linux/module.h> | |
124b53d0 HX |
25 | #include <linux/scatterlist.h> |
26 | #include <linux/sched.h> | |
27 | #include <linux/slab.h> | |
3e56e168 | 28 | #include <linux/workqueue.h> |
124b53d0 | 29 | |
eaf356e4 | 30 | static unsigned int cryptd_max_cpu_qlen = 1000; |
c3a53605 JM |
31 | module_param(cryptd_max_cpu_qlen, uint, 0); |
32 | MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); | |
124b53d0 | 33 | |
3e56e168 EB |
34 | static struct workqueue_struct *cryptd_wq; |
35 | ||
254eff77 | 36 | struct cryptd_cpu_queue { |
124b53d0 | 37 | struct crypto_queue queue; |
254eff77 HY |
38 | struct work_struct work; |
39 | }; | |
40 | ||
41 | struct cryptd_queue { | |
91e8bcd7 SAS |
42 | /* |
43 | * Protected by disabling BH to allow enqueueing from softinterrupt and | |
44 | * dequeuing from kworker (cryptd_queue_worker()). | |
45 | */ | |
a29d8b8e | 46 | struct cryptd_cpu_queue __percpu *cpu_queue; |
124b53d0 HX |
47 | }; |
48 | ||
49 | struct cryptd_instance_ctx { | |
50 | struct crypto_spawn spawn; | |
254eff77 | 51 | struct cryptd_queue *queue; |
124b53d0 HX |
52 | }; |
53 | ||
4e0958d1 HX |
54 | struct skcipherd_instance_ctx { |
55 | struct crypto_skcipher_spawn spawn; | |
56 | struct cryptd_queue *queue; | |
57 | }; | |
58 | ||
46309d89 HX |
59 | struct hashd_instance_ctx { |
60 | struct crypto_shash_spawn spawn; | |
61 | struct cryptd_queue *queue; | |
62 | }; | |
63 | ||
298c926c AH |
64 | struct aead_instance_ctx { |
65 | struct crypto_aead_spawn aead_spawn; | |
66 | struct cryptd_queue *queue; | |
67 | }; | |
68 | ||
4e0958d1 | 69 | struct cryptd_skcipher_ctx { |
43b970fa | 70 | refcount_t refcnt; |
36b3875a | 71 | struct crypto_sync_skcipher *child; |
4e0958d1 HX |
72 | }; |
73 | ||
74 | struct cryptd_skcipher_request_ctx { | |
75 | crypto_completion_t complete; | |
76 | }; | |
77 | ||
b8a28251 | 78 | struct cryptd_hash_ctx { |
43b970fa | 79 | refcount_t refcnt; |
46309d89 | 80 | struct crypto_shash *child; |
b8a28251 LH |
81 | }; |
82 | ||
83 | struct cryptd_hash_request_ctx { | |
84 | crypto_completion_t complete; | |
46309d89 | 85 | struct shash_desc desc; |
b8a28251 | 86 | }; |
124b53d0 | 87 | |
298c926c | 88 | struct cryptd_aead_ctx { |
43b970fa | 89 | refcount_t refcnt; |
298c926c AH |
90 | struct crypto_aead *child; |
91 | }; | |
92 | ||
93 | struct cryptd_aead_request_ctx { | |
94 | crypto_completion_t complete; | |
95 | }; | |
96 | ||
254eff77 HY |
97 | static void cryptd_queue_worker(struct work_struct *work); |
98 | ||
99 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
100 | unsigned int max_cpu_qlen) | |
101 | { | |
102 | int cpu; | |
103 | struct cryptd_cpu_queue *cpu_queue; | |
104 | ||
105 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
106 | if (!queue->cpu_queue) | |
107 | return -ENOMEM; | |
108 | for_each_possible_cpu(cpu) { | |
109 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
110 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
111 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
112 | } | |
c3a53605 | 113 | pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); |
254eff77 HY |
114 | return 0; |
115 | } | |
116 | ||
117 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
118 | { | |
119 | int cpu; | |
120 | struct cryptd_cpu_queue *cpu_queue; | |
121 | ||
122 | for_each_possible_cpu(cpu) { | |
123 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
124 | BUG_ON(cpu_queue->queue.qlen); | |
125 | } | |
126 | free_percpu(queue->cpu_queue); | |
127 | } | |
128 | ||
129 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
130 | struct crypto_async_request *request) | |
131 | { | |
91e8bcd7 | 132 | int err; |
254eff77 | 133 | struct cryptd_cpu_queue *cpu_queue; |
43b970fa | 134 | refcount_t *refcnt; |
254eff77 | 135 | |
91e8bcd7 | 136 | local_bh_disable(); |
0b44f486 | 137 | cpu_queue = this_cpu_ptr(queue->cpu_queue); |
254eff77 | 138 | err = crypto_enqueue_request(&cpu_queue->queue, request); |
81760ea6 HX |
139 | |
140 | refcnt = crypto_tfm_ctx(request->tfm); | |
81760ea6 | 141 | |
6b80ea38 | 142 | if (err == -ENOSPC) |
91e8bcd7 | 143 | goto out; |
81760ea6 | 144 | |
91e8bcd7 | 145 | queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work); |
81760ea6 | 146 | |
43b970fa | 147 | if (!refcount_read(refcnt)) |
91e8bcd7 | 148 | goto out; |
81760ea6 | 149 | |
43b970fa | 150 | refcount_inc(refcnt); |
81760ea6 | 151 | |
91e8bcd7 SAS |
152 | out: |
153 | local_bh_enable(); | |
254eff77 HY |
154 | |
155 | return err; | |
156 | } | |
157 | ||
158 | /* Called in workqueue context, do one real cryption work (via | |
159 | * req->complete) and reschedule itself if there are more work to | |
160 | * do. */ | |
161 | static void cryptd_queue_worker(struct work_struct *work) | |
162 | { | |
163 | struct cryptd_cpu_queue *cpu_queue; | |
164 | struct crypto_async_request *req, *backlog; | |
165 | ||
166 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
9efade1b JK |
167 | /* |
168 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
9efade1b JK |
169 | */ |
170 | local_bh_disable(); | |
254eff77 HY |
171 | backlog = crypto_get_backlog(&cpu_queue->queue); |
172 | req = crypto_dequeue_request(&cpu_queue->queue); | |
9efade1b | 173 | local_bh_enable(); |
254eff77 HY |
174 | |
175 | if (!req) | |
176 | return; | |
177 | ||
178 | if (backlog) | |
179 | backlog->complete(backlog, -EINPROGRESS); | |
180 | req->complete(req, 0); | |
181 | ||
182 | if (cpu_queue->queue.qlen) | |
3e56e168 | 183 | queue_work(cryptd_wq, &cpu_queue->work); |
254eff77 HY |
184 | } |
185 | ||
186 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
187 | { |
188 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
189 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 190 | return ictx->queue; |
124b53d0 HX |
191 | } |
192 | ||
7bcb2c99 EB |
193 | static void cryptd_type_and_mask(struct crypto_attr_type *algt, |
194 | u32 *type, u32 *mask) | |
466a7b9e | 195 | { |
7bcb2c99 EB |
196 | /* |
197 | * cryptd is allowed to wrap internal algorithms, but in that case the | |
198 | * resulting cryptd instance will be marked as internal as well. | |
199 | */ | |
200 | *type = algt->type & CRYPTO_ALG_INTERNAL; | |
201 | *mask = algt->mask & CRYPTO_ALG_INTERNAL; | |
466a7b9e | 202 | |
7bcb2c99 EB |
203 | /* No point in cryptd wrapping an algorithm that's already async. */ |
204 | *mask |= CRYPTO_ALG_ASYNC; | |
f6da3205 | 205 | |
7bcb2c99 | 206 | *mask |= crypto_algt_inherited_mask(algt); |
466a7b9e SM |
207 | } |
208 | ||
9b8c456e HX |
209 | static int cryptd_init_instance(struct crypto_instance *inst, |
210 | struct crypto_alg *alg) | |
211 | { | |
212 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
213 | "cryptd(%s)", | |
214 | alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
215 | return -ENAMETOOLONG; | |
216 | ||
217 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
218 | ||
219 | inst->alg.cra_priority = alg->cra_priority + 50; | |
220 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
221 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
222 | ||
223 | return 0; | |
224 | } | |
225 | ||
4e0958d1 HX |
226 | static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, |
227 | const u8 *key, unsigned int keylen) | |
228 | { | |
229 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); | |
36b3875a | 230 | struct crypto_sync_skcipher *child = ctx->child; |
4e0958d1 | 231 | |
36b3875a KC |
232 | crypto_sync_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
233 | crypto_sync_skcipher_set_flags(child, | |
234 | crypto_skcipher_get_flags(parent) & | |
4e0958d1 | 235 | CRYPTO_TFM_REQ_MASK); |
af5034e8 | 236 | return crypto_sync_skcipher_setkey(child, key, keylen); |
4e0958d1 HX |
237 | } |
238 | ||
239 | static void cryptd_skcipher_complete(struct skcipher_request *req, int err) | |
240 | { | |
241 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
242 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
243 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
43b970fa | 244 | int refcnt = refcount_read(&ctx->refcnt); |
4e0958d1 HX |
245 | |
246 | local_bh_disable(); | |
247 | rctx->complete(&req->base, err); | |
248 | local_bh_enable(); | |
249 | ||
43b970fa | 250 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
251 | crypto_free_skcipher(tfm); |
252 | } | |
253 | ||
254 | static void cryptd_skcipher_encrypt(struct crypto_async_request *base, | |
255 | int err) | |
256 | { | |
257 | struct skcipher_request *req = skcipher_request_cast(base); | |
258 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
259 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
260 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
36b3875a KC |
261 | struct crypto_sync_skcipher *child = ctx->child; |
262 | SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child); | |
4e0958d1 HX |
263 | |
264 | if (unlikely(err == -EINPROGRESS)) | |
265 | goto out; | |
266 | ||
36b3875a | 267 | skcipher_request_set_sync_tfm(subreq, child); |
4e0958d1 HX |
268 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
269 | NULL, NULL); | |
270 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
271 | req->iv); | |
272 | ||
273 | err = crypto_skcipher_encrypt(subreq); | |
274 | skcipher_request_zero(subreq); | |
275 | ||
276 | req->base.complete = rctx->complete; | |
277 | ||
278 | out: | |
279 | cryptd_skcipher_complete(req, err); | |
280 | } | |
281 | ||
282 | static void cryptd_skcipher_decrypt(struct crypto_async_request *base, | |
283 | int err) | |
284 | { | |
285 | struct skcipher_request *req = skcipher_request_cast(base); | |
286 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
287 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
288 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
36b3875a KC |
289 | struct crypto_sync_skcipher *child = ctx->child; |
290 | SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child); | |
4e0958d1 HX |
291 | |
292 | if (unlikely(err == -EINPROGRESS)) | |
293 | goto out; | |
294 | ||
36b3875a | 295 | skcipher_request_set_sync_tfm(subreq, child); |
4e0958d1 HX |
296 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
297 | NULL, NULL); | |
298 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
299 | req->iv); | |
300 | ||
301 | err = crypto_skcipher_decrypt(subreq); | |
302 | skcipher_request_zero(subreq); | |
303 | ||
304 | req->base.complete = rctx->complete; | |
305 | ||
306 | out: | |
307 | cryptd_skcipher_complete(req, err); | |
308 | } | |
309 | ||
310 | static int cryptd_skcipher_enqueue(struct skcipher_request *req, | |
311 | crypto_completion_t compl) | |
312 | { | |
313 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
314 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
315 | struct cryptd_queue *queue; | |
316 | ||
317 | queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); | |
318 | rctx->complete = req->base.complete; | |
319 | req->base.complete = compl; | |
320 | ||
321 | return cryptd_enqueue_request(queue, &req->base); | |
322 | } | |
323 | ||
324 | static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) | |
325 | { | |
326 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); | |
327 | } | |
328 | ||
329 | static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) | |
330 | { | |
331 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); | |
332 | } | |
333 | ||
334 | static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) | |
335 | { | |
336 | struct skcipher_instance *inst = skcipher_alg_instance(tfm); | |
337 | struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); | |
338 | struct crypto_skcipher_spawn *spawn = &ictx->spawn; | |
339 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
340 | struct crypto_skcipher *cipher; | |
341 | ||
342 | cipher = crypto_spawn_skcipher(spawn); | |
343 | if (IS_ERR(cipher)) | |
344 | return PTR_ERR(cipher); | |
345 | ||
36b3875a | 346 | ctx->child = (struct crypto_sync_skcipher *)cipher; |
4e0958d1 HX |
347 | crypto_skcipher_set_reqsize( |
348 | tfm, sizeof(struct cryptd_skcipher_request_ctx)); | |
349 | return 0; | |
350 | } | |
351 | ||
352 | static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) | |
353 | { | |
354 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
355 | ||
36b3875a | 356 | crypto_free_sync_skcipher(ctx->child); |
4e0958d1 HX |
357 | } |
358 | ||
359 | static void cryptd_skcipher_free(struct skcipher_instance *inst) | |
360 | { | |
361 | struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); | |
362 | ||
363 | crypto_drop_skcipher(&ctx->spawn); | |
1a0fad63 | 364 | kfree(inst); |
4e0958d1 HX |
365 | } |
366 | ||
367 | static int cryptd_create_skcipher(struct crypto_template *tmpl, | |
368 | struct rtattr **tb, | |
7bcb2c99 | 369 | struct crypto_attr_type *algt, |
4e0958d1 HX |
370 | struct cryptd_queue *queue) |
371 | { | |
372 | struct skcipherd_instance_ctx *ctx; | |
373 | struct skcipher_instance *inst; | |
374 | struct skcipher_alg *alg; | |
4e0958d1 HX |
375 | u32 type; |
376 | u32 mask; | |
377 | int err; | |
378 | ||
7bcb2c99 | 379 | cryptd_type_and_mask(algt, &type, &mask); |
4e0958d1 | 380 | |
4e0958d1 HX |
381 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
382 | if (!inst) | |
383 | return -ENOMEM; | |
384 | ||
385 | ctx = skcipher_instance_ctx(inst); | |
386 | ctx->queue = queue; | |
387 | ||
b9f76ddd | 388 | err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), |
b8c0d74a | 389 | crypto_attr_alg_name(tb[1]), type, mask); |
4e0958d1 | 390 | if (err) |
b8c0d74a | 391 | goto err_free_inst; |
4e0958d1 HX |
392 | |
393 | alg = crypto_spawn_skcipher_alg(&ctx->spawn); | |
394 | err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); | |
395 | if (err) | |
b8c0d74a | 396 | goto err_free_inst; |
4e0958d1 | 397 | |
7bcb2c99 EB |
398 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
399 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
4e0958d1 HX |
400 | inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg); |
401 | inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); | |
402 | inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); | |
403 | inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg); | |
404 | ||
405 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); | |
406 | ||
407 | inst->alg.init = cryptd_skcipher_init_tfm; | |
408 | inst->alg.exit = cryptd_skcipher_exit_tfm; | |
409 | ||
410 | inst->alg.setkey = cryptd_skcipher_setkey; | |
411 | inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; | |
412 | inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; | |
413 | ||
414 | inst->free = cryptd_skcipher_free; | |
415 | ||
416 | err = skcipher_register_instance(tmpl, inst); | |
417 | if (err) { | |
b8c0d74a EB |
418 | err_free_inst: |
419 | cryptd_skcipher_free(inst); | |
4e0958d1 HX |
420 | } |
421 | return err; | |
422 | } | |
423 | ||
b8a28251 LH |
424 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
425 | { | |
426 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
427 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
428 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 429 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 430 | struct crypto_shash *hash; |
b8a28251 | 431 | |
46309d89 HX |
432 | hash = crypto_spawn_shash(spawn); |
433 | if (IS_ERR(hash)) | |
434 | return PTR_ERR(hash); | |
b8a28251 | 435 | |
46309d89 | 436 | ctx->child = hash; |
0d6669e2 HX |
437 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
438 | sizeof(struct cryptd_hash_request_ctx) + | |
439 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
440 | return 0; |
441 | } | |
442 | ||
443 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
444 | { | |
445 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 446 | |
46309d89 | 447 | crypto_free_shash(ctx->child); |
b8a28251 LH |
448 | } |
449 | ||
450 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
451 | const u8 *key, unsigned int keylen) | |
452 | { | |
453 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 454 | struct crypto_shash *child = ctx->child; |
b8a28251 | 455 | |
46309d89 HX |
456 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
457 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
458 | CRYPTO_TFM_REQ_MASK); | |
af5034e8 | 459 | return crypto_shash_setkey(child, key, keylen); |
b8a28251 LH |
460 | } |
461 | ||
462 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
3e3dc25f | 463 | crypto_completion_t compl) |
b8a28251 LH |
464 | { |
465 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
466 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
467 | struct cryptd_queue *queue = |
468 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
469 | |
470 | rctx->complete = req->base.complete; | |
3e3dc25f | 471 | req->base.complete = compl; |
b8a28251 | 472 | |
254eff77 | 473 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
474 | } |
475 | ||
81760ea6 HX |
476 | static void cryptd_hash_complete(struct ahash_request *req, int err) |
477 | { | |
478 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
479 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
480 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
43b970fa | 481 | int refcnt = refcount_read(&ctx->refcnt); |
81760ea6 HX |
482 | |
483 | local_bh_disable(); | |
484 | rctx->complete(&req->base, err); | |
485 | local_bh_enable(); | |
486 | ||
43b970fa | 487 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 HX |
488 | crypto_free_ahash(tfm); |
489 | } | |
490 | ||
b8a28251 LH |
491 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) |
492 | { | |
46309d89 HX |
493 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
494 | struct crypto_shash *child = ctx->child; | |
495 | struct ahash_request *req = ahash_request_cast(req_async); | |
496 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
497 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
498 | |
499 | if (unlikely(err == -EINPROGRESS)) | |
500 | goto out; | |
501 | ||
46309d89 | 502 | desc->tfm = child; |
b8a28251 | 503 | |
46309d89 | 504 | err = crypto_shash_init(desc); |
b8a28251 LH |
505 | |
506 | req->base.complete = rctx->complete; | |
507 | ||
508 | out: | |
81760ea6 | 509 | cryptd_hash_complete(req, err); |
b8a28251 LH |
510 | } |
511 | ||
512 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
513 | { | |
514 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
515 | } | |
516 | ||
517 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
518 | { | |
46309d89 | 519 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 520 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
521 | |
522 | rctx = ahash_request_ctx(req); | |
523 | ||
524 | if (unlikely(err == -EINPROGRESS)) | |
525 | goto out; | |
526 | ||
46309d89 | 527 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
528 | |
529 | req->base.complete = rctx->complete; | |
530 | ||
531 | out: | |
81760ea6 | 532 | cryptd_hash_complete(req, err); |
b8a28251 LH |
533 | } |
534 | ||
535 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
536 | { | |
537 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
538 | } | |
539 | ||
540 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
541 | { | |
46309d89 HX |
542 | struct ahash_request *req = ahash_request_cast(req_async); |
543 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
544 | |
545 | if (unlikely(err == -EINPROGRESS)) | |
546 | goto out; | |
547 | ||
46309d89 | 548 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
549 | |
550 | req->base.complete = rctx->complete; | |
551 | ||
552 | out: | |
81760ea6 | 553 | cryptd_hash_complete(req, err); |
b8a28251 LH |
554 | } |
555 | ||
556 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
557 | { | |
558 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
559 | } | |
560 | ||
6fba00d1 HX |
561 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) |
562 | { | |
563 | struct ahash_request *req = ahash_request_cast(req_async); | |
564 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
565 | ||
566 | if (unlikely(err == -EINPROGRESS)) | |
567 | goto out; | |
568 | ||
569 | err = shash_ahash_finup(req, &rctx->desc); | |
570 | ||
571 | req->base.complete = rctx->complete; | |
572 | ||
573 | out: | |
81760ea6 | 574 | cryptd_hash_complete(req, err); |
6fba00d1 HX |
575 | } |
576 | ||
577 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
578 | { | |
579 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
580 | } | |
581 | ||
b8a28251 LH |
582 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) |
583 | { | |
46309d89 HX |
584 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
585 | struct crypto_shash *child = ctx->child; | |
586 | struct ahash_request *req = ahash_request_cast(req_async); | |
587 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
588 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
589 | |
590 | if (unlikely(err == -EINPROGRESS)) | |
591 | goto out; | |
592 | ||
46309d89 | 593 | desc->tfm = child; |
b8a28251 | 594 | |
46309d89 | 595 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
596 | |
597 | req->base.complete = rctx->complete; | |
598 | ||
599 | out: | |
81760ea6 | 600 | cryptd_hash_complete(req, err); |
b8a28251 LH |
601 | } |
602 | ||
603 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
604 | { | |
605 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
606 | } | |
607 | ||
6fba00d1 HX |
608 | static int cryptd_hash_export(struct ahash_request *req, void *out) |
609 | { | |
610 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
611 | ||
612 | return crypto_shash_export(&rctx->desc, out); | |
613 | } | |
614 | ||
615 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
616 | { | |
0bd22235 AB |
617 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
618 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
619 | struct shash_desc *desc = cryptd_shash_desc(req); | |
620 | ||
621 | desc->tfm = ctx->child; | |
6fba00d1 | 622 | |
0bd22235 | 623 | return crypto_shash_import(desc, in); |
6fba00d1 HX |
624 | } |
625 | ||
758ec5ac EB |
626 | static void cryptd_hash_free(struct ahash_instance *inst) |
627 | { | |
628 | struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); | |
629 | ||
630 | crypto_drop_shash(&ctx->spawn); | |
631 | kfree(inst); | |
632 | } | |
633 | ||
9cd899a3 | 634 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
7bcb2c99 | 635 | struct crypto_attr_type *algt, |
9cd899a3 | 636 | struct cryptd_queue *queue) |
b8a28251 | 637 | { |
46309d89 | 638 | struct hashd_instance_ctx *ctx; |
0b535adf | 639 | struct ahash_instance *inst; |
218c5035 | 640 | struct shash_alg *alg; |
7bcb2c99 EB |
641 | u32 type; |
642 | u32 mask; | |
46309d89 | 643 | int err; |
b8a28251 | 644 | |
7bcb2c99 | 645 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 646 | |
218c5035 EB |
647 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
648 | if (!inst) | |
649 | return -ENOMEM; | |
b8a28251 | 650 | |
0b535adf | 651 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
652 | ctx->queue = queue; |
653 | ||
218c5035 EB |
654 | err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), |
655 | crypto_attr_alg_name(tb[1]), type, mask); | |
46309d89 | 656 | if (err) |
218c5035 EB |
657 | goto err_free_inst; |
658 | alg = crypto_spawn_shash_alg(&ctx->spawn); | |
659 | ||
660 | err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); | |
661 | if (err) | |
662 | goto err_free_inst; | |
46309d89 | 663 | |
7bcb2c99 EB |
664 | inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
665 | (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL| | |
218c5035 | 666 | CRYPTO_ALG_OPTIONAL_KEY)); |
218c5035 EB |
667 | inst->alg.halg.digestsize = alg->digestsize; |
668 | inst->alg.halg.statesize = alg->statesize; | |
0b535adf | 669 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); |
b8a28251 | 670 | |
0b535adf HX |
671 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
672 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 673 | |
0b535adf HX |
674 | inst->alg.init = cryptd_hash_init_enqueue; |
675 | inst->alg.update = cryptd_hash_update_enqueue; | |
676 | inst->alg.final = cryptd_hash_final_enqueue; | |
6fba00d1 HX |
677 | inst->alg.finup = cryptd_hash_finup_enqueue; |
678 | inst->alg.export = cryptd_hash_export; | |
679 | inst->alg.import = cryptd_hash_import; | |
218c5035 | 680 | if (crypto_shash_alg_has_setkey(alg)) |
841a3ff3 | 681 | inst->alg.setkey = cryptd_hash_setkey; |
0b535adf | 682 | inst->alg.digest = cryptd_hash_digest_enqueue; |
b8a28251 | 683 | |
758ec5ac EB |
684 | inst->free = cryptd_hash_free; |
685 | ||
0b535adf | 686 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 | 687 | if (err) { |
218c5035 | 688 | err_free_inst: |
b8c0d74a | 689 | cryptd_hash_free(inst); |
9cd899a3 | 690 | } |
9cd899a3 | 691 | return err; |
b8a28251 LH |
692 | } |
693 | ||
92b9876b HX |
694 | static int cryptd_aead_setkey(struct crypto_aead *parent, |
695 | const u8 *key, unsigned int keylen) | |
696 | { | |
697 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
698 | struct crypto_aead *child = ctx->child; | |
699 | ||
700 | return crypto_aead_setkey(child, key, keylen); | |
701 | } | |
702 | ||
703 | static int cryptd_aead_setauthsize(struct crypto_aead *parent, | |
704 | unsigned int authsize) | |
705 | { | |
706 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
707 | struct crypto_aead *child = ctx->child; | |
708 | ||
709 | return crypto_aead_setauthsize(child, authsize); | |
710 | } | |
711 | ||
298c926c AH |
712 | static void cryptd_aead_crypt(struct aead_request *req, |
713 | struct crypto_aead *child, | |
714 | int err, | |
715 | int (*crypt)(struct aead_request *req)) | |
716 | { | |
717 | struct cryptd_aead_request_ctx *rctx; | |
81760ea6 | 718 | struct cryptd_aead_ctx *ctx; |
ec9f2006 | 719 | crypto_completion_t compl; |
81760ea6 HX |
720 | struct crypto_aead *tfm; |
721 | int refcnt; | |
ec9f2006 | 722 | |
298c926c | 723 | rctx = aead_request_ctx(req); |
ec9f2006 | 724 | compl = rctx->complete; |
298c926c | 725 | |
31bd44e7 HX |
726 | tfm = crypto_aead_reqtfm(req); |
727 | ||
298c926c AH |
728 | if (unlikely(err == -EINPROGRESS)) |
729 | goto out; | |
730 | aead_request_set_tfm(req, child); | |
731 | err = crypt( req ); | |
81760ea6 | 732 | |
298c926c | 733 | out: |
81760ea6 | 734 | ctx = crypto_aead_ctx(tfm); |
43b970fa | 735 | refcnt = refcount_read(&ctx->refcnt); |
81760ea6 | 736 | |
298c926c | 737 | local_bh_disable(); |
ec9f2006 | 738 | compl(&req->base, err); |
298c926c | 739 | local_bh_enable(); |
81760ea6 | 740 | |
43b970fa | 741 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 742 | crypto_free_aead(tfm); |
298c926c AH |
743 | } |
744 | ||
745 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
746 | { | |
747 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
748 | struct crypto_aead *child = ctx->child; | |
749 | struct aead_request *req; | |
750 | ||
751 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 752 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); |
298c926c AH |
753 | } |
754 | ||
755 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
756 | { | |
757 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
758 | struct crypto_aead *child = ctx->child; | |
759 | struct aead_request *req; | |
760 | ||
761 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 762 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); |
298c926c AH |
763 | } |
764 | ||
765 | static int cryptd_aead_enqueue(struct aead_request *req, | |
3e3dc25f | 766 | crypto_completion_t compl) |
298c926c AH |
767 | { |
768 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
769 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
770 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
771 | ||
772 | rctx->complete = req->base.complete; | |
3e3dc25f | 773 | req->base.complete = compl; |
298c926c AH |
774 | return cryptd_enqueue_request(queue, &req->base); |
775 | } | |
776 | ||
777 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
778 | { | |
779 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
780 | } | |
781 | ||
782 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
783 | { | |
784 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
785 | } | |
786 | ||
f614e546 | 787 | static int cryptd_aead_init_tfm(struct crypto_aead *tfm) |
298c926c | 788 | { |
f614e546 HX |
789 | struct aead_instance *inst = aead_alg_instance(tfm); |
790 | struct aead_instance_ctx *ictx = aead_instance_ctx(inst); | |
298c926c | 791 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; |
f614e546 | 792 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
793 | struct crypto_aead *cipher; |
794 | ||
795 | cipher = crypto_spawn_aead(spawn); | |
796 | if (IS_ERR(cipher)) | |
797 | return PTR_ERR(cipher); | |
798 | ||
298c926c | 799 | ctx->child = cipher; |
ec9f2006 HX |
800 | crypto_aead_set_reqsize( |
801 | tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), | |
802 | crypto_aead_reqsize(cipher))); | |
298c926c AH |
803 | return 0; |
804 | } | |
805 | ||
f614e546 | 806 | static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) |
298c926c | 807 | { |
f614e546 | 808 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
809 | crypto_free_aead(ctx->child); |
810 | } | |
811 | ||
758ec5ac EB |
812 | static void cryptd_aead_free(struct aead_instance *inst) |
813 | { | |
814 | struct aead_instance_ctx *ctx = aead_instance_ctx(inst); | |
815 | ||
816 | crypto_drop_aead(&ctx->aead_spawn); | |
817 | kfree(inst); | |
818 | } | |
819 | ||
298c926c AH |
820 | static int cryptd_create_aead(struct crypto_template *tmpl, |
821 | struct rtattr **tb, | |
7bcb2c99 | 822 | struct crypto_attr_type *algt, |
298c926c AH |
823 | struct cryptd_queue *queue) |
824 | { | |
825 | struct aead_instance_ctx *ctx; | |
f614e546 HX |
826 | struct aead_instance *inst; |
827 | struct aead_alg *alg; | |
7bcb2c99 EB |
828 | u32 type; |
829 | u32 mask; | |
298c926c AH |
830 | int err; |
831 | ||
7bcb2c99 | 832 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 833 | |
9b8c456e HX |
834 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
835 | if (!inst) | |
836 | return -ENOMEM; | |
298c926c | 837 | |
f614e546 | 838 | ctx = aead_instance_ctx(inst); |
298c926c AH |
839 | ctx->queue = queue; |
840 | ||
cd900f0c | 841 | err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), |
b8c0d74a | 842 | crypto_attr_alg_name(tb[1]), type, mask); |
298c926c | 843 | if (err) |
b8c0d74a | 844 | goto err_free_inst; |
298c926c | 845 | |
f614e546 HX |
846 | alg = crypto_spawn_aead_alg(&ctx->aead_spawn); |
847 | err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); | |
9b8c456e | 848 | if (err) |
b8c0d74a | 849 | goto err_free_inst; |
9b8c456e | 850 | |
7bcb2c99 EB |
851 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
852 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
f614e546 | 853 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); |
298c926c | 854 | |
f614e546 HX |
855 | inst->alg.ivsize = crypto_aead_alg_ivsize(alg); |
856 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); | |
857 | ||
858 | inst->alg.init = cryptd_aead_init_tfm; | |
859 | inst->alg.exit = cryptd_aead_exit_tfm; | |
860 | inst->alg.setkey = cryptd_aead_setkey; | |
861 | inst->alg.setauthsize = cryptd_aead_setauthsize; | |
862 | inst->alg.encrypt = cryptd_aead_encrypt_enqueue; | |
863 | inst->alg.decrypt = cryptd_aead_decrypt_enqueue; | |
864 | ||
758ec5ac EB |
865 | inst->free = cryptd_aead_free; |
866 | ||
f614e546 | 867 | err = aead_register_instance(tmpl, inst); |
298c926c | 868 | if (err) { |
b8c0d74a EB |
869 | err_free_inst: |
870 | cryptd_aead_free(inst); | |
298c926c | 871 | } |
298c926c AH |
872 | return err; |
873 | } | |
874 | ||
254eff77 | 875 | static struct cryptd_queue queue; |
124b53d0 | 876 | |
9cd899a3 | 877 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
878 | { |
879 | struct crypto_attr_type *algt; | |
880 | ||
881 | algt = crypto_get_attr_type(tb); | |
882 | if (IS_ERR(algt)) | |
9cd899a3 | 883 | return PTR_ERR(algt); |
124b53d0 HX |
884 | |
885 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
c65058b7 | 886 | case CRYPTO_ALG_TYPE_SKCIPHER: |
7bcb2c99 | 887 | return cryptd_create_skcipher(tmpl, tb, algt, &queue); |
84ede58d | 888 | case CRYPTO_ALG_TYPE_HASH: |
7bcb2c99 | 889 | return cryptd_create_hash(tmpl, tb, algt, &queue); |
298c926c | 890 | case CRYPTO_ALG_TYPE_AEAD: |
7bcb2c99 | 891 | return cryptd_create_aead(tmpl, tb, algt, &queue); |
124b53d0 HX |
892 | } |
893 | ||
9cd899a3 | 894 | return -EINVAL; |
124b53d0 HX |
895 | } |
896 | ||
124b53d0 HX |
897 | static struct crypto_template cryptd_tmpl = { |
898 | .name = "cryptd", | |
9cd899a3 | 899 | .create = cryptd_create, |
124b53d0 HX |
900 | .module = THIS_MODULE, |
901 | }; | |
902 | ||
4e0958d1 HX |
903 | struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, |
904 | u32 type, u32 mask) | |
905 | { | |
906 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
907 | struct cryptd_skcipher_ctx *ctx; | |
908 | struct crypto_skcipher *tfm; | |
909 | ||
910 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
911 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
912 | return ERR_PTR(-EINVAL); | |
913 | ||
914 | tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); | |
915 | if (IS_ERR(tfm)) | |
916 | return ERR_CAST(tfm); | |
917 | ||
918 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
919 | crypto_free_skcipher(tfm); | |
920 | return ERR_PTR(-EINVAL); | |
921 | } | |
922 | ||
923 | ctx = crypto_skcipher_ctx(tfm); | |
43b970fa | 924 | refcount_set(&ctx->refcnt, 1); |
4e0958d1 HX |
925 | |
926 | return container_of(tfm, struct cryptd_skcipher, base); | |
927 | } | |
928 | EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); | |
929 | ||
930 | struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) | |
931 | { | |
932 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
933 | ||
36b3875a | 934 | return &ctx->child->base; |
4e0958d1 HX |
935 | } |
936 | EXPORT_SYMBOL_GPL(cryptd_skcipher_child); | |
937 | ||
938 | bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) | |
939 | { | |
940 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
941 | ||
43b970fa | 942 | return refcount_read(&ctx->refcnt) - 1; |
4e0958d1 HX |
943 | } |
944 | EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); | |
945 | ||
946 | void cryptd_free_skcipher(struct cryptd_skcipher *tfm) | |
947 | { | |
948 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
949 | ||
43b970fa | 950 | if (refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
951 | crypto_free_skcipher(&tfm->base); |
952 | } | |
953 | EXPORT_SYMBOL_GPL(cryptd_free_skcipher); | |
954 | ||
ace13663 HY |
955 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, |
956 | u32 type, u32 mask) | |
957 | { | |
958 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 959 | struct cryptd_hash_ctx *ctx; |
ace13663 HY |
960 | struct crypto_ahash *tfm; |
961 | ||
962 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
963 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
964 | return ERR_PTR(-EINVAL); | |
965 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
966 | if (IS_ERR(tfm)) | |
967 | return ERR_CAST(tfm); | |
968 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
969 | crypto_free_ahash(tfm); | |
970 | return ERR_PTR(-EINVAL); | |
971 | } | |
972 | ||
81760ea6 | 973 | ctx = crypto_ahash_ctx(tfm); |
43b970fa | 974 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 975 | |
ace13663 HY |
976 | return __cryptd_ahash_cast(tfm); |
977 | } | |
978 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
979 | ||
980 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
981 | { | |
982 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
983 | ||
984 | return ctx->child; | |
985 | } | |
986 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
987 | ||
0e1227d3 HY |
988 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) |
989 | { | |
990 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
991 | return &rctx->desc; | |
992 | } | |
993 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
994 | ||
81760ea6 HX |
995 | bool cryptd_ahash_queued(struct cryptd_ahash *tfm) |
996 | { | |
997 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
998 | ||
43b970fa | 999 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1000 | } |
1001 | EXPORT_SYMBOL_GPL(cryptd_ahash_queued); | |
1002 | ||
ace13663 HY |
1003 | void cryptd_free_ahash(struct cryptd_ahash *tfm) |
1004 | { | |
81760ea6 HX |
1005 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); |
1006 | ||
43b970fa | 1007 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1008 | crypto_free_ahash(&tfm->base); |
ace13663 HY |
1009 | } |
1010 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
1011 | ||
298c926c AH |
1012 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, |
1013 | u32 type, u32 mask) | |
1014 | { | |
1015 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 1016 | struct cryptd_aead_ctx *ctx; |
298c926c AH |
1017 | struct crypto_aead *tfm; |
1018 | ||
1019 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
1020 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
1021 | return ERR_PTR(-EINVAL); | |
1022 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
1023 | if (IS_ERR(tfm)) | |
1024 | return ERR_CAST(tfm); | |
1025 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1026 | crypto_free_aead(tfm); | |
1027 | return ERR_PTR(-EINVAL); | |
1028 | } | |
81760ea6 HX |
1029 | |
1030 | ctx = crypto_aead_ctx(tfm); | |
43b970fa | 1031 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 1032 | |
298c926c AH |
1033 | return __cryptd_aead_cast(tfm); |
1034 | } | |
1035 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
1036 | ||
1037 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
1038 | { | |
1039 | struct cryptd_aead_ctx *ctx; | |
1040 | ctx = crypto_aead_ctx(&tfm->base); | |
1041 | return ctx->child; | |
1042 | } | |
1043 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
1044 | ||
81760ea6 HX |
1045 | bool cryptd_aead_queued(struct cryptd_aead *tfm) |
1046 | { | |
1047 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); | |
1048 | ||
43b970fa | 1049 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1050 | } |
1051 | EXPORT_SYMBOL_GPL(cryptd_aead_queued); | |
1052 | ||
298c926c AH |
1053 | void cryptd_free_aead(struct cryptd_aead *tfm) |
1054 | { | |
81760ea6 HX |
1055 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); |
1056 | ||
43b970fa | 1057 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1058 | crypto_free_aead(&tfm->base); |
298c926c AH |
1059 | } |
1060 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
1061 | ||
124b53d0 HX |
1062 | static int __init cryptd_init(void) |
1063 | { | |
1064 | int err; | |
1065 | ||
3e56e168 EB |
1066 | cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, |
1067 | 1); | |
1068 | if (!cryptd_wq) | |
1069 | return -ENOMEM; | |
1070 | ||
c3a53605 | 1071 | err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); |
124b53d0 | 1072 | if (err) |
3e56e168 | 1073 | goto err_destroy_wq; |
124b53d0 HX |
1074 | |
1075 | err = crypto_register_template(&cryptd_tmpl); | |
1076 | if (err) | |
3e56e168 | 1077 | goto err_fini_queue; |
124b53d0 | 1078 | |
3e56e168 EB |
1079 | return 0; |
1080 | ||
1081 | err_fini_queue: | |
1082 | cryptd_fini_queue(&queue); | |
1083 | err_destroy_wq: | |
1084 | destroy_workqueue(cryptd_wq); | |
124b53d0 HX |
1085 | return err; |
1086 | } | |
1087 | ||
1088 | static void __exit cryptd_exit(void) | |
1089 | { | |
3e56e168 | 1090 | destroy_workqueue(cryptd_wq); |
254eff77 | 1091 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1092 | crypto_unregister_template(&cryptd_tmpl); |
1093 | } | |
1094 | ||
b2bac6ac | 1095 | subsys_initcall(cryptd_init); |
124b53d0 HX |
1096 | module_exit(cryptd_exit); |
1097 | ||
1098 | MODULE_LICENSE("GPL"); | |
1099 | MODULE_DESCRIPTION("Software async crypto daemon"); | |
4943ba16 | 1100 | MODULE_ALIAS_CRYPTO("cryptd"); |