Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
124b53d0 HX |
2 | /* |
3 | * Software async crypto daemon. | |
4 | * | |
5 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
6 | * | |
298c926c AH |
7 | * Added AEAD support to cryptd. |
8 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
9 | * Adrian Hoban <adrian.hoban@intel.com> | |
10 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
11 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
12 | * Copyright (c) 2010, Intel Corporation. | |
124b53d0 HX |
13 | */ |
14 | ||
18e33e6d | 15 | #include <crypto/internal/hash.h> |
298c926c | 16 | #include <crypto/internal/aead.h> |
4e0958d1 | 17 | #include <crypto/internal/skcipher.h> |
1cac2cbc | 18 | #include <crypto/cryptd.h> |
43b970fa | 19 | #include <linux/refcount.h> |
124b53d0 HX |
20 | #include <linux/err.h> |
21 | #include <linux/init.h> | |
22 | #include <linux/kernel.h> | |
124b53d0 HX |
23 | #include <linux/list.h> |
24 | #include <linux/module.h> | |
124b53d0 HX |
25 | #include <linux/scatterlist.h> |
26 | #include <linux/sched.h> | |
27 | #include <linux/slab.h> | |
3e56e168 | 28 | #include <linux/workqueue.h> |
124b53d0 | 29 | |
eaf356e4 | 30 | static unsigned int cryptd_max_cpu_qlen = 1000; |
c3a53605 JM |
31 | module_param(cryptd_max_cpu_qlen, uint, 0); |
32 | MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); | |
124b53d0 | 33 | |
3e56e168 EB |
34 | static struct workqueue_struct *cryptd_wq; |
35 | ||
254eff77 | 36 | struct cryptd_cpu_queue { |
124b53d0 | 37 | struct crypto_queue queue; |
254eff77 HY |
38 | struct work_struct work; |
39 | }; | |
40 | ||
41 | struct cryptd_queue { | |
91e8bcd7 SAS |
42 | /* |
43 | * Protected by disabling BH to allow enqueueing from softinterrupt and | |
44 | * dequeuing from kworker (cryptd_queue_worker()). | |
45 | */ | |
a29d8b8e | 46 | struct cryptd_cpu_queue __percpu *cpu_queue; |
124b53d0 HX |
47 | }; |
48 | ||
49 | struct cryptd_instance_ctx { | |
50 | struct crypto_spawn spawn; | |
254eff77 | 51 | struct cryptd_queue *queue; |
124b53d0 HX |
52 | }; |
53 | ||
4e0958d1 HX |
54 | struct skcipherd_instance_ctx { |
55 | struct crypto_skcipher_spawn spawn; | |
56 | struct cryptd_queue *queue; | |
57 | }; | |
58 | ||
46309d89 HX |
59 | struct hashd_instance_ctx { |
60 | struct crypto_shash_spawn spawn; | |
61 | struct cryptd_queue *queue; | |
62 | }; | |
63 | ||
298c926c AH |
64 | struct aead_instance_ctx { |
65 | struct crypto_aead_spawn aead_spawn; | |
66 | struct cryptd_queue *queue; | |
67 | }; | |
68 | ||
4e0958d1 | 69 | struct cryptd_skcipher_ctx { |
43b970fa | 70 | refcount_t refcnt; |
3a58c231 | 71 | struct crypto_skcipher *child; |
4e0958d1 HX |
72 | }; |
73 | ||
74 | struct cryptd_skcipher_request_ctx { | |
75 | crypto_completion_t complete; | |
3a58c231 | 76 | struct skcipher_request req; |
4e0958d1 HX |
77 | }; |
78 | ||
b8a28251 | 79 | struct cryptd_hash_ctx { |
43b970fa | 80 | refcount_t refcnt; |
46309d89 | 81 | struct crypto_shash *child; |
b8a28251 LH |
82 | }; |
83 | ||
84 | struct cryptd_hash_request_ctx { | |
85 | crypto_completion_t complete; | |
46309d89 | 86 | struct shash_desc desc; |
b8a28251 | 87 | }; |
124b53d0 | 88 | |
298c926c | 89 | struct cryptd_aead_ctx { |
43b970fa | 90 | refcount_t refcnt; |
298c926c AH |
91 | struct crypto_aead *child; |
92 | }; | |
93 | ||
94 | struct cryptd_aead_request_ctx { | |
95 | crypto_completion_t complete; | |
96 | }; | |
97 | ||
254eff77 HY |
98 | static void cryptd_queue_worker(struct work_struct *work); |
99 | ||
100 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
101 | unsigned int max_cpu_qlen) | |
102 | { | |
103 | int cpu; | |
104 | struct cryptd_cpu_queue *cpu_queue; | |
105 | ||
106 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
107 | if (!queue->cpu_queue) | |
108 | return -ENOMEM; | |
109 | for_each_possible_cpu(cpu) { | |
110 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
111 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
112 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
113 | } | |
c3a53605 | 114 | pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); |
254eff77 HY |
115 | return 0; |
116 | } | |
117 | ||
118 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
119 | { | |
120 | int cpu; | |
121 | struct cryptd_cpu_queue *cpu_queue; | |
122 | ||
123 | for_each_possible_cpu(cpu) { | |
124 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
125 | BUG_ON(cpu_queue->queue.qlen); | |
126 | } | |
127 | free_percpu(queue->cpu_queue); | |
128 | } | |
129 | ||
130 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
131 | struct crypto_async_request *request) | |
132 | { | |
91e8bcd7 | 133 | int err; |
254eff77 | 134 | struct cryptd_cpu_queue *cpu_queue; |
43b970fa | 135 | refcount_t *refcnt; |
254eff77 | 136 | |
91e8bcd7 | 137 | local_bh_disable(); |
0b44f486 | 138 | cpu_queue = this_cpu_ptr(queue->cpu_queue); |
254eff77 | 139 | err = crypto_enqueue_request(&cpu_queue->queue, request); |
81760ea6 HX |
140 | |
141 | refcnt = crypto_tfm_ctx(request->tfm); | |
81760ea6 | 142 | |
6b80ea38 | 143 | if (err == -ENOSPC) |
91e8bcd7 | 144 | goto out; |
81760ea6 | 145 | |
91e8bcd7 | 146 | queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work); |
81760ea6 | 147 | |
43b970fa | 148 | if (!refcount_read(refcnt)) |
91e8bcd7 | 149 | goto out; |
81760ea6 | 150 | |
43b970fa | 151 | refcount_inc(refcnt); |
81760ea6 | 152 | |
91e8bcd7 SAS |
153 | out: |
154 | local_bh_enable(); | |
254eff77 HY |
155 | |
156 | return err; | |
157 | } | |
158 | ||
159 | /* Called in workqueue context, do one real cryption work (via | |
160 | * req->complete) and reschedule itself if there are more work to | |
161 | * do. */ | |
162 | static void cryptd_queue_worker(struct work_struct *work) | |
163 | { | |
164 | struct cryptd_cpu_queue *cpu_queue; | |
165 | struct crypto_async_request *req, *backlog; | |
166 | ||
167 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
9efade1b JK |
168 | /* |
169 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
9efade1b JK |
170 | */ |
171 | local_bh_disable(); | |
254eff77 HY |
172 | backlog = crypto_get_backlog(&cpu_queue->queue); |
173 | req = crypto_dequeue_request(&cpu_queue->queue); | |
9efade1b | 174 | local_bh_enable(); |
254eff77 HY |
175 | |
176 | if (!req) | |
177 | return; | |
178 | ||
179 | if (backlog) | |
180 | backlog->complete(backlog, -EINPROGRESS); | |
181 | req->complete(req, 0); | |
182 | ||
183 | if (cpu_queue->queue.qlen) | |
3e56e168 | 184 | queue_work(cryptd_wq, &cpu_queue->work); |
254eff77 HY |
185 | } |
186 | ||
187 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
188 | { |
189 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
190 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 191 | return ictx->queue; |
124b53d0 HX |
192 | } |
193 | ||
7bcb2c99 EB |
194 | static void cryptd_type_and_mask(struct crypto_attr_type *algt, |
195 | u32 *type, u32 *mask) | |
466a7b9e | 196 | { |
7bcb2c99 EB |
197 | /* |
198 | * cryptd is allowed to wrap internal algorithms, but in that case the | |
199 | * resulting cryptd instance will be marked as internal as well. | |
200 | */ | |
201 | *type = algt->type & CRYPTO_ALG_INTERNAL; | |
202 | *mask = algt->mask & CRYPTO_ALG_INTERNAL; | |
466a7b9e | 203 | |
7bcb2c99 EB |
204 | /* No point in cryptd wrapping an algorithm that's already async. */ |
205 | *mask |= CRYPTO_ALG_ASYNC; | |
f6da3205 | 206 | |
7bcb2c99 | 207 | *mask |= crypto_algt_inherited_mask(algt); |
466a7b9e SM |
208 | } |
209 | ||
9b8c456e HX |
210 | static int cryptd_init_instance(struct crypto_instance *inst, |
211 | struct crypto_alg *alg) | |
212 | { | |
213 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
214 | "cryptd(%s)", | |
215 | alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
216 | return -ENAMETOOLONG; | |
217 | ||
218 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
219 | ||
220 | inst->alg.cra_priority = alg->cra_priority + 50; | |
221 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
222 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
223 | ||
224 | return 0; | |
225 | } | |
226 | ||
4e0958d1 HX |
227 | static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, |
228 | const u8 *key, unsigned int keylen) | |
229 | { | |
230 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); | |
3a58c231 | 231 | struct crypto_skcipher *child = ctx->child; |
4e0958d1 | 232 | |
3a58c231 HX |
233 | crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
234 | crypto_skcipher_set_flags(child, | |
235 | crypto_skcipher_get_flags(parent) & | |
236 | CRYPTO_TFM_REQ_MASK); | |
237 | return crypto_skcipher_setkey(child, key, keylen); | |
4e0958d1 HX |
238 | } |
239 | ||
240 | static void cryptd_skcipher_complete(struct skcipher_request *req, int err) | |
241 | { | |
242 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
243 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
244 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
43b970fa | 245 | int refcnt = refcount_read(&ctx->refcnt); |
4e0958d1 HX |
246 | |
247 | local_bh_disable(); | |
248 | rctx->complete(&req->base, err); | |
249 | local_bh_enable(); | |
250 | ||
43b970fa | 251 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
252 | crypto_free_skcipher(tfm); |
253 | } | |
254 | ||
255 | static void cryptd_skcipher_encrypt(struct crypto_async_request *base, | |
256 | int err) | |
257 | { | |
258 | struct skcipher_request *req = skcipher_request_cast(base); | |
259 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
260 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
261 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
3a58c231 HX |
262 | struct skcipher_request *subreq = &rctx->req; |
263 | struct crypto_skcipher *child = ctx->child; | |
4e0958d1 HX |
264 | |
265 | if (unlikely(err == -EINPROGRESS)) | |
266 | goto out; | |
267 | ||
3a58c231 | 268 | skcipher_request_set_tfm(subreq, child); |
4e0958d1 HX |
269 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
270 | NULL, NULL); | |
271 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
272 | req->iv); | |
273 | ||
274 | err = crypto_skcipher_encrypt(subreq); | |
275 | skcipher_request_zero(subreq); | |
276 | ||
277 | req->base.complete = rctx->complete; | |
278 | ||
279 | out: | |
280 | cryptd_skcipher_complete(req, err); | |
281 | } | |
282 | ||
283 | static void cryptd_skcipher_decrypt(struct crypto_async_request *base, | |
284 | int err) | |
285 | { | |
286 | struct skcipher_request *req = skcipher_request_cast(base); | |
287 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
288 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
289 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
3a58c231 HX |
290 | struct skcipher_request *subreq = &rctx->req; |
291 | struct crypto_skcipher *child = ctx->child; | |
4e0958d1 HX |
292 | |
293 | if (unlikely(err == -EINPROGRESS)) | |
294 | goto out; | |
295 | ||
3a58c231 | 296 | skcipher_request_set_tfm(subreq, child); |
4e0958d1 HX |
297 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
298 | NULL, NULL); | |
299 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
300 | req->iv); | |
301 | ||
302 | err = crypto_skcipher_decrypt(subreq); | |
303 | skcipher_request_zero(subreq); | |
304 | ||
305 | req->base.complete = rctx->complete; | |
306 | ||
307 | out: | |
308 | cryptd_skcipher_complete(req, err); | |
309 | } | |
310 | ||
311 | static int cryptd_skcipher_enqueue(struct skcipher_request *req, | |
312 | crypto_completion_t compl) | |
313 | { | |
314 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
315 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
316 | struct cryptd_queue *queue; | |
317 | ||
318 | queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); | |
319 | rctx->complete = req->base.complete; | |
320 | req->base.complete = compl; | |
321 | ||
322 | return cryptd_enqueue_request(queue, &req->base); | |
323 | } | |
324 | ||
325 | static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) | |
326 | { | |
327 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); | |
328 | } | |
329 | ||
330 | static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) | |
331 | { | |
332 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); | |
333 | } | |
334 | ||
335 | static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) | |
336 | { | |
337 | struct skcipher_instance *inst = skcipher_alg_instance(tfm); | |
338 | struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); | |
339 | struct crypto_skcipher_spawn *spawn = &ictx->spawn; | |
340 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
341 | struct crypto_skcipher *cipher; | |
342 | ||
343 | cipher = crypto_spawn_skcipher(spawn); | |
344 | if (IS_ERR(cipher)) | |
345 | return PTR_ERR(cipher); | |
346 | ||
3a58c231 | 347 | ctx->child = cipher; |
4e0958d1 | 348 | crypto_skcipher_set_reqsize( |
3a58c231 HX |
349 | tfm, sizeof(struct cryptd_skcipher_request_ctx) + |
350 | crypto_skcipher_reqsize(cipher)); | |
4e0958d1 HX |
351 | return 0; |
352 | } | |
353 | ||
354 | static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) | |
355 | { | |
356 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
357 | ||
3a58c231 | 358 | crypto_free_skcipher(ctx->child); |
4e0958d1 HX |
359 | } |
360 | ||
361 | static void cryptd_skcipher_free(struct skcipher_instance *inst) | |
362 | { | |
363 | struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); | |
364 | ||
365 | crypto_drop_skcipher(&ctx->spawn); | |
1a0fad63 | 366 | kfree(inst); |
4e0958d1 HX |
367 | } |
368 | ||
369 | static int cryptd_create_skcipher(struct crypto_template *tmpl, | |
370 | struct rtattr **tb, | |
7bcb2c99 | 371 | struct crypto_attr_type *algt, |
4e0958d1 HX |
372 | struct cryptd_queue *queue) |
373 | { | |
374 | struct skcipherd_instance_ctx *ctx; | |
375 | struct skcipher_instance *inst; | |
376 | struct skcipher_alg *alg; | |
4e0958d1 HX |
377 | u32 type; |
378 | u32 mask; | |
379 | int err; | |
380 | ||
7bcb2c99 | 381 | cryptd_type_and_mask(algt, &type, &mask); |
4e0958d1 | 382 | |
4e0958d1 HX |
383 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
384 | if (!inst) | |
385 | return -ENOMEM; | |
386 | ||
387 | ctx = skcipher_instance_ctx(inst); | |
388 | ctx->queue = queue; | |
389 | ||
b9f76ddd | 390 | err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), |
b8c0d74a | 391 | crypto_attr_alg_name(tb[1]), type, mask); |
4e0958d1 | 392 | if (err) |
b8c0d74a | 393 | goto err_free_inst; |
4e0958d1 HX |
394 | |
395 | alg = crypto_spawn_skcipher_alg(&ctx->spawn); | |
396 | err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); | |
397 | if (err) | |
b8c0d74a | 398 | goto err_free_inst; |
4e0958d1 | 399 | |
7bcb2c99 EB |
400 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
401 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
4e0958d1 HX |
402 | inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg); |
403 | inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); | |
404 | inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); | |
405 | inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg); | |
406 | ||
407 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); | |
408 | ||
409 | inst->alg.init = cryptd_skcipher_init_tfm; | |
410 | inst->alg.exit = cryptd_skcipher_exit_tfm; | |
411 | ||
412 | inst->alg.setkey = cryptd_skcipher_setkey; | |
413 | inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; | |
414 | inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; | |
415 | ||
416 | inst->free = cryptd_skcipher_free; | |
417 | ||
418 | err = skcipher_register_instance(tmpl, inst); | |
419 | if (err) { | |
b8c0d74a EB |
420 | err_free_inst: |
421 | cryptd_skcipher_free(inst); | |
4e0958d1 HX |
422 | } |
423 | return err; | |
424 | } | |
425 | ||
b8a28251 LH |
426 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
427 | { | |
428 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
429 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
430 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 431 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 432 | struct crypto_shash *hash; |
b8a28251 | 433 | |
46309d89 HX |
434 | hash = crypto_spawn_shash(spawn); |
435 | if (IS_ERR(hash)) | |
436 | return PTR_ERR(hash); | |
b8a28251 | 437 | |
46309d89 | 438 | ctx->child = hash; |
0d6669e2 HX |
439 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
440 | sizeof(struct cryptd_hash_request_ctx) + | |
441 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
442 | return 0; |
443 | } | |
444 | ||
445 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
446 | { | |
447 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 448 | |
46309d89 | 449 | crypto_free_shash(ctx->child); |
b8a28251 LH |
450 | } |
451 | ||
452 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
453 | const u8 *key, unsigned int keylen) | |
454 | { | |
455 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 456 | struct crypto_shash *child = ctx->child; |
b8a28251 | 457 | |
46309d89 HX |
458 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
459 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
460 | CRYPTO_TFM_REQ_MASK); | |
af5034e8 | 461 | return crypto_shash_setkey(child, key, keylen); |
b8a28251 LH |
462 | } |
463 | ||
464 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
3e3dc25f | 465 | crypto_completion_t compl) |
b8a28251 LH |
466 | { |
467 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
468 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
469 | struct cryptd_queue *queue = |
470 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
471 | |
472 | rctx->complete = req->base.complete; | |
3e3dc25f | 473 | req->base.complete = compl; |
b8a28251 | 474 | |
254eff77 | 475 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
476 | } |
477 | ||
81760ea6 HX |
478 | static void cryptd_hash_complete(struct ahash_request *req, int err) |
479 | { | |
480 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
481 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
482 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
43b970fa | 483 | int refcnt = refcount_read(&ctx->refcnt); |
81760ea6 HX |
484 | |
485 | local_bh_disable(); | |
486 | rctx->complete(&req->base, err); | |
487 | local_bh_enable(); | |
488 | ||
43b970fa | 489 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 HX |
490 | crypto_free_ahash(tfm); |
491 | } | |
492 | ||
b8a28251 LH |
493 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) |
494 | { | |
46309d89 HX |
495 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
496 | struct crypto_shash *child = ctx->child; | |
497 | struct ahash_request *req = ahash_request_cast(req_async); | |
498 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
499 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
500 | |
501 | if (unlikely(err == -EINPROGRESS)) | |
502 | goto out; | |
503 | ||
46309d89 | 504 | desc->tfm = child; |
b8a28251 | 505 | |
46309d89 | 506 | err = crypto_shash_init(desc); |
b8a28251 LH |
507 | |
508 | req->base.complete = rctx->complete; | |
509 | ||
510 | out: | |
81760ea6 | 511 | cryptd_hash_complete(req, err); |
b8a28251 LH |
512 | } |
513 | ||
514 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
515 | { | |
516 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
517 | } | |
518 | ||
519 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
520 | { | |
46309d89 | 521 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 522 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
523 | |
524 | rctx = ahash_request_ctx(req); | |
525 | ||
526 | if (unlikely(err == -EINPROGRESS)) | |
527 | goto out; | |
528 | ||
46309d89 | 529 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
530 | |
531 | req->base.complete = rctx->complete; | |
532 | ||
533 | out: | |
81760ea6 | 534 | cryptd_hash_complete(req, err); |
b8a28251 LH |
535 | } |
536 | ||
537 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
538 | { | |
539 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
540 | } | |
541 | ||
542 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
543 | { | |
46309d89 HX |
544 | struct ahash_request *req = ahash_request_cast(req_async); |
545 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
546 | |
547 | if (unlikely(err == -EINPROGRESS)) | |
548 | goto out; | |
549 | ||
46309d89 | 550 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
551 | |
552 | req->base.complete = rctx->complete; | |
553 | ||
554 | out: | |
81760ea6 | 555 | cryptd_hash_complete(req, err); |
b8a28251 LH |
556 | } |
557 | ||
558 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
559 | { | |
560 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
561 | } | |
562 | ||
6fba00d1 HX |
563 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) |
564 | { | |
565 | struct ahash_request *req = ahash_request_cast(req_async); | |
566 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
567 | ||
568 | if (unlikely(err == -EINPROGRESS)) | |
569 | goto out; | |
570 | ||
571 | err = shash_ahash_finup(req, &rctx->desc); | |
572 | ||
573 | req->base.complete = rctx->complete; | |
574 | ||
575 | out: | |
81760ea6 | 576 | cryptd_hash_complete(req, err); |
6fba00d1 HX |
577 | } |
578 | ||
579 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
580 | { | |
581 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
582 | } | |
583 | ||
b8a28251 LH |
584 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) |
585 | { | |
46309d89 HX |
586 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
587 | struct crypto_shash *child = ctx->child; | |
588 | struct ahash_request *req = ahash_request_cast(req_async); | |
589 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
590 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
591 | |
592 | if (unlikely(err == -EINPROGRESS)) | |
593 | goto out; | |
594 | ||
46309d89 | 595 | desc->tfm = child; |
b8a28251 | 596 | |
46309d89 | 597 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
598 | |
599 | req->base.complete = rctx->complete; | |
600 | ||
601 | out: | |
81760ea6 | 602 | cryptd_hash_complete(req, err); |
b8a28251 LH |
603 | } |
604 | ||
605 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
606 | { | |
607 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
608 | } | |
609 | ||
6fba00d1 HX |
610 | static int cryptd_hash_export(struct ahash_request *req, void *out) |
611 | { | |
612 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
613 | ||
614 | return crypto_shash_export(&rctx->desc, out); | |
615 | } | |
616 | ||
617 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
618 | { | |
0bd22235 AB |
619 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
620 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
621 | struct shash_desc *desc = cryptd_shash_desc(req); | |
622 | ||
623 | desc->tfm = ctx->child; | |
6fba00d1 | 624 | |
0bd22235 | 625 | return crypto_shash_import(desc, in); |
6fba00d1 HX |
626 | } |
627 | ||
758ec5ac EB |
628 | static void cryptd_hash_free(struct ahash_instance *inst) |
629 | { | |
630 | struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); | |
631 | ||
632 | crypto_drop_shash(&ctx->spawn); | |
633 | kfree(inst); | |
634 | } | |
635 | ||
9cd899a3 | 636 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
7bcb2c99 | 637 | struct crypto_attr_type *algt, |
9cd899a3 | 638 | struct cryptd_queue *queue) |
b8a28251 | 639 | { |
46309d89 | 640 | struct hashd_instance_ctx *ctx; |
0b535adf | 641 | struct ahash_instance *inst; |
218c5035 | 642 | struct shash_alg *alg; |
7bcb2c99 EB |
643 | u32 type; |
644 | u32 mask; | |
46309d89 | 645 | int err; |
b8a28251 | 646 | |
7bcb2c99 | 647 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 648 | |
218c5035 EB |
649 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
650 | if (!inst) | |
651 | return -ENOMEM; | |
b8a28251 | 652 | |
0b535adf | 653 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
654 | ctx->queue = queue; |
655 | ||
218c5035 EB |
656 | err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), |
657 | crypto_attr_alg_name(tb[1]), type, mask); | |
46309d89 | 658 | if (err) |
218c5035 EB |
659 | goto err_free_inst; |
660 | alg = crypto_spawn_shash_alg(&ctx->spawn); | |
661 | ||
662 | err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); | |
663 | if (err) | |
664 | goto err_free_inst; | |
46309d89 | 665 | |
7bcb2c99 EB |
666 | inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
667 | (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL| | |
218c5035 | 668 | CRYPTO_ALG_OPTIONAL_KEY)); |
218c5035 EB |
669 | inst->alg.halg.digestsize = alg->digestsize; |
670 | inst->alg.halg.statesize = alg->statesize; | |
0b535adf | 671 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); |
b8a28251 | 672 | |
0b535adf HX |
673 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
674 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 675 | |
0b535adf HX |
676 | inst->alg.init = cryptd_hash_init_enqueue; |
677 | inst->alg.update = cryptd_hash_update_enqueue; | |
678 | inst->alg.final = cryptd_hash_final_enqueue; | |
6fba00d1 HX |
679 | inst->alg.finup = cryptd_hash_finup_enqueue; |
680 | inst->alg.export = cryptd_hash_export; | |
681 | inst->alg.import = cryptd_hash_import; | |
218c5035 | 682 | if (crypto_shash_alg_has_setkey(alg)) |
841a3ff3 | 683 | inst->alg.setkey = cryptd_hash_setkey; |
0b535adf | 684 | inst->alg.digest = cryptd_hash_digest_enqueue; |
b8a28251 | 685 | |
758ec5ac EB |
686 | inst->free = cryptd_hash_free; |
687 | ||
0b535adf | 688 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 | 689 | if (err) { |
218c5035 | 690 | err_free_inst: |
b8c0d74a | 691 | cryptd_hash_free(inst); |
9cd899a3 | 692 | } |
9cd899a3 | 693 | return err; |
b8a28251 LH |
694 | } |
695 | ||
92b9876b HX |
696 | static int cryptd_aead_setkey(struct crypto_aead *parent, |
697 | const u8 *key, unsigned int keylen) | |
698 | { | |
699 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
700 | struct crypto_aead *child = ctx->child; | |
701 | ||
702 | return crypto_aead_setkey(child, key, keylen); | |
703 | } | |
704 | ||
705 | static int cryptd_aead_setauthsize(struct crypto_aead *parent, | |
706 | unsigned int authsize) | |
707 | { | |
708 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
709 | struct crypto_aead *child = ctx->child; | |
710 | ||
711 | return crypto_aead_setauthsize(child, authsize); | |
712 | } | |
713 | ||
298c926c AH |
714 | static void cryptd_aead_crypt(struct aead_request *req, |
715 | struct crypto_aead *child, | |
716 | int err, | |
717 | int (*crypt)(struct aead_request *req)) | |
718 | { | |
719 | struct cryptd_aead_request_ctx *rctx; | |
81760ea6 | 720 | struct cryptd_aead_ctx *ctx; |
ec9f2006 | 721 | crypto_completion_t compl; |
81760ea6 HX |
722 | struct crypto_aead *tfm; |
723 | int refcnt; | |
ec9f2006 | 724 | |
298c926c | 725 | rctx = aead_request_ctx(req); |
ec9f2006 | 726 | compl = rctx->complete; |
298c926c | 727 | |
31bd44e7 HX |
728 | tfm = crypto_aead_reqtfm(req); |
729 | ||
298c926c AH |
730 | if (unlikely(err == -EINPROGRESS)) |
731 | goto out; | |
732 | aead_request_set_tfm(req, child); | |
733 | err = crypt( req ); | |
81760ea6 | 734 | |
298c926c | 735 | out: |
81760ea6 | 736 | ctx = crypto_aead_ctx(tfm); |
43b970fa | 737 | refcnt = refcount_read(&ctx->refcnt); |
81760ea6 | 738 | |
298c926c | 739 | local_bh_disable(); |
ec9f2006 | 740 | compl(&req->base, err); |
298c926c | 741 | local_bh_enable(); |
81760ea6 | 742 | |
43b970fa | 743 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 744 | crypto_free_aead(tfm); |
298c926c AH |
745 | } |
746 | ||
747 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
748 | { | |
749 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
750 | struct crypto_aead *child = ctx->child; | |
751 | struct aead_request *req; | |
752 | ||
753 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 754 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); |
298c926c AH |
755 | } |
756 | ||
757 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
758 | { | |
759 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
760 | struct crypto_aead *child = ctx->child; | |
761 | struct aead_request *req; | |
762 | ||
763 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 764 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); |
298c926c AH |
765 | } |
766 | ||
767 | static int cryptd_aead_enqueue(struct aead_request *req, | |
3e3dc25f | 768 | crypto_completion_t compl) |
298c926c AH |
769 | { |
770 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
771 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
772 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
773 | ||
774 | rctx->complete = req->base.complete; | |
3e3dc25f | 775 | req->base.complete = compl; |
298c926c AH |
776 | return cryptd_enqueue_request(queue, &req->base); |
777 | } | |
778 | ||
779 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
780 | { | |
781 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
782 | } | |
783 | ||
784 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
785 | { | |
786 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
787 | } | |
788 | ||
f614e546 | 789 | static int cryptd_aead_init_tfm(struct crypto_aead *tfm) |
298c926c | 790 | { |
f614e546 HX |
791 | struct aead_instance *inst = aead_alg_instance(tfm); |
792 | struct aead_instance_ctx *ictx = aead_instance_ctx(inst); | |
298c926c | 793 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; |
f614e546 | 794 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
795 | struct crypto_aead *cipher; |
796 | ||
797 | cipher = crypto_spawn_aead(spawn); | |
798 | if (IS_ERR(cipher)) | |
799 | return PTR_ERR(cipher); | |
800 | ||
298c926c | 801 | ctx->child = cipher; |
ec9f2006 HX |
802 | crypto_aead_set_reqsize( |
803 | tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), | |
804 | crypto_aead_reqsize(cipher))); | |
298c926c AH |
805 | return 0; |
806 | } | |
807 | ||
f614e546 | 808 | static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) |
298c926c | 809 | { |
f614e546 | 810 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
811 | crypto_free_aead(ctx->child); |
812 | } | |
813 | ||
758ec5ac EB |
814 | static void cryptd_aead_free(struct aead_instance *inst) |
815 | { | |
816 | struct aead_instance_ctx *ctx = aead_instance_ctx(inst); | |
817 | ||
818 | crypto_drop_aead(&ctx->aead_spawn); | |
819 | kfree(inst); | |
820 | } | |
821 | ||
298c926c AH |
822 | static int cryptd_create_aead(struct crypto_template *tmpl, |
823 | struct rtattr **tb, | |
7bcb2c99 | 824 | struct crypto_attr_type *algt, |
298c926c AH |
825 | struct cryptd_queue *queue) |
826 | { | |
827 | struct aead_instance_ctx *ctx; | |
f614e546 HX |
828 | struct aead_instance *inst; |
829 | struct aead_alg *alg; | |
7bcb2c99 EB |
830 | u32 type; |
831 | u32 mask; | |
298c926c AH |
832 | int err; |
833 | ||
7bcb2c99 | 834 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 835 | |
9b8c456e HX |
836 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
837 | if (!inst) | |
838 | return -ENOMEM; | |
298c926c | 839 | |
f614e546 | 840 | ctx = aead_instance_ctx(inst); |
298c926c AH |
841 | ctx->queue = queue; |
842 | ||
cd900f0c | 843 | err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), |
b8c0d74a | 844 | crypto_attr_alg_name(tb[1]), type, mask); |
298c926c | 845 | if (err) |
b8c0d74a | 846 | goto err_free_inst; |
298c926c | 847 | |
f614e546 HX |
848 | alg = crypto_spawn_aead_alg(&ctx->aead_spawn); |
849 | err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); | |
9b8c456e | 850 | if (err) |
b8c0d74a | 851 | goto err_free_inst; |
9b8c456e | 852 | |
7bcb2c99 EB |
853 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
854 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
f614e546 | 855 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); |
298c926c | 856 | |
f614e546 HX |
857 | inst->alg.ivsize = crypto_aead_alg_ivsize(alg); |
858 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); | |
859 | ||
860 | inst->alg.init = cryptd_aead_init_tfm; | |
861 | inst->alg.exit = cryptd_aead_exit_tfm; | |
862 | inst->alg.setkey = cryptd_aead_setkey; | |
863 | inst->alg.setauthsize = cryptd_aead_setauthsize; | |
864 | inst->alg.encrypt = cryptd_aead_encrypt_enqueue; | |
865 | inst->alg.decrypt = cryptd_aead_decrypt_enqueue; | |
866 | ||
758ec5ac EB |
867 | inst->free = cryptd_aead_free; |
868 | ||
f614e546 | 869 | err = aead_register_instance(tmpl, inst); |
298c926c | 870 | if (err) { |
b8c0d74a EB |
871 | err_free_inst: |
872 | cryptd_aead_free(inst); | |
298c926c | 873 | } |
298c926c AH |
874 | return err; |
875 | } | |
876 | ||
254eff77 | 877 | static struct cryptd_queue queue; |
124b53d0 | 878 | |
9cd899a3 | 879 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
880 | { |
881 | struct crypto_attr_type *algt; | |
882 | ||
883 | algt = crypto_get_attr_type(tb); | |
884 | if (IS_ERR(algt)) | |
9cd899a3 | 885 | return PTR_ERR(algt); |
124b53d0 HX |
886 | |
887 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
c65058b7 | 888 | case CRYPTO_ALG_TYPE_SKCIPHER: |
7bcb2c99 | 889 | return cryptd_create_skcipher(tmpl, tb, algt, &queue); |
84ede58d | 890 | case CRYPTO_ALG_TYPE_HASH: |
7bcb2c99 | 891 | return cryptd_create_hash(tmpl, tb, algt, &queue); |
298c926c | 892 | case CRYPTO_ALG_TYPE_AEAD: |
7bcb2c99 | 893 | return cryptd_create_aead(tmpl, tb, algt, &queue); |
124b53d0 HX |
894 | } |
895 | ||
9cd899a3 | 896 | return -EINVAL; |
124b53d0 HX |
897 | } |
898 | ||
124b53d0 HX |
899 | static struct crypto_template cryptd_tmpl = { |
900 | .name = "cryptd", | |
9cd899a3 | 901 | .create = cryptd_create, |
124b53d0 HX |
902 | .module = THIS_MODULE, |
903 | }; | |
904 | ||
4e0958d1 HX |
905 | struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, |
906 | u32 type, u32 mask) | |
907 | { | |
908 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
909 | struct cryptd_skcipher_ctx *ctx; | |
910 | struct crypto_skcipher *tfm; | |
911 | ||
912 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
913 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
914 | return ERR_PTR(-EINVAL); | |
915 | ||
916 | tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); | |
917 | if (IS_ERR(tfm)) | |
918 | return ERR_CAST(tfm); | |
919 | ||
920 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
921 | crypto_free_skcipher(tfm); | |
922 | return ERR_PTR(-EINVAL); | |
923 | } | |
924 | ||
925 | ctx = crypto_skcipher_ctx(tfm); | |
43b970fa | 926 | refcount_set(&ctx->refcnt, 1); |
4e0958d1 HX |
927 | |
928 | return container_of(tfm, struct cryptd_skcipher, base); | |
929 | } | |
930 | EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); | |
931 | ||
932 | struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) | |
933 | { | |
934 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
935 | ||
3a58c231 | 936 | return ctx->child; |
4e0958d1 HX |
937 | } |
938 | EXPORT_SYMBOL_GPL(cryptd_skcipher_child); | |
939 | ||
940 | bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) | |
941 | { | |
942 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
943 | ||
43b970fa | 944 | return refcount_read(&ctx->refcnt) - 1; |
4e0958d1 HX |
945 | } |
946 | EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); | |
947 | ||
948 | void cryptd_free_skcipher(struct cryptd_skcipher *tfm) | |
949 | { | |
950 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
951 | ||
43b970fa | 952 | if (refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
953 | crypto_free_skcipher(&tfm->base); |
954 | } | |
955 | EXPORT_SYMBOL_GPL(cryptd_free_skcipher); | |
956 | ||
ace13663 HY |
957 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, |
958 | u32 type, u32 mask) | |
959 | { | |
960 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 961 | struct cryptd_hash_ctx *ctx; |
ace13663 HY |
962 | struct crypto_ahash *tfm; |
963 | ||
964 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
965 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
966 | return ERR_PTR(-EINVAL); | |
967 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
968 | if (IS_ERR(tfm)) | |
969 | return ERR_CAST(tfm); | |
970 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
971 | crypto_free_ahash(tfm); | |
972 | return ERR_PTR(-EINVAL); | |
973 | } | |
974 | ||
81760ea6 | 975 | ctx = crypto_ahash_ctx(tfm); |
43b970fa | 976 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 977 | |
ace13663 HY |
978 | return __cryptd_ahash_cast(tfm); |
979 | } | |
980 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
981 | ||
982 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
983 | { | |
984 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
985 | ||
986 | return ctx->child; | |
987 | } | |
988 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
989 | ||
0e1227d3 HY |
990 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) |
991 | { | |
992 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
993 | return &rctx->desc; | |
994 | } | |
995 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
996 | ||
81760ea6 HX |
997 | bool cryptd_ahash_queued(struct cryptd_ahash *tfm) |
998 | { | |
999 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
1000 | ||
43b970fa | 1001 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1002 | } |
1003 | EXPORT_SYMBOL_GPL(cryptd_ahash_queued); | |
1004 | ||
ace13663 HY |
1005 | void cryptd_free_ahash(struct cryptd_ahash *tfm) |
1006 | { | |
81760ea6 HX |
1007 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); |
1008 | ||
43b970fa | 1009 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1010 | crypto_free_ahash(&tfm->base); |
ace13663 HY |
1011 | } |
1012 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
1013 | ||
298c926c AH |
1014 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, |
1015 | u32 type, u32 mask) | |
1016 | { | |
1017 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 1018 | struct cryptd_aead_ctx *ctx; |
298c926c AH |
1019 | struct crypto_aead *tfm; |
1020 | ||
1021 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
1022 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
1023 | return ERR_PTR(-EINVAL); | |
1024 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
1025 | if (IS_ERR(tfm)) | |
1026 | return ERR_CAST(tfm); | |
1027 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1028 | crypto_free_aead(tfm); | |
1029 | return ERR_PTR(-EINVAL); | |
1030 | } | |
81760ea6 HX |
1031 | |
1032 | ctx = crypto_aead_ctx(tfm); | |
43b970fa | 1033 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 1034 | |
298c926c AH |
1035 | return __cryptd_aead_cast(tfm); |
1036 | } | |
1037 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
1038 | ||
1039 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
1040 | { | |
1041 | struct cryptd_aead_ctx *ctx; | |
1042 | ctx = crypto_aead_ctx(&tfm->base); | |
1043 | return ctx->child; | |
1044 | } | |
1045 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
1046 | ||
81760ea6 HX |
1047 | bool cryptd_aead_queued(struct cryptd_aead *tfm) |
1048 | { | |
1049 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); | |
1050 | ||
43b970fa | 1051 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1052 | } |
1053 | EXPORT_SYMBOL_GPL(cryptd_aead_queued); | |
1054 | ||
298c926c AH |
1055 | void cryptd_free_aead(struct cryptd_aead *tfm) |
1056 | { | |
81760ea6 HX |
1057 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); |
1058 | ||
43b970fa | 1059 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1060 | crypto_free_aead(&tfm->base); |
298c926c AH |
1061 | } |
1062 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
1063 | ||
124b53d0 HX |
1064 | static int __init cryptd_init(void) |
1065 | { | |
1066 | int err; | |
1067 | ||
3e56e168 EB |
1068 | cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, |
1069 | 1); | |
1070 | if (!cryptd_wq) | |
1071 | return -ENOMEM; | |
1072 | ||
c3a53605 | 1073 | err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); |
124b53d0 | 1074 | if (err) |
3e56e168 | 1075 | goto err_destroy_wq; |
124b53d0 HX |
1076 | |
1077 | err = crypto_register_template(&cryptd_tmpl); | |
1078 | if (err) | |
3e56e168 | 1079 | goto err_fini_queue; |
124b53d0 | 1080 | |
3e56e168 EB |
1081 | return 0; |
1082 | ||
1083 | err_fini_queue: | |
1084 | cryptd_fini_queue(&queue); | |
1085 | err_destroy_wq: | |
1086 | destroy_workqueue(cryptd_wq); | |
124b53d0 HX |
1087 | return err; |
1088 | } | |
1089 | ||
1090 | static void __exit cryptd_exit(void) | |
1091 | { | |
3e56e168 | 1092 | destroy_workqueue(cryptd_wq); |
254eff77 | 1093 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1094 | crypto_unregister_template(&cryptd_tmpl); |
1095 | } | |
1096 | ||
b2bac6ac | 1097 | subsys_initcall(cryptd_init); |
124b53d0 HX |
1098 | module_exit(cryptd_exit); |
1099 | ||
1100 | MODULE_LICENSE("GPL"); | |
1101 | MODULE_DESCRIPTION("Software async crypto daemon"); | |
4943ba16 | 1102 | MODULE_ALIAS_CRYPTO("cryptd"); |