Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
124b53d0 HX |
2 | /* |
3 | * Software async crypto daemon. | |
4 | * | |
5 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
6 | * | |
298c926c AH |
7 | * Added AEAD support to cryptd. |
8 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
9 | * Adrian Hoban <adrian.hoban@intel.com> | |
10 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
11 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
12 | * Copyright (c) 2010, Intel Corporation. | |
124b53d0 HX |
13 | */ |
14 | ||
18e33e6d | 15 | #include <crypto/internal/hash.h> |
298c926c | 16 | #include <crypto/internal/aead.h> |
4e0958d1 | 17 | #include <crypto/internal/skcipher.h> |
1cac2cbc | 18 | #include <crypto/cryptd.h> |
43b970fa | 19 | #include <linux/refcount.h> |
124b53d0 HX |
20 | #include <linux/err.h> |
21 | #include <linux/init.h> | |
22 | #include <linux/kernel.h> | |
124b53d0 HX |
23 | #include <linux/list.h> |
24 | #include <linux/module.h> | |
124b53d0 HX |
25 | #include <linux/scatterlist.h> |
26 | #include <linux/sched.h> | |
27 | #include <linux/slab.h> | |
3e56e168 | 28 | #include <linux/workqueue.h> |
124b53d0 | 29 | |
eaf356e4 | 30 | static unsigned int cryptd_max_cpu_qlen = 1000; |
c3a53605 JM |
31 | module_param(cryptd_max_cpu_qlen, uint, 0); |
32 | MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); | |
124b53d0 | 33 | |
3e56e168 EB |
34 | static struct workqueue_struct *cryptd_wq; |
35 | ||
254eff77 | 36 | struct cryptd_cpu_queue { |
124b53d0 | 37 | struct crypto_queue queue; |
254eff77 HY |
38 | struct work_struct work; |
39 | }; | |
40 | ||
41 | struct cryptd_queue { | |
91e8bcd7 SAS |
42 | /* |
43 | * Protected by disabling BH to allow enqueueing from softinterrupt and | |
44 | * dequeuing from kworker (cryptd_queue_worker()). | |
45 | */ | |
a29d8b8e | 46 | struct cryptd_cpu_queue __percpu *cpu_queue; |
124b53d0 HX |
47 | }; |
48 | ||
49 | struct cryptd_instance_ctx { | |
50 | struct crypto_spawn spawn; | |
254eff77 | 51 | struct cryptd_queue *queue; |
124b53d0 HX |
52 | }; |
53 | ||
4e0958d1 HX |
54 | struct skcipherd_instance_ctx { |
55 | struct crypto_skcipher_spawn spawn; | |
56 | struct cryptd_queue *queue; | |
57 | }; | |
58 | ||
46309d89 HX |
59 | struct hashd_instance_ctx { |
60 | struct crypto_shash_spawn spawn; | |
61 | struct cryptd_queue *queue; | |
62 | }; | |
63 | ||
298c926c AH |
64 | struct aead_instance_ctx { |
65 | struct crypto_aead_spawn aead_spawn; | |
66 | struct cryptd_queue *queue; | |
67 | }; | |
68 | ||
4e0958d1 | 69 | struct cryptd_skcipher_ctx { |
43b970fa | 70 | refcount_t refcnt; |
3a58c231 | 71 | struct crypto_skcipher *child; |
4e0958d1 HX |
72 | }; |
73 | ||
74 | struct cryptd_skcipher_request_ctx { | |
75 | crypto_completion_t complete; | |
3a58c231 | 76 | struct skcipher_request req; |
4e0958d1 HX |
77 | }; |
78 | ||
b8a28251 | 79 | struct cryptd_hash_ctx { |
43b970fa | 80 | refcount_t refcnt; |
46309d89 | 81 | struct crypto_shash *child; |
b8a28251 LH |
82 | }; |
83 | ||
84 | struct cryptd_hash_request_ctx { | |
85 | crypto_completion_t complete; | |
46309d89 | 86 | struct shash_desc desc; |
b8a28251 | 87 | }; |
124b53d0 | 88 | |
298c926c | 89 | struct cryptd_aead_ctx { |
43b970fa | 90 | refcount_t refcnt; |
298c926c AH |
91 | struct crypto_aead *child; |
92 | }; | |
93 | ||
94 | struct cryptd_aead_request_ctx { | |
95 | crypto_completion_t complete; | |
96 | }; | |
97 | ||
254eff77 HY |
98 | static void cryptd_queue_worker(struct work_struct *work); |
99 | ||
100 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
101 | unsigned int max_cpu_qlen) | |
102 | { | |
103 | int cpu; | |
104 | struct cryptd_cpu_queue *cpu_queue; | |
105 | ||
106 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
107 | if (!queue->cpu_queue) | |
108 | return -ENOMEM; | |
109 | for_each_possible_cpu(cpu) { | |
110 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
111 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
112 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
113 | } | |
c3a53605 | 114 | pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); |
254eff77 HY |
115 | return 0; |
116 | } | |
117 | ||
118 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
119 | { | |
120 | int cpu; | |
121 | struct cryptd_cpu_queue *cpu_queue; | |
122 | ||
123 | for_each_possible_cpu(cpu) { | |
124 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
125 | BUG_ON(cpu_queue->queue.qlen); | |
126 | } | |
127 | free_percpu(queue->cpu_queue); | |
128 | } | |
129 | ||
130 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
131 | struct crypto_async_request *request) | |
132 | { | |
91e8bcd7 | 133 | int err; |
254eff77 | 134 | struct cryptd_cpu_queue *cpu_queue; |
43b970fa | 135 | refcount_t *refcnt; |
254eff77 | 136 | |
91e8bcd7 | 137 | local_bh_disable(); |
0b44f486 | 138 | cpu_queue = this_cpu_ptr(queue->cpu_queue); |
254eff77 | 139 | err = crypto_enqueue_request(&cpu_queue->queue, request); |
81760ea6 HX |
140 | |
141 | refcnt = crypto_tfm_ctx(request->tfm); | |
81760ea6 | 142 | |
6b80ea38 | 143 | if (err == -ENOSPC) |
91e8bcd7 | 144 | goto out; |
81760ea6 | 145 | |
91e8bcd7 | 146 | queue_work_on(smp_processor_id(), cryptd_wq, &cpu_queue->work); |
81760ea6 | 147 | |
43b970fa | 148 | if (!refcount_read(refcnt)) |
91e8bcd7 | 149 | goto out; |
81760ea6 | 150 | |
43b970fa | 151 | refcount_inc(refcnt); |
81760ea6 | 152 | |
91e8bcd7 SAS |
153 | out: |
154 | local_bh_enable(); | |
254eff77 HY |
155 | |
156 | return err; | |
157 | } | |
158 | ||
159 | /* Called in workqueue context, do one real cryption work (via | |
160 | * req->complete) and reschedule itself if there are more work to | |
161 | * do. */ | |
162 | static void cryptd_queue_worker(struct work_struct *work) | |
163 | { | |
164 | struct cryptd_cpu_queue *cpu_queue; | |
165 | struct crypto_async_request *req, *backlog; | |
166 | ||
167 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
9efade1b JK |
168 | /* |
169 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
9efade1b JK |
170 | */ |
171 | local_bh_disable(); | |
254eff77 HY |
172 | backlog = crypto_get_backlog(&cpu_queue->queue); |
173 | req = crypto_dequeue_request(&cpu_queue->queue); | |
9efade1b | 174 | local_bh_enable(); |
254eff77 HY |
175 | |
176 | if (!req) | |
177 | return; | |
178 | ||
179 | if (backlog) | |
180 | backlog->complete(backlog, -EINPROGRESS); | |
181 | req->complete(req, 0); | |
182 | ||
183 | if (cpu_queue->queue.qlen) | |
3e56e168 | 184 | queue_work(cryptd_wq, &cpu_queue->work); |
254eff77 HY |
185 | } |
186 | ||
187 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
188 | { |
189 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
190 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 191 | return ictx->queue; |
124b53d0 HX |
192 | } |
193 | ||
7bcb2c99 EB |
194 | static void cryptd_type_and_mask(struct crypto_attr_type *algt, |
195 | u32 *type, u32 *mask) | |
466a7b9e | 196 | { |
7bcb2c99 EB |
197 | /* |
198 | * cryptd is allowed to wrap internal algorithms, but in that case the | |
199 | * resulting cryptd instance will be marked as internal as well. | |
200 | */ | |
201 | *type = algt->type & CRYPTO_ALG_INTERNAL; | |
202 | *mask = algt->mask & CRYPTO_ALG_INTERNAL; | |
466a7b9e | 203 | |
7bcb2c99 EB |
204 | /* No point in cryptd wrapping an algorithm that's already async. */ |
205 | *mask |= CRYPTO_ALG_ASYNC; | |
f6da3205 | 206 | |
7bcb2c99 | 207 | *mask |= crypto_algt_inherited_mask(algt); |
466a7b9e SM |
208 | } |
209 | ||
9b8c456e HX |
210 | static int cryptd_init_instance(struct crypto_instance *inst, |
211 | struct crypto_alg *alg) | |
212 | { | |
213 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
214 | "cryptd(%s)", | |
215 | alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
216 | return -ENAMETOOLONG; | |
217 | ||
218 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
219 | ||
220 | inst->alg.cra_priority = alg->cra_priority + 50; | |
221 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
222 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
223 | ||
224 | return 0; | |
225 | } | |
226 | ||
4e0958d1 HX |
227 | static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, |
228 | const u8 *key, unsigned int keylen) | |
229 | { | |
230 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); | |
3a58c231 | 231 | struct crypto_skcipher *child = ctx->child; |
4e0958d1 | 232 | |
3a58c231 HX |
233 | crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
234 | crypto_skcipher_set_flags(child, | |
235 | crypto_skcipher_get_flags(parent) & | |
236 | CRYPTO_TFM_REQ_MASK); | |
237 | return crypto_skcipher_setkey(child, key, keylen); | |
4e0958d1 HX |
238 | } |
239 | ||
240 | static void cryptd_skcipher_complete(struct skcipher_request *req, int err) | |
241 | { | |
242 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
243 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
244 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
43b970fa | 245 | int refcnt = refcount_read(&ctx->refcnt); |
4e0958d1 HX |
246 | |
247 | local_bh_disable(); | |
248 | rctx->complete(&req->base, err); | |
249 | local_bh_enable(); | |
250 | ||
43b970fa | 251 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
252 | crypto_free_skcipher(tfm); |
253 | } | |
254 | ||
255 | static void cryptd_skcipher_encrypt(struct crypto_async_request *base, | |
256 | int err) | |
257 | { | |
258 | struct skcipher_request *req = skcipher_request_cast(base); | |
259 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
260 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
261 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
3a58c231 HX |
262 | struct skcipher_request *subreq = &rctx->req; |
263 | struct crypto_skcipher *child = ctx->child; | |
4e0958d1 HX |
264 | |
265 | if (unlikely(err == -EINPROGRESS)) | |
266 | goto out; | |
267 | ||
3a58c231 | 268 | skcipher_request_set_tfm(subreq, child); |
4e0958d1 HX |
269 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
270 | NULL, NULL); | |
271 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
272 | req->iv); | |
273 | ||
274 | err = crypto_skcipher_encrypt(subreq); | |
4e0958d1 HX |
275 | |
276 | req->base.complete = rctx->complete; | |
277 | ||
278 | out: | |
279 | cryptd_skcipher_complete(req, err); | |
280 | } | |
281 | ||
282 | static void cryptd_skcipher_decrypt(struct crypto_async_request *base, | |
283 | int err) | |
284 | { | |
285 | struct skcipher_request *req = skcipher_request_cast(base); | |
286 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
287 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
288 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
3a58c231 HX |
289 | struct skcipher_request *subreq = &rctx->req; |
290 | struct crypto_skcipher *child = ctx->child; | |
4e0958d1 HX |
291 | |
292 | if (unlikely(err == -EINPROGRESS)) | |
293 | goto out; | |
294 | ||
3a58c231 | 295 | skcipher_request_set_tfm(subreq, child); |
4e0958d1 HX |
296 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
297 | NULL, NULL); | |
298 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
299 | req->iv); | |
300 | ||
301 | err = crypto_skcipher_decrypt(subreq); | |
4e0958d1 HX |
302 | |
303 | req->base.complete = rctx->complete; | |
304 | ||
305 | out: | |
306 | cryptd_skcipher_complete(req, err); | |
307 | } | |
308 | ||
309 | static int cryptd_skcipher_enqueue(struct skcipher_request *req, | |
310 | crypto_completion_t compl) | |
311 | { | |
312 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
313 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
314 | struct cryptd_queue *queue; | |
315 | ||
316 | queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); | |
317 | rctx->complete = req->base.complete; | |
318 | req->base.complete = compl; | |
319 | ||
320 | return cryptd_enqueue_request(queue, &req->base); | |
321 | } | |
322 | ||
323 | static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) | |
324 | { | |
325 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); | |
326 | } | |
327 | ||
328 | static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) | |
329 | { | |
330 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); | |
331 | } | |
332 | ||
333 | static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) | |
334 | { | |
335 | struct skcipher_instance *inst = skcipher_alg_instance(tfm); | |
336 | struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); | |
337 | struct crypto_skcipher_spawn *spawn = &ictx->spawn; | |
338 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
339 | struct crypto_skcipher *cipher; | |
340 | ||
341 | cipher = crypto_spawn_skcipher(spawn); | |
342 | if (IS_ERR(cipher)) | |
343 | return PTR_ERR(cipher); | |
344 | ||
3a58c231 | 345 | ctx->child = cipher; |
4e0958d1 | 346 | crypto_skcipher_set_reqsize( |
3a58c231 HX |
347 | tfm, sizeof(struct cryptd_skcipher_request_ctx) + |
348 | crypto_skcipher_reqsize(cipher)); | |
4e0958d1 HX |
349 | return 0; |
350 | } | |
351 | ||
352 | static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) | |
353 | { | |
354 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
355 | ||
3a58c231 | 356 | crypto_free_skcipher(ctx->child); |
4e0958d1 HX |
357 | } |
358 | ||
359 | static void cryptd_skcipher_free(struct skcipher_instance *inst) | |
360 | { | |
361 | struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); | |
362 | ||
363 | crypto_drop_skcipher(&ctx->spawn); | |
1a0fad63 | 364 | kfree(inst); |
4e0958d1 HX |
365 | } |
366 | ||
367 | static int cryptd_create_skcipher(struct crypto_template *tmpl, | |
368 | struct rtattr **tb, | |
7bcb2c99 | 369 | struct crypto_attr_type *algt, |
4e0958d1 HX |
370 | struct cryptd_queue *queue) |
371 | { | |
372 | struct skcipherd_instance_ctx *ctx; | |
373 | struct skcipher_instance *inst; | |
374 | struct skcipher_alg *alg; | |
4e0958d1 HX |
375 | u32 type; |
376 | u32 mask; | |
377 | int err; | |
378 | ||
7bcb2c99 | 379 | cryptd_type_and_mask(algt, &type, &mask); |
4e0958d1 | 380 | |
4e0958d1 HX |
381 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
382 | if (!inst) | |
383 | return -ENOMEM; | |
384 | ||
385 | ctx = skcipher_instance_ctx(inst); | |
386 | ctx->queue = queue; | |
387 | ||
b9f76ddd | 388 | err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), |
b8c0d74a | 389 | crypto_attr_alg_name(tb[1]), type, mask); |
4e0958d1 | 390 | if (err) |
b8c0d74a | 391 | goto err_free_inst; |
4e0958d1 HX |
392 | |
393 | alg = crypto_spawn_skcipher_alg(&ctx->spawn); | |
394 | err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); | |
395 | if (err) | |
b8c0d74a | 396 | goto err_free_inst; |
4e0958d1 | 397 | |
7bcb2c99 EB |
398 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
399 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
4e0958d1 HX |
400 | inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg); |
401 | inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); | |
402 | inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); | |
403 | inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg); | |
404 | ||
405 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); | |
406 | ||
407 | inst->alg.init = cryptd_skcipher_init_tfm; | |
408 | inst->alg.exit = cryptd_skcipher_exit_tfm; | |
409 | ||
410 | inst->alg.setkey = cryptd_skcipher_setkey; | |
411 | inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; | |
412 | inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; | |
413 | ||
414 | inst->free = cryptd_skcipher_free; | |
415 | ||
416 | err = skcipher_register_instance(tmpl, inst); | |
417 | if (err) { | |
b8c0d74a EB |
418 | err_free_inst: |
419 | cryptd_skcipher_free(inst); | |
4e0958d1 HX |
420 | } |
421 | return err; | |
422 | } | |
423 | ||
b8a28251 LH |
424 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
425 | { | |
426 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
427 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
428 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 429 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 430 | struct crypto_shash *hash; |
b8a28251 | 431 | |
46309d89 HX |
432 | hash = crypto_spawn_shash(spawn); |
433 | if (IS_ERR(hash)) | |
434 | return PTR_ERR(hash); | |
b8a28251 | 435 | |
46309d89 | 436 | ctx->child = hash; |
0d6669e2 HX |
437 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
438 | sizeof(struct cryptd_hash_request_ctx) + | |
439 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
440 | return 0; |
441 | } | |
442 | ||
443 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
444 | { | |
445 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 446 | |
46309d89 | 447 | crypto_free_shash(ctx->child); |
b8a28251 LH |
448 | } |
449 | ||
450 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
451 | const u8 *key, unsigned int keylen) | |
452 | { | |
453 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 454 | struct crypto_shash *child = ctx->child; |
b8a28251 | 455 | |
46309d89 HX |
456 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
457 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
458 | CRYPTO_TFM_REQ_MASK); | |
af5034e8 | 459 | return crypto_shash_setkey(child, key, keylen); |
b8a28251 LH |
460 | } |
461 | ||
462 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
3e3dc25f | 463 | crypto_completion_t compl) |
b8a28251 LH |
464 | { |
465 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
466 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
467 | struct cryptd_queue *queue = |
468 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
469 | |
470 | rctx->complete = req->base.complete; | |
3e3dc25f | 471 | req->base.complete = compl; |
b8a28251 | 472 | |
254eff77 | 473 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
474 | } |
475 | ||
81760ea6 HX |
476 | static void cryptd_hash_complete(struct ahash_request *req, int err) |
477 | { | |
478 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
479 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
480 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
43b970fa | 481 | int refcnt = refcount_read(&ctx->refcnt); |
81760ea6 HX |
482 | |
483 | local_bh_disable(); | |
484 | rctx->complete(&req->base, err); | |
485 | local_bh_enable(); | |
486 | ||
43b970fa | 487 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 HX |
488 | crypto_free_ahash(tfm); |
489 | } | |
490 | ||
b8a28251 LH |
491 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) |
492 | { | |
46309d89 HX |
493 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
494 | struct crypto_shash *child = ctx->child; | |
495 | struct ahash_request *req = ahash_request_cast(req_async); | |
496 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
497 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
498 | |
499 | if (unlikely(err == -EINPROGRESS)) | |
500 | goto out; | |
501 | ||
46309d89 | 502 | desc->tfm = child; |
b8a28251 | 503 | |
46309d89 | 504 | err = crypto_shash_init(desc); |
b8a28251 LH |
505 | |
506 | req->base.complete = rctx->complete; | |
507 | ||
508 | out: | |
81760ea6 | 509 | cryptd_hash_complete(req, err); |
b8a28251 LH |
510 | } |
511 | ||
512 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
513 | { | |
514 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
515 | } | |
516 | ||
517 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
518 | { | |
46309d89 | 519 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 520 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
521 | |
522 | rctx = ahash_request_ctx(req); | |
523 | ||
524 | if (unlikely(err == -EINPROGRESS)) | |
525 | goto out; | |
526 | ||
46309d89 | 527 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
528 | |
529 | req->base.complete = rctx->complete; | |
530 | ||
531 | out: | |
81760ea6 | 532 | cryptd_hash_complete(req, err); |
b8a28251 LH |
533 | } |
534 | ||
535 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
536 | { | |
537 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
538 | } | |
539 | ||
540 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
541 | { | |
46309d89 HX |
542 | struct ahash_request *req = ahash_request_cast(req_async); |
543 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
544 | |
545 | if (unlikely(err == -EINPROGRESS)) | |
546 | goto out; | |
547 | ||
46309d89 | 548 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
549 | |
550 | req->base.complete = rctx->complete; | |
551 | ||
552 | out: | |
81760ea6 | 553 | cryptd_hash_complete(req, err); |
b8a28251 LH |
554 | } |
555 | ||
556 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
557 | { | |
558 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
559 | } | |
560 | ||
6fba00d1 HX |
561 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) |
562 | { | |
563 | struct ahash_request *req = ahash_request_cast(req_async); | |
564 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
565 | ||
566 | if (unlikely(err == -EINPROGRESS)) | |
567 | goto out; | |
568 | ||
569 | err = shash_ahash_finup(req, &rctx->desc); | |
570 | ||
571 | req->base.complete = rctx->complete; | |
572 | ||
573 | out: | |
81760ea6 | 574 | cryptd_hash_complete(req, err); |
6fba00d1 HX |
575 | } |
576 | ||
577 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
578 | { | |
579 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
580 | } | |
581 | ||
b8a28251 LH |
582 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) |
583 | { | |
46309d89 HX |
584 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
585 | struct crypto_shash *child = ctx->child; | |
586 | struct ahash_request *req = ahash_request_cast(req_async); | |
587 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
588 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
589 | |
590 | if (unlikely(err == -EINPROGRESS)) | |
591 | goto out; | |
592 | ||
46309d89 | 593 | desc->tfm = child; |
b8a28251 | 594 | |
46309d89 | 595 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
596 | |
597 | req->base.complete = rctx->complete; | |
598 | ||
599 | out: | |
81760ea6 | 600 | cryptd_hash_complete(req, err); |
b8a28251 LH |
601 | } |
602 | ||
603 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
604 | { | |
605 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
606 | } | |
607 | ||
6fba00d1 HX |
608 | static int cryptd_hash_export(struct ahash_request *req, void *out) |
609 | { | |
610 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
611 | ||
612 | return crypto_shash_export(&rctx->desc, out); | |
613 | } | |
614 | ||
615 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
616 | { | |
0bd22235 AB |
617 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
618 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
619 | struct shash_desc *desc = cryptd_shash_desc(req); | |
620 | ||
621 | desc->tfm = ctx->child; | |
6fba00d1 | 622 | |
0bd22235 | 623 | return crypto_shash_import(desc, in); |
6fba00d1 HX |
624 | } |
625 | ||
758ec5ac EB |
626 | static void cryptd_hash_free(struct ahash_instance *inst) |
627 | { | |
628 | struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); | |
629 | ||
630 | crypto_drop_shash(&ctx->spawn); | |
631 | kfree(inst); | |
632 | } | |
633 | ||
9cd899a3 | 634 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
7bcb2c99 | 635 | struct crypto_attr_type *algt, |
9cd899a3 | 636 | struct cryptd_queue *queue) |
b8a28251 | 637 | { |
46309d89 | 638 | struct hashd_instance_ctx *ctx; |
0b535adf | 639 | struct ahash_instance *inst; |
218c5035 | 640 | struct shash_alg *alg; |
7bcb2c99 EB |
641 | u32 type; |
642 | u32 mask; | |
46309d89 | 643 | int err; |
b8a28251 | 644 | |
7bcb2c99 | 645 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 646 | |
218c5035 EB |
647 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
648 | if (!inst) | |
649 | return -ENOMEM; | |
b8a28251 | 650 | |
0b535adf | 651 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
652 | ctx->queue = queue; |
653 | ||
218c5035 EB |
654 | err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), |
655 | crypto_attr_alg_name(tb[1]), type, mask); | |
46309d89 | 656 | if (err) |
218c5035 EB |
657 | goto err_free_inst; |
658 | alg = crypto_spawn_shash_alg(&ctx->spawn); | |
659 | ||
660 | err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); | |
661 | if (err) | |
662 | goto err_free_inst; | |
46309d89 | 663 | |
7bcb2c99 EB |
664 | inst->alg.halg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
665 | (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL| | |
218c5035 | 666 | CRYPTO_ALG_OPTIONAL_KEY)); |
218c5035 EB |
667 | inst->alg.halg.digestsize = alg->digestsize; |
668 | inst->alg.halg.statesize = alg->statesize; | |
0b535adf | 669 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); |
b8a28251 | 670 | |
0b535adf HX |
671 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
672 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 673 | |
0b535adf HX |
674 | inst->alg.init = cryptd_hash_init_enqueue; |
675 | inst->alg.update = cryptd_hash_update_enqueue; | |
676 | inst->alg.final = cryptd_hash_final_enqueue; | |
6fba00d1 HX |
677 | inst->alg.finup = cryptd_hash_finup_enqueue; |
678 | inst->alg.export = cryptd_hash_export; | |
679 | inst->alg.import = cryptd_hash_import; | |
218c5035 | 680 | if (crypto_shash_alg_has_setkey(alg)) |
841a3ff3 | 681 | inst->alg.setkey = cryptd_hash_setkey; |
0b535adf | 682 | inst->alg.digest = cryptd_hash_digest_enqueue; |
b8a28251 | 683 | |
758ec5ac EB |
684 | inst->free = cryptd_hash_free; |
685 | ||
0b535adf | 686 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 | 687 | if (err) { |
218c5035 | 688 | err_free_inst: |
b8c0d74a | 689 | cryptd_hash_free(inst); |
9cd899a3 | 690 | } |
9cd899a3 | 691 | return err; |
b8a28251 LH |
692 | } |
693 | ||
92b9876b HX |
694 | static int cryptd_aead_setkey(struct crypto_aead *parent, |
695 | const u8 *key, unsigned int keylen) | |
696 | { | |
697 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
698 | struct crypto_aead *child = ctx->child; | |
699 | ||
700 | return crypto_aead_setkey(child, key, keylen); | |
701 | } | |
702 | ||
703 | static int cryptd_aead_setauthsize(struct crypto_aead *parent, | |
704 | unsigned int authsize) | |
705 | { | |
706 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
707 | struct crypto_aead *child = ctx->child; | |
708 | ||
709 | return crypto_aead_setauthsize(child, authsize); | |
710 | } | |
711 | ||
298c926c AH |
712 | static void cryptd_aead_crypt(struct aead_request *req, |
713 | struct crypto_aead *child, | |
714 | int err, | |
715 | int (*crypt)(struct aead_request *req)) | |
716 | { | |
717 | struct cryptd_aead_request_ctx *rctx; | |
81760ea6 | 718 | struct cryptd_aead_ctx *ctx; |
ec9f2006 | 719 | crypto_completion_t compl; |
81760ea6 HX |
720 | struct crypto_aead *tfm; |
721 | int refcnt; | |
ec9f2006 | 722 | |
298c926c | 723 | rctx = aead_request_ctx(req); |
ec9f2006 | 724 | compl = rctx->complete; |
298c926c | 725 | |
31bd44e7 HX |
726 | tfm = crypto_aead_reqtfm(req); |
727 | ||
298c926c AH |
728 | if (unlikely(err == -EINPROGRESS)) |
729 | goto out; | |
730 | aead_request_set_tfm(req, child); | |
731 | err = crypt( req ); | |
81760ea6 | 732 | |
298c926c | 733 | out: |
81760ea6 | 734 | ctx = crypto_aead_ctx(tfm); |
43b970fa | 735 | refcnt = refcount_read(&ctx->refcnt); |
81760ea6 | 736 | |
298c926c | 737 | local_bh_disable(); |
ec9f2006 | 738 | compl(&req->base, err); |
298c926c | 739 | local_bh_enable(); |
81760ea6 | 740 | |
43b970fa | 741 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 742 | crypto_free_aead(tfm); |
298c926c AH |
743 | } |
744 | ||
745 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
746 | { | |
747 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
748 | struct crypto_aead *child = ctx->child; | |
749 | struct aead_request *req; | |
750 | ||
751 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 752 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); |
298c926c AH |
753 | } |
754 | ||
755 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
756 | { | |
757 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
758 | struct crypto_aead *child = ctx->child; | |
759 | struct aead_request *req; | |
760 | ||
761 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 762 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); |
298c926c AH |
763 | } |
764 | ||
765 | static int cryptd_aead_enqueue(struct aead_request *req, | |
3e3dc25f | 766 | crypto_completion_t compl) |
298c926c AH |
767 | { |
768 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
769 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
770 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
771 | ||
772 | rctx->complete = req->base.complete; | |
3e3dc25f | 773 | req->base.complete = compl; |
298c926c AH |
774 | return cryptd_enqueue_request(queue, &req->base); |
775 | } | |
776 | ||
777 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
778 | { | |
779 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
780 | } | |
781 | ||
782 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
783 | { | |
784 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
785 | } | |
786 | ||
f614e546 | 787 | static int cryptd_aead_init_tfm(struct crypto_aead *tfm) |
298c926c | 788 | { |
f614e546 HX |
789 | struct aead_instance *inst = aead_alg_instance(tfm); |
790 | struct aead_instance_ctx *ictx = aead_instance_ctx(inst); | |
298c926c | 791 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; |
f614e546 | 792 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
793 | struct crypto_aead *cipher; |
794 | ||
795 | cipher = crypto_spawn_aead(spawn); | |
796 | if (IS_ERR(cipher)) | |
797 | return PTR_ERR(cipher); | |
798 | ||
298c926c | 799 | ctx->child = cipher; |
ec9f2006 HX |
800 | crypto_aead_set_reqsize( |
801 | tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), | |
802 | crypto_aead_reqsize(cipher))); | |
298c926c AH |
803 | return 0; |
804 | } | |
805 | ||
f614e546 | 806 | static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) |
298c926c | 807 | { |
f614e546 | 808 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
809 | crypto_free_aead(ctx->child); |
810 | } | |
811 | ||
758ec5ac EB |
812 | static void cryptd_aead_free(struct aead_instance *inst) |
813 | { | |
814 | struct aead_instance_ctx *ctx = aead_instance_ctx(inst); | |
815 | ||
816 | crypto_drop_aead(&ctx->aead_spawn); | |
817 | kfree(inst); | |
818 | } | |
819 | ||
298c926c AH |
820 | static int cryptd_create_aead(struct crypto_template *tmpl, |
821 | struct rtattr **tb, | |
7bcb2c99 | 822 | struct crypto_attr_type *algt, |
298c926c AH |
823 | struct cryptd_queue *queue) |
824 | { | |
825 | struct aead_instance_ctx *ctx; | |
f614e546 HX |
826 | struct aead_instance *inst; |
827 | struct aead_alg *alg; | |
7bcb2c99 EB |
828 | u32 type; |
829 | u32 mask; | |
298c926c AH |
830 | int err; |
831 | ||
7bcb2c99 | 832 | cryptd_type_and_mask(algt, &type, &mask); |
466a7b9e | 833 | |
9b8c456e HX |
834 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
835 | if (!inst) | |
836 | return -ENOMEM; | |
298c926c | 837 | |
f614e546 | 838 | ctx = aead_instance_ctx(inst); |
298c926c AH |
839 | ctx->queue = queue; |
840 | ||
cd900f0c | 841 | err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), |
b8c0d74a | 842 | crypto_attr_alg_name(tb[1]), type, mask); |
298c926c | 843 | if (err) |
b8c0d74a | 844 | goto err_free_inst; |
298c926c | 845 | |
f614e546 HX |
846 | alg = crypto_spawn_aead_alg(&ctx->aead_spawn); |
847 | err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); | |
9b8c456e | 848 | if (err) |
b8c0d74a | 849 | goto err_free_inst; |
9b8c456e | 850 | |
7bcb2c99 EB |
851 | inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC | |
852 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
f614e546 | 853 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); |
298c926c | 854 | |
f614e546 HX |
855 | inst->alg.ivsize = crypto_aead_alg_ivsize(alg); |
856 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); | |
857 | ||
858 | inst->alg.init = cryptd_aead_init_tfm; | |
859 | inst->alg.exit = cryptd_aead_exit_tfm; | |
860 | inst->alg.setkey = cryptd_aead_setkey; | |
861 | inst->alg.setauthsize = cryptd_aead_setauthsize; | |
862 | inst->alg.encrypt = cryptd_aead_encrypt_enqueue; | |
863 | inst->alg.decrypt = cryptd_aead_decrypt_enqueue; | |
864 | ||
758ec5ac EB |
865 | inst->free = cryptd_aead_free; |
866 | ||
f614e546 | 867 | err = aead_register_instance(tmpl, inst); |
298c926c | 868 | if (err) { |
b8c0d74a EB |
869 | err_free_inst: |
870 | cryptd_aead_free(inst); | |
298c926c | 871 | } |
298c926c AH |
872 | return err; |
873 | } | |
874 | ||
254eff77 | 875 | static struct cryptd_queue queue; |
124b53d0 | 876 | |
9cd899a3 | 877 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
878 | { |
879 | struct crypto_attr_type *algt; | |
880 | ||
881 | algt = crypto_get_attr_type(tb); | |
882 | if (IS_ERR(algt)) | |
9cd899a3 | 883 | return PTR_ERR(algt); |
124b53d0 HX |
884 | |
885 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
c65058b7 | 886 | case CRYPTO_ALG_TYPE_SKCIPHER: |
7bcb2c99 | 887 | return cryptd_create_skcipher(tmpl, tb, algt, &queue); |
84ede58d | 888 | case CRYPTO_ALG_TYPE_HASH: |
7bcb2c99 | 889 | return cryptd_create_hash(tmpl, tb, algt, &queue); |
298c926c | 890 | case CRYPTO_ALG_TYPE_AEAD: |
7bcb2c99 | 891 | return cryptd_create_aead(tmpl, tb, algt, &queue); |
124b53d0 HX |
892 | } |
893 | ||
9cd899a3 | 894 | return -EINVAL; |
124b53d0 HX |
895 | } |
896 | ||
124b53d0 HX |
897 | static struct crypto_template cryptd_tmpl = { |
898 | .name = "cryptd", | |
9cd899a3 | 899 | .create = cryptd_create, |
124b53d0 HX |
900 | .module = THIS_MODULE, |
901 | }; | |
902 | ||
4e0958d1 HX |
903 | struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, |
904 | u32 type, u32 mask) | |
905 | { | |
906 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
907 | struct cryptd_skcipher_ctx *ctx; | |
908 | struct crypto_skcipher *tfm; | |
909 | ||
910 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
911 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
912 | return ERR_PTR(-EINVAL); | |
913 | ||
914 | tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); | |
915 | if (IS_ERR(tfm)) | |
916 | return ERR_CAST(tfm); | |
917 | ||
918 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
919 | crypto_free_skcipher(tfm); | |
920 | return ERR_PTR(-EINVAL); | |
921 | } | |
922 | ||
923 | ctx = crypto_skcipher_ctx(tfm); | |
43b970fa | 924 | refcount_set(&ctx->refcnt, 1); |
4e0958d1 HX |
925 | |
926 | return container_of(tfm, struct cryptd_skcipher, base); | |
927 | } | |
928 | EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); | |
929 | ||
930 | struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) | |
931 | { | |
932 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
933 | ||
3a58c231 | 934 | return ctx->child; |
4e0958d1 HX |
935 | } |
936 | EXPORT_SYMBOL_GPL(cryptd_skcipher_child); | |
937 | ||
938 | bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) | |
939 | { | |
940 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
941 | ||
43b970fa | 942 | return refcount_read(&ctx->refcnt) - 1; |
4e0958d1 HX |
943 | } |
944 | EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); | |
945 | ||
946 | void cryptd_free_skcipher(struct cryptd_skcipher *tfm) | |
947 | { | |
948 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
949 | ||
43b970fa | 950 | if (refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
951 | crypto_free_skcipher(&tfm->base); |
952 | } | |
953 | EXPORT_SYMBOL_GPL(cryptd_free_skcipher); | |
954 | ||
ace13663 HY |
955 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, |
956 | u32 type, u32 mask) | |
957 | { | |
958 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 959 | struct cryptd_hash_ctx *ctx; |
ace13663 HY |
960 | struct crypto_ahash *tfm; |
961 | ||
962 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
963 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
964 | return ERR_PTR(-EINVAL); | |
965 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
966 | if (IS_ERR(tfm)) | |
967 | return ERR_CAST(tfm); | |
968 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
969 | crypto_free_ahash(tfm); | |
970 | return ERR_PTR(-EINVAL); | |
971 | } | |
972 | ||
81760ea6 | 973 | ctx = crypto_ahash_ctx(tfm); |
43b970fa | 974 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 975 | |
ace13663 HY |
976 | return __cryptd_ahash_cast(tfm); |
977 | } | |
978 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
979 | ||
980 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
981 | { | |
982 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
983 | ||
984 | return ctx->child; | |
985 | } | |
986 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
987 | ||
0e1227d3 HY |
988 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) |
989 | { | |
990 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
991 | return &rctx->desc; | |
992 | } | |
993 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
994 | ||
81760ea6 HX |
995 | bool cryptd_ahash_queued(struct cryptd_ahash *tfm) |
996 | { | |
997 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
998 | ||
43b970fa | 999 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1000 | } |
1001 | EXPORT_SYMBOL_GPL(cryptd_ahash_queued); | |
1002 | ||
ace13663 HY |
1003 | void cryptd_free_ahash(struct cryptd_ahash *tfm) |
1004 | { | |
81760ea6 HX |
1005 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); |
1006 | ||
43b970fa | 1007 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1008 | crypto_free_ahash(&tfm->base); |
ace13663 HY |
1009 | } |
1010 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
1011 | ||
298c926c AH |
1012 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, |
1013 | u32 type, u32 mask) | |
1014 | { | |
1015 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 1016 | struct cryptd_aead_ctx *ctx; |
298c926c AH |
1017 | struct crypto_aead *tfm; |
1018 | ||
1019 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
1020 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
1021 | return ERR_PTR(-EINVAL); | |
1022 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
1023 | if (IS_ERR(tfm)) | |
1024 | return ERR_CAST(tfm); | |
1025 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1026 | crypto_free_aead(tfm); | |
1027 | return ERR_PTR(-EINVAL); | |
1028 | } | |
81760ea6 HX |
1029 | |
1030 | ctx = crypto_aead_ctx(tfm); | |
43b970fa | 1031 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 1032 | |
298c926c AH |
1033 | return __cryptd_aead_cast(tfm); |
1034 | } | |
1035 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
1036 | ||
1037 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
1038 | { | |
1039 | struct cryptd_aead_ctx *ctx; | |
1040 | ctx = crypto_aead_ctx(&tfm->base); | |
1041 | return ctx->child; | |
1042 | } | |
1043 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
1044 | ||
81760ea6 HX |
1045 | bool cryptd_aead_queued(struct cryptd_aead *tfm) |
1046 | { | |
1047 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); | |
1048 | ||
43b970fa | 1049 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1050 | } |
1051 | EXPORT_SYMBOL_GPL(cryptd_aead_queued); | |
1052 | ||
298c926c AH |
1053 | void cryptd_free_aead(struct cryptd_aead *tfm) |
1054 | { | |
81760ea6 HX |
1055 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); |
1056 | ||
43b970fa | 1057 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1058 | crypto_free_aead(&tfm->base); |
298c926c AH |
1059 | } |
1060 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
1061 | ||
124b53d0 HX |
1062 | static int __init cryptd_init(void) |
1063 | { | |
1064 | int err; | |
1065 | ||
3e56e168 EB |
1066 | cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, |
1067 | 1); | |
1068 | if (!cryptd_wq) | |
1069 | return -ENOMEM; | |
1070 | ||
c3a53605 | 1071 | err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); |
124b53d0 | 1072 | if (err) |
3e56e168 | 1073 | goto err_destroy_wq; |
124b53d0 HX |
1074 | |
1075 | err = crypto_register_template(&cryptd_tmpl); | |
1076 | if (err) | |
3e56e168 | 1077 | goto err_fini_queue; |
124b53d0 | 1078 | |
3e56e168 EB |
1079 | return 0; |
1080 | ||
1081 | err_fini_queue: | |
1082 | cryptd_fini_queue(&queue); | |
1083 | err_destroy_wq: | |
1084 | destroy_workqueue(cryptd_wq); | |
124b53d0 HX |
1085 | return err; |
1086 | } | |
1087 | ||
1088 | static void __exit cryptd_exit(void) | |
1089 | { | |
3e56e168 | 1090 | destroy_workqueue(cryptd_wq); |
254eff77 | 1091 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1092 | crypto_unregister_template(&cryptd_tmpl); |
1093 | } | |
1094 | ||
b2bac6ac | 1095 | subsys_initcall(cryptd_init); |
124b53d0 HX |
1096 | module_exit(cryptd_exit); |
1097 | ||
1098 | MODULE_LICENSE("GPL"); | |
1099 | MODULE_DESCRIPTION("Software async crypto daemon"); | |
4943ba16 | 1100 | MODULE_ALIAS_CRYPTO("cryptd"); |