ipv6: Allow non-gateway ECMP for IPv6
[linux-2.6-block.git] / crypto / crypto_engine.c
1 /*
2  * Handle async block request by crypto hardware engine.
3  *
4  * Copyright (C) 2016 Linaro, Inc.
5  *
6  * Author: Baolin Wang <baolin.wang@linaro.org>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU General Public License as published by the Free
10  * Software Foundation; either version 2 of the License, or (at your option)
11  * any later version.
12  *
13  */
14
15 #include <linux/err.h>
16 #include <linux/delay.h>
17 #include <crypto/engine.h>
18 #include <uapi/linux/sched/types.h>
19 #include "internal.h"
20
21 #define CRYPTO_ENGINE_MAX_QLEN 10
22
23 /**
24  * crypto_finalize_request - finalize one request if the request is done
25  * @engine: the hardware engine
26  * @req: the request need to be finalized
27  * @err: error number
28  */
29 static void crypto_finalize_request(struct crypto_engine *engine,
30                              struct crypto_async_request *req, int err)
31 {
32         unsigned long flags;
33         bool finalize_cur_req = false;
34         int ret;
35         struct crypto_engine_ctx *enginectx;
36
37         spin_lock_irqsave(&engine->queue_lock, flags);
38         if (engine->cur_req == req)
39                 finalize_cur_req = true;
40         spin_unlock_irqrestore(&engine->queue_lock, flags);
41
42         if (finalize_cur_req) {
43                 enginectx = crypto_tfm_ctx(req->tfm);
44                 if (engine->cur_req_prepared &&
45                     enginectx->op.unprepare_request) {
46                         ret = enginectx->op.unprepare_request(engine, req);
47                         if (ret)
48                                 dev_err(engine->dev, "failed to unprepare request\n");
49                 }
50                 spin_lock_irqsave(&engine->queue_lock, flags);
51                 engine->cur_req = NULL;
52                 engine->cur_req_prepared = false;
53                 spin_unlock_irqrestore(&engine->queue_lock, flags);
54         }
55
56         req->complete(req, err);
57
58         kthread_queue_work(engine->kworker, &engine->pump_requests);
59 }
60
61 /**
62  * crypto_pump_requests - dequeue one request from engine queue to process
63  * @engine: the hardware engine
64  * @in_kthread: true if we are in the context of the request pump thread
65  *
66  * This function checks if there is any request in the engine queue that
67  * needs processing and if so call out to the driver to initialize hardware
68  * and handle each request.
69  */
70 static void crypto_pump_requests(struct crypto_engine *engine,
71                                  bool in_kthread)
72 {
73         struct crypto_async_request *async_req, *backlog;
74         unsigned long flags;
75         bool was_busy = false;
76         int ret;
77         struct crypto_engine_ctx *enginectx;
78
79         spin_lock_irqsave(&engine->queue_lock, flags);
80
81         /* Make sure we are not already running a request */
82         if (engine->cur_req)
83                 goto out;
84
85         /* If another context is idling then defer */
86         if (engine->idling) {
87                 kthread_queue_work(engine->kworker, &engine->pump_requests);
88                 goto out;
89         }
90
91         /* Check if the engine queue is idle */
92         if (!crypto_queue_len(&engine->queue) || !engine->running) {
93                 if (!engine->busy)
94                         goto out;
95
96                 /* Only do teardown in the thread */
97                 if (!in_kthread) {
98                         kthread_queue_work(engine->kworker,
99                                            &engine->pump_requests);
100                         goto out;
101                 }
102
103                 engine->busy = false;
104                 engine->idling = true;
105                 spin_unlock_irqrestore(&engine->queue_lock, flags);
106
107                 if (engine->unprepare_crypt_hardware &&
108                     engine->unprepare_crypt_hardware(engine))
109                         dev_err(engine->dev, "failed to unprepare crypt hardware\n");
110
111                 spin_lock_irqsave(&engine->queue_lock, flags);
112                 engine->idling = false;
113                 goto out;
114         }
115
116         /* Get the fist request from the engine queue to handle */
117         backlog = crypto_get_backlog(&engine->queue);
118         async_req = crypto_dequeue_request(&engine->queue);
119         if (!async_req)
120                 goto out;
121
122         engine->cur_req = async_req;
123         if (backlog)
124                 backlog->complete(backlog, -EINPROGRESS);
125
126         if (engine->busy)
127                 was_busy = true;
128         else
129                 engine->busy = true;
130
131         spin_unlock_irqrestore(&engine->queue_lock, flags);
132
133         /* Until here we get the request need to be encrypted successfully */
134         if (!was_busy && engine->prepare_crypt_hardware) {
135                 ret = engine->prepare_crypt_hardware(engine);
136                 if (ret) {
137                         dev_err(engine->dev, "failed to prepare crypt hardware\n");
138                         goto req_err;
139                 }
140         }
141
142         enginectx = crypto_tfm_ctx(async_req->tfm);
143
144         if (enginectx->op.prepare_request) {
145                 ret = enginectx->op.prepare_request(engine, async_req);
146                 if (ret) {
147                         dev_err(engine->dev, "failed to prepare request: %d\n",
148                                 ret);
149                         goto req_err;
150                 }
151                 engine->cur_req_prepared = true;
152         }
153         if (!enginectx->op.do_one_request) {
154                 dev_err(engine->dev, "failed to do request\n");
155                 ret = -EINVAL;
156                 goto req_err;
157         }
158         ret = enginectx->op.do_one_request(engine, async_req);
159         if (ret) {
160                 dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret);
161                 goto req_err;
162         }
163         return;
164
165 req_err:
166         crypto_finalize_request(engine, async_req, ret);
167         return;
168
169 out:
170         spin_unlock_irqrestore(&engine->queue_lock, flags);
171 }
172
173 static void crypto_pump_work(struct kthread_work *work)
174 {
175         struct crypto_engine *engine =
176                 container_of(work, struct crypto_engine, pump_requests);
177
178         crypto_pump_requests(engine, true);
179 }
180
181 /**
182  * crypto_transfer_request - transfer the new request into the engine queue
183  * @engine: the hardware engine
184  * @req: the request need to be listed into the engine queue
185  */
186 static int crypto_transfer_request(struct crypto_engine *engine,
187                                    struct crypto_async_request *req,
188                                    bool need_pump)
189 {
190         unsigned long flags;
191         int ret;
192
193         spin_lock_irqsave(&engine->queue_lock, flags);
194
195         if (!engine->running) {
196                 spin_unlock_irqrestore(&engine->queue_lock, flags);
197                 return -ESHUTDOWN;
198         }
199
200         ret = crypto_enqueue_request(&engine->queue, req);
201
202         if (!engine->busy && need_pump)
203                 kthread_queue_work(engine->kworker, &engine->pump_requests);
204
205         spin_unlock_irqrestore(&engine->queue_lock, flags);
206         return ret;
207 }
208
209 /**
210  * crypto_transfer_request_to_engine - transfer one request to list
211  * into the engine queue
212  * @engine: the hardware engine
213  * @req: the request need to be listed into the engine queue
214  */
215 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
216                                              struct crypto_async_request *req)
217 {
218         return crypto_transfer_request(engine, req, true);
219 }
220
221 /**
222  * crypto_transfer_ablkcipher_request_to_engine - transfer one ablkcipher_request
223  * to list into the engine queue
224  * @engine: the hardware engine
225  * @req: the request need to be listed into the engine queue
226  * TODO: Remove this function when skcipher conversion is finished
227  */
228 int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine,
229                                                  struct ablkcipher_request *req)
230 {
231         return crypto_transfer_request_to_engine(engine, &req->base);
232 }
233 EXPORT_SYMBOL_GPL(crypto_transfer_ablkcipher_request_to_engine);
234
235 /**
236  * crypto_transfer_aead_request_to_engine - transfer one aead_request
237  * to list into the engine queue
238  * @engine: the hardware engine
239  * @req: the request need to be listed into the engine queue
240  */
241 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
242                                            struct aead_request *req)
243 {
244         return crypto_transfer_request_to_engine(engine, &req->base);
245 }
246 EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
247
248 /**
249  * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
250  * to list into the engine queue
251  * @engine: the hardware engine
252  * @req: the request need to be listed into the engine queue
253  */
254 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
255                                                struct akcipher_request *req)
256 {
257         return crypto_transfer_request_to_engine(engine, &req->base);
258 }
259 EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
260
261 /**
262  * crypto_transfer_hash_request_to_engine - transfer one ahash_request
263  * to list into the engine queue
264  * @engine: the hardware engine
265  * @req: the request need to be listed into the engine queue
266  */
267 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
268                                            struct ahash_request *req)
269 {
270         return crypto_transfer_request_to_engine(engine, &req->base);
271 }
272 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
273
274 /**
275  * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
276  * to list into the engine queue
277  * @engine: the hardware engine
278  * @req: the request need to be listed into the engine queue
279  */
280 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
281                                                struct skcipher_request *req)
282 {
283         return crypto_transfer_request_to_engine(engine, &req->base);
284 }
285 EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
286
287 /**
288  * crypto_finalize_ablkcipher_request - finalize one ablkcipher_request if
289  * the request is done
290  * @engine: the hardware engine
291  * @req: the request need to be finalized
292  * @err: error number
293  * TODO: Remove this function when skcipher conversion is finished
294  */
295 void crypto_finalize_ablkcipher_request(struct crypto_engine *engine,
296                                         struct ablkcipher_request *req, int err)
297 {
298         return crypto_finalize_request(engine, &req->base, err);
299 }
300 EXPORT_SYMBOL_GPL(crypto_finalize_ablkcipher_request);
301
302 /**
303  * crypto_finalize_aead_request - finalize one aead_request if
304  * the request is done
305  * @engine: the hardware engine
306  * @req: the request need to be finalized
307  * @err: error number
308  */
309 void crypto_finalize_aead_request(struct crypto_engine *engine,
310                                   struct aead_request *req, int err)
311 {
312         return crypto_finalize_request(engine, &req->base, err);
313 }
314 EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
315
316 /**
317  * crypto_finalize_akcipher_request - finalize one akcipher_request if
318  * the request is done
319  * @engine: the hardware engine
320  * @req: the request need to be finalized
321  * @err: error number
322  */
323 void crypto_finalize_akcipher_request(struct crypto_engine *engine,
324                                       struct akcipher_request *req, int err)
325 {
326         return crypto_finalize_request(engine, &req->base, err);
327 }
328 EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
329
330 /**
331  * crypto_finalize_hash_request - finalize one ahash_request if
332  * the request is done
333  * @engine: the hardware engine
334  * @req: the request need to be finalized
335  * @err: error number
336  */
337 void crypto_finalize_hash_request(struct crypto_engine *engine,
338                                   struct ahash_request *req, int err)
339 {
340         return crypto_finalize_request(engine, &req->base, err);
341 }
342 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
343
344 /**
345  * crypto_finalize_skcipher_request - finalize one skcipher_request if
346  * the request is done
347  * @engine: the hardware engine
348  * @req: the request need to be finalized
349  * @err: error number
350  */
351 void crypto_finalize_skcipher_request(struct crypto_engine *engine,
352                                       struct skcipher_request *req, int err)
353 {
354         return crypto_finalize_request(engine, &req->base, err);
355 }
356 EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
357
358 /**
359  * crypto_engine_start - start the hardware engine
360  * @engine: the hardware engine need to be started
361  *
362  * Return 0 on success, else on fail.
363  */
364 int crypto_engine_start(struct crypto_engine *engine)
365 {
366         unsigned long flags;
367
368         spin_lock_irqsave(&engine->queue_lock, flags);
369
370         if (engine->running || engine->busy) {
371                 spin_unlock_irqrestore(&engine->queue_lock, flags);
372                 return -EBUSY;
373         }
374
375         engine->running = true;
376         spin_unlock_irqrestore(&engine->queue_lock, flags);
377
378         kthread_queue_work(engine->kworker, &engine->pump_requests);
379
380         return 0;
381 }
382 EXPORT_SYMBOL_GPL(crypto_engine_start);
383
384 /**
385  * crypto_engine_stop - stop the hardware engine
386  * @engine: the hardware engine need to be stopped
387  *
388  * Return 0 on success, else on fail.
389  */
390 int crypto_engine_stop(struct crypto_engine *engine)
391 {
392         unsigned long flags;
393         unsigned int limit = 500;
394         int ret = 0;
395
396         spin_lock_irqsave(&engine->queue_lock, flags);
397
398         /*
399          * If the engine queue is not empty or the engine is on busy state,
400          * we need to wait for a while to pump the requests of engine queue.
401          */
402         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
403                 spin_unlock_irqrestore(&engine->queue_lock, flags);
404                 msleep(20);
405                 spin_lock_irqsave(&engine->queue_lock, flags);
406         }
407
408         if (crypto_queue_len(&engine->queue) || engine->busy)
409                 ret = -EBUSY;
410         else
411                 engine->running = false;
412
413         spin_unlock_irqrestore(&engine->queue_lock, flags);
414
415         if (ret)
416                 dev_warn(engine->dev, "could not stop engine\n");
417
418         return ret;
419 }
420 EXPORT_SYMBOL_GPL(crypto_engine_stop);
421
422 /**
423  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
424  * initialize it.
425  * @dev: the device attached with one hardware engine
426  * @rt: whether this queue is set to run as a realtime task
427  *
428  * This must be called from context that can sleep.
429  * Return: the crypto engine structure on success, else NULL.
430  */
431 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
432 {
433         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
434         struct crypto_engine *engine;
435
436         if (!dev)
437                 return NULL;
438
439         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
440         if (!engine)
441                 return NULL;
442
443         engine->dev = dev;
444         engine->rt = rt;
445         engine->running = false;
446         engine->busy = false;
447         engine->idling = false;
448         engine->cur_req_prepared = false;
449         engine->priv_data = dev;
450         snprintf(engine->name, sizeof(engine->name),
451                  "%s-engine", dev_name(dev));
452
453         crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
454         spin_lock_init(&engine->queue_lock);
455
456         engine->kworker = kthread_create_worker(0, "%s", engine->name);
457         if (IS_ERR(engine->kworker)) {
458                 dev_err(dev, "failed to create crypto request pump task\n");
459                 return NULL;
460         }
461         kthread_init_work(&engine->pump_requests, crypto_pump_work);
462
463         if (engine->rt) {
464                 dev_info(dev, "will run requests pump with realtime priority\n");
465                 sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
466         }
467
468         return engine;
469 }
470 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
471
472 /**
473  * crypto_engine_exit - free the resources of hardware engine when exit
474  * @engine: the hardware engine need to be freed
475  *
476  * Return 0 for success.
477  */
478 int crypto_engine_exit(struct crypto_engine *engine)
479 {
480         int ret;
481
482         ret = crypto_engine_stop(engine);
483         if (ret)
484                 return ret;
485
486         kthread_destroy_worker(engine->kworker);
487
488         return 0;
489 }
490 EXPORT_SYMBOL_GPL(crypto_engine_exit);
491
492 MODULE_LICENSE("GPL");
493 MODULE_DESCRIPTION("Crypto hardware engine framework");