projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-v5.10' of git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux...
[linux-block.git] / drivers / crypto / img-hash.c
CommitLineData
d2912cb1 1// SPDX-License-Identifier: GPL-2.0-only
d358f1ab
JH		 2/*
3 * Copyright (c) 2014 Imagination Technologies
4 * Authors: Will Thomas, James Hartley
5 *
d358f1ab
JH		 6 * Interface structure taken from omap-sham driver
7 */
8
9#include <linux/clk.h>
0c3dc787 10#include <linux/dma-mapping.h>
d358f1ab
JH		 11#include <linux/dmaengine.h>
12#include <linux/interrupt.h>
13#include <linux/io.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/of_device.h>
17#include <linux/platform_device.h>
18#include <linux/scatterlist.h>
19
20#include <crypto/internal/hash.h>
21#include <crypto/md5.h>
22#include <crypto/sha.h>
23
24#define CR_RESET 0
25#define CR_RESET_SET 1
26#define CR_RESET_UNSET 0
27
28#define CR_MESSAGE_LENGTH_H 0x4
29#define CR_MESSAGE_LENGTH_L 0x8
30
31#define CR_CONTROL 0xc
32#define CR_CONTROL_BYTE_ORDER_3210 0
33#define CR_CONTROL_BYTE_ORDER_0123 1
34#define CR_CONTROL_BYTE_ORDER_2310 2
35#define CR_CONTROL_BYTE_ORDER_1032 3
36#define CR_CONTROL_BYTE_ORDER_SHIFT 8
37#define CR_CONTROL_ALGO_MD5 0
38#define CR_CONTROL_ALGO_SHA1 1
39#define CR_CONTROL_ALGO_SHA224 2
40#define CR_CONTROL_ALGO_SHA256 3
41
42#define CR_INTSTAT 0x10
43#define CR_INTENAB 0x14
44#define CR_INTCLEAR 0x18
45#define CR_INT_RESULTS_AVAILABLE BIT(0)
46#define CR_INT_NEW_RESULTS_SET BIT(1)
47#define CR_INT_RESULT_READ_ERR BIT(2)
48#define CR_INT_MESSAGE_WRITE_ERROR BIT(3)
49#define CR_INT_STATUS BIT(8)
50
51#define CR_RESULT_QUEUE 0x1c
52#define CR_RSD0 0x40
53#define CR_CORE_REV 0x50
54#define CR_CORE_DES1 0x60
55#define CR_CORE_DES2 0x70
56
57#define DRIVER_FLAGS_BUSY BIT(0)
58#define DRIVER_FLAGS_FINAL BIT(1)
59#define DRIVER_FLAGS_DMA_ACTIVE BIT(2)
60#define DRIVER_FLAGS_OUTPUT_READY BIT(3)
61#define DRIVER_FLAGS_INIT BIT(4)
62#define DRIVER_FLAGS_CPU BIT(5)
63#define DRIVER_FLAGS_DMA_READY BIT(6)
64#define DRIVER_FLAGS_ERROR BIT(7)
65#define DRIVER_FLAGS_SG BIT(8)
66#define DRIVER_FLAGS_SHA1 BIT(18)
67#define DRIVER_FLAGS_SHA224 BIT(19)
68#define DRIVER_FLAGS_SHA256 BIT(20)
69#define DRIVER_FLAGS_MD5 BIT(21)
70
71#define IMG_HASH_QUEUE_LENGTH 20
302a1bee 72#define IMG_HASH_DMA_BURST 4
d358f1ab
JH		 73#define IMG_HASH_DMA_THRESHOLD 64
74
75#ifdef __LITTLE_ENDIAN
76#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210
77#else
78#define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123
79#endif
80
81struct img_hash_dev;
82
83struct img_hash_request_ctx {
84 struct img_hash_dev *hdev;
85 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
86 unsigned long flags;
87 size_t digsize;
88
89 dma_addr_t dma_addr;
90 size_t dma_ct;
91
92 /* sg root */
93 struct scatterlist *sgfirst;
94 /* walk state */
95 struct scatterlist *sg;
96 size_t nents;
97 size_t offset;
98 unsigned int total;
99 size_t sent;
100
101 unsigned long op;
102
103 size_t bufcnt;
d358f1ab 104 struct ahash_request fallback_req;
dd4f677b
WT		 105
106 /* Zero length buffer must remain last member of struct */
e44362ab 107 u8 buffer[] __aligned(sizeof(u32));
d358f1ab
JH		 108};
109
110struct img_hash_ctx {
111 struct img_hash_dev *hdev;
112 unsigned long flags;
113 struct crypto_ahash *fallback;
114};
115
116struct img_hash_dev {
117 struct list_head list;
118 struct device *dev;
119 struct clk *hash_clk;
120 struct clk *sys_clk;
121 void __iomem *io_base;
122
123 phys_addr_t bus_addr;
124 void __iomem *cpu_addr;
125
126 spinlock_t lock;
127 int err;
128 struct tasklet_struct done_task;
129 struct tasklet_struct dma_task;
130
131 unsigned long flags;
132 struct crypto_queue queue;
133 struct ahash_request *req;
134
135 struct dma_chan *dma_lch;
136};
137
138struct img_hash_drv {
139 struct list_head dev_list;
140 spinlock_t lock;
141};
142
143static struct img_hash_drv img_hash = {
144 .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
145 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
146};
147
148static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
149{
150 return readl_relaxed(hdev->io_base + offset);
151}
152
153static inline void img_hash_write(struct img_hash_dev *hdev,
154 u32 offset, u32 value)
155{
156 writel_relaxed(value, hdev->io_base + offset);
157}
158
159static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
160{
161 return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
162}
163
164static void img_hash_start(struct img_hash_dev *hdev, bool dma)
165{
166 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
167 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
168
169 if (ctx->flags & DRIVER_FLAGS_MD5)
170 cr |= CR_CONTROL_ALGO_MD5;
171 else if (ctx->flags & DRIVER_FLAGS_SHA1)
172 cr |= CR_CONTROL_ALGO_SHA1;
173 else if (ctx->flags & DRIVER_FLAGS_SHA224)
174 cr |= CR_CONTROL_ALGO_SHA224;
175 else if (ctx->flags & DRIVER_FLAGS_SHA256)
176 cr |= CR_CONTROL_ALGO_SHA256;
177 dev_dbg(hdev->dev, "Starting hash process\n");
178 img_hash_write(hdev, CR_CONTROL, cr);
179
180 /*
181 * The hardware block requires two cycles between writing the control
182 * register and writing the first word of data in non DMA mode, to
183 * ensure the first data write is not grouped in burst with the control
184 * register write a read is issued to 'flush' the bus.
185 */
186 if (!dma)
187 img_hash_read(hdev, CR_CONTROL);
188}
189
190static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
191 size_t length, int final)
192{
193 u32 count, len32;
194 const u32 *buffer = (const u32 *)buf;
195
900831a4 196 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length);
d358f1ab
JH		 197
198 if (final)
199 hdev->flags |= DRIVER_FLAGS_FINAL;
200
201 len32 = DIV_ROUND_UP(length, sizeof(u32));
202
203 for (count = 0; count < len32; count++)
204 writel_relaxed(buffer[count], hdev->cpu_addr);
205
206 return -EINPROGRESS;
207}
208
209static void img_hash_dma_callback(void *data)
210{
211 struct img_hash_dev *hdev = (struct img_hash_dev *)data;
212 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
213
214 if (ctx->bufcnt) {
215 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
216 ctx->bufcnt = 0;
217 }
218 if (ctx->sg)
219 tasklet_schedule(&hdev->dma_task);
220}
221
222static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
223{
224 struct dma_async_tx_descriptor *desc;
225 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
226
3bfb2e6b 227 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
d358f1ab
JH		 228 if (ctx->dma_ct == 0) {
229 dev_err(hdev->dev, "Invalid DMA sg\n");
230 hdev->err = -EINVAL;
231 return -EINVAL;
232 }
233
234 desc = dmaengine_prep_slave_sg(hdev->dma_lch,
235 sg,
236 ctx->dma_ct,
237 DMA_MEM_TO_DEV,
238 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
239 if (!desc) {
240 dev_err(hdev->dev, "Null DMA descriptor\n");
241 hdev->err = -EINVAL;
3bfb2e6b 242 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
d358f1ab
JH		 243 return -EINVAL;
244 }
245 desc->callback = img_hash_dma_callback;
246 desc->callback_param = hdev;
247 dmaengine_submit(desc);
248 dma_async_issue_pending(hdev->dma_lch);
249
250 return 0;
251}
252
253static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
254{
255 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
256
257 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
258 ctx->buffer, hdev->req->nbytes);
259
260 ctx->total = hdev->req->nbytes;
261 ctx->bufcnt = 0;
262
263 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
264
265 img_hash_start(hdev, false);
266
267 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
268}
269
270static int img_hash_finish(struct ahash_request *req)
271{
272 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
273
274 if (!req->result)
275 return -EINVAL;
276
277 memcpy(req->result, ctx->digest, ctx->digsize);
278
279 return 0;
280}
281
282static void img_hash_copy_hash(struct ahash_request *req)
283{
284 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
285 u32 *hash = (u32 *)ctx->digest;
286 int i;
287
288 for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
289 hash[i] = img_hash_read_result_queue(ctx->hdev);
290}
291
292static void img_hash_finish_req(struct ahash_request *req, int err)
293{
294 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
295 struct img_hash_dev *hdev = ctx->hdev;
296
297 if (!err) {
298 img_hash_copy_hash(req);
299 if (DRIVER_FLAGS_FINAL & hdev->flags)
300 err = img_hash_finish(req);
301 } else {
302 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
303 ctx->flags |= DRIVER_FLAGS_ERROR;
304 }
305
306 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
307 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
308
309 if (req->base.complete)
310 req->base.complete(&req->base, err);
311}
312
313static int img_hash_write_via_dma(struct img_hash_dev *hdev)
314{
315 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
316
317 img_hash_start(hdev, true);
318
319 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
320
321 if (!ctx->total)
322 hdev->flags |= DRIVER_FLAGS_FINAL;
323
324 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
325
326 tasklet_schedule(&hdev->dma_task);
327
328 return -EINPROGRESS;
329}
330
331static int img_hash_dma_init(struct img_hash_dev *hdev)
332{
333 struct dma_slave_config dma_conf;
84d840e1 334 int err;
d358f1ab 335
8004865c
PU		 336 hdev->dma_lch = dma_request_chan(hdev->dev, "tx");
337 if (IS_ERR(hdev->dma_lch)) {
327cbbab 338 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
8004865c 339 return PTR_ERR(hdev->dma_lch);
d358f1ab
JH		 340 }
341 dma_conf.direction = DMA_MEM_TO_DEV;
342 dma_conf.dst_addr = hdev->bus_addr;
343 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
302a1bee 344 dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;
d358f1ab
JH		 345 dma_conf.device_fc = false;
346
347 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
348 if (err) {
349 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
350 dma_release_channel(hdev->dma_lch);
351 return err;
352 }
353
354 return 0;
355}
356
357static void img_hash_dma_task(unsigned long d)
358{
359 struct img_hash_dev *hdev = (struct img_hash_dev *)d;
360 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
361 u8 *addr;
362 size_t nbytes, bleft, wsend, len, tbc;
363 struct scatterlist tsg;
364
10badea2 365 if (!hdev->req || !ctx->sg)
d358f1ab
JH		 366 return;
367
368 addr = sg_virt(ctx->sg);
369 nbytes = ctx->sg->length - ctx->offset;
370
371 /*
372 * The hash accelerator does not support a data valid mask. This means
373 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
374 * padding bytes in the last word written by that dma would erroneously
375 * be included in the hash. To avoid this we round down the transfer,
376 * and add the excess to the start of the next dma. It does not matter
377 * that the final dma may not be a multiple of 4 bytes as the hashing
378 * block is programmed to accept the correct number of bytes.
379 */
380
381 bleft = nbytes % 4;
382 wsend = (nbytes / 4);
383
384 if (wsend) {
385 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
386 if (img_hash_xmit_dma(hdev, &tsg)) {
387 dev_err(hdev->dev, "DMA failed, falling back to CPU");
388 ctx->flags |= DRIVER_FLAGS_CPU;
389 hdev->err = 0;
390 img_hash_xmit_cpu(hdev, addr + ctx->offset,
391 wsend * 4, 0);
392 ctx->sent += wsend * 4;
393 wsend = 0;
394 } else {
395 ctx->sent += wsend * 4;
396 }
397 }
398
399 if (bleft) {
400 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
401 ctx->buffer, bleft, ctx->sent);
402 tbc = 0;
403 ctx->sg = sg_next(ctx->sg);
404 while (ctx->sg && (ctx->bufcnt < 4)) {
405 len = ctx->sg->length;
406 if (likely(len > (4 - ctx->bufcnt)))
407 len = 4 - ctx->bufcnt;
408 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
409 ctx->buffer + ctx->bufcnt, len,
410 ctx->sent + ctx->bufcnt);
411 ctx->bufcnt += tbc;
412 if (tbc >= ctx->sg->length) {
413 ctx->sg = sg_next(ctx->sg);
414 tbc = 0;
415 }
416 }
417
418 ctx->sent += ctx->bufcnt;
419 ctx->offset = tbc;
420
421 if (!wsend)
422 img_hash_dma_callback(hdev);
423 } else {
424 ctx->offset = 0;
425 ctx->sg = sg_next(ctx->sg);
426 }
427}
428
429static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
430{
431 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
432
433 if (ctx->flags & DRIVER_FLAGS_SG)
434 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
435
436 return 0;
437}
438
439static int img_hash_process_data(struct img_hash_dev *hdev)
440{
441 struct ahash_request *req = hdev->req;
442 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
443 int err = 0;
444
445 ctx->bufcnt = 0;
446
447 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
448 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
449 req->nbytes);
450 err = img_hash_write_via_dma(hdev);
451 } else {
452 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
453 req->nbytes);
454 err = img_hash_write_via_cpu(hdev);
455 }
456 return err;
457}
458
459static int img_hash_hw_init(struct img_hash_dev *hdev)
460{
461 unsigned long long nbits;
462 u32 u, l;
d358f1ab
JH		 463
464 img_hash_write(hdev, CR_RESET, CR_RESET_SET);
465 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
466 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
467
a83034f5 468 nbits = (u64)hdev->req->nbytes << 3;
d358f1ab
JH		 469 u = nbits >> 32;
470 l = nbits;
471 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
472 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
473
474 if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
475 hdev->flags |= DRIVER_FLAGS_INIT;
476 hdev->err = 0;
477 }
478 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
479 return 0;
480}
481
482static int img_hash_init(struct ahash_request *req)
483{
484 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
485 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
486 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
487
488 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
489 rctx->fallback_req.base.flags = req->base.flags
490 & CRYPTO_TFM_REQ_MAY_SLEEP;
491
492 return crypto_ahash_init(&rctx->fallback_req);
493}
494
495static int img_hash_handle_queue(struct img_hash_dev *hdev,
496 struct ahash_request *req)
497{
498 struct crypto_async_request *async_req, *backlog;
499 struct img_hash_request_ctx *ctx;
500 unsigned long flags;
501 int err = 0, res = 0;
502
503 spin_lock_irqsave(&hdev->lock, flags);
504
505 if (req)
506 res = ahash_enqueue_request(&hdev->queue, req);
507
508 if (DRIVER_FLAGS_BUSY & hdev->flags) {
509 spin_unlock_irqrestore(&hdev->lock, flags);
510 return res;
511 }
512
513 backlog = crypto_get_backlog(&hdev->queue);
514 async_req = crypto_dequeue_request(&hdev->queue);
515 if (async_req)
516 hdev->flags |= DRIVER_FLAGS_BUSY;
517
518 spin_unlock_irqrestore(&hdev->lock, flags);
519
520 if (!async_req)
521 return res;
522
523 if (backlog)
524 backlog->complete(backlog, -EINPROGRESS);
525
526 req = ahash_request_cast(async_req);
527 hdev->req = req;
528
529 ctx = ahash_request_ctx(req);
530
531 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
532 ctx->op, req->nbytes);
533
534 err = img_hash_hw_init(hdev);
535
536 if (!err)
537 err = img_hash_process_data(hdev);
538
539 if (err != -EINPROGRESS) {
540 /* done_task will not finish so do it here */
541 img_hash_finish_req(req, err);
542 }
543 return res;
544}
545
546static int img_hash_update(struct ahash_request *req)
547{
548 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
549 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
550 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
551
552 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
553 rctx->fallback_req.base.flags = req->base.flags
554 & CRYPTO_TFM_REQ_MAY_SLEEP;
555 rctx->fallback_req.nbytes = req->nbytes;
556 rctx->fallback_req.src = req->src;
557
558 return crypto_ahash_update(&rctx->fallback_req);
559}
560
561static int img_hash_final(struct ahash_request *req)
562{
563 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
564 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
565 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
566
567 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
568 rctx->fallback_req.base.flags = req->base.flags
569 & CRYPTO_TFM_REQ_MAY_SLEEP;
570 rctx->fallback_req.result = req->result;
571
572 return crypto_ahash_final(&rctx->fallback_req);
573}
574
575static int img_hash_finup(struct ahash_request *req)
576{
577 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
578 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
579 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
580
581 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
582 rctx->fallback_req.base.flags = req->base.flags
583 & CRYPTO_TFM_REQ_MAY_SLEEP;
584 rctx->fallback_req.nbytes = req->nbytes;
585 rctx->fallback_req.src = req->src;
586 rctx->fallback_req.result = req->result;
587
588 return crypto_ahash_finup(&rctx->fallback_req);
589}
590
436e3bb5
JH		 591static int img_hash_import(struct ahash_request *req, const void *in)
592{
593 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
594 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
595 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
596
597 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
598 rctx->fallback_req.base.flags = req->base.flags
599 & CRYPTO_TFM_REQ_MAY_SLEEP;
600
601 return crypto_ahash_import(&rctx->fallback_req, in);
602}
603
604static int img_hash_export(struct ahash_request *req, void *out)
605{
606 struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
607 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
608 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
609
610 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
611 rctx->fallback_req.base.flags = req->base.flags
612 & CRYPTO_TFM_REQ_MAY_SLEEP;
613
614 return crypto_ahash_export(&rctx->fallback_req, out);
615}
616
d358f1ab
JH		 617static int img_hash_digest(struct ahash_request *req)
618{
619 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
620 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
621 struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
622 struct img_hash_dev *hdev = NULL;
623 struct img_hash_dev *tmp;
624 int err;
625
626 spin_lock(&img_hash.lock);
627 if (!tctx->hdev) {
628 list_for_each_entry(tmp, &img_hash.dev_list, list) {
629 hdev = tmp;
630 break;
631 }
632 tctx->hdev = hdev;
633
634 } else {
635 hdev = tctx->hdev;
636 }
637
638 spin_unlock(&img_hash.lock);
639 ctx->hdev = hdev;
640 ctx->flags = 0;
641 ctx->digsize = crypto_ahash_digestsize(tfm);
642
643 switch (ctx->digsize) {
644 case SHA1_DIGEST_SIZE:
645 ctx->flags |= DRIVER_FLAGS_SHA1;
646 break;
647 case SHA256_DIGEST_SIZE:
648 ctx->flags |= DRIVER_FLAGS_SHA256;
649 break;
650 case SHA224_DIGEST_SIZE:
651 ctx->flags |= DRIVER_FLAGS_SHA224;
652 break;
653 case MD5_DIGEST_SIZE:
654 ctx->flags |= DRIVER_FLAGS_MD5;
655 break;
656 default:
657 return -EINVAL;
658 }
659
660 ctx->bufcnt = 0;
661 ctx->offset = 0;
662 ctx->sent = 0;
663 ctx->total = req->nbytes;
664 ctx->sg = req->src;
665 ctx->sgfirst = req->src;
666 ctx->nents = sg_nents(ctx->sg);
667
668 err = img_hash_handle_queue(tctx->hdev, req);
669
670 return err;
671}
672
436e3bb5 673static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)
d358f1ab
JH		 674{
675 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
d358f1ab
JH		 676 int err = -ENOMEM;
677
678 ctx->fallback = crypto_alloc_ahash(alg_name, 0,
679 CRYPTO_ALG_NEED_FALLBACK);
680 if (IS_ERR(ctx->fallback)) {
681 pr_err("img_hash: Could not load fallback driver.\n");
682 err = PTR_ERR(ctx->fallback);
683 goto err;
684 }
685 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
686 sizeof(struct img_hash_request_ctx) +
5e38d200 687 crypto_ahash_reqsize(ctx->fallback) +
d358f1ab
JH		 688 IMG_HASH_DMA_THRESHOLD);
689
690 return 0;
691
692err:
693 return err;
694}
695
436e3bb5
JH		 696static int img_hash_cra_md5_init(struct crypto_tfm *tfm)
697{
698 return img_hash_cra_init(tfm, "md5-generic");
699}
700
701static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)
702{
703 return img_hash_cra_init(tfm, "sha1-generic");
704}
705
706static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)
707{
708 return img_hash_cra_init(tfm, "sha224-generic");
709}
710
711static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)
712{
713 return img_hash_cra_init(tfm, "sha256-generic");
714}
715
d358f1ab
JH		 716static void img_hash_cra_exit(struct crypto_tfm *tfm)
717{
718 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
719
720 crypto_free_ahash(tctx->fallback);
721}
722
723static irqreturn_t img_irq_handler(int irq, void *dev_id)
724{
725 struct img_hash_dev *hdev = dev_id;
726 u32 reg;
727
728 reg = img_hash_read(hdev, CR_INTSTAT);
729 img_hash_write(hdev, CR_INTCLEAR, reg);
730
731 if (reg & CR_INT_NEW_RESULTS_SET) {
732 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
733 if (DRIVER_FLAGS_BUSY & hdev->flags) {
734 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
735 if (!(DRIVER_FLAGS_CPU & hdev->flags))
736 hdev->flags |= DRIVER_FLAGS_DMA_READY;
737 tasklet_schedule(&hdev->done_task);
738 } else {
739 dev_warn(hdev->dev,
740 "HASH interrupt when no active requests.\n");
741 }
742 } else if (reg & CR_INT_RESULTS_AVAILABLE) {
743 dev_warn(hdev->dev,
744 "IRQ triggered before the hash had completed\n");
745 } else if (reg & CR_INT_RESULT_READ_ERR) {
746 dev_warn(hdev->dev,
747 "Attempt to read from an empty result queue\n");
748 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
749 dev_warn(hdev->dev,
750 "Data written before the hardware was configured\n");
751 }
752 return IRQ_HANDLED;
753}
754
755static struct ahash_alg img_algs[] = {
756 {
757 .init = img_hash_init,
758 .update = img_hash_update,
759 .final = img_hash_final,
760 .finup = img_hash_finup,
436e3bb5
JH		 761 .export = img_hash_export,
762 .import = img_hash_import,
d358f1ab
JH		 763 .digest = img_hash_digest,
764 .halg = {
765 .digestsize = MD5_DIGEST_SIZE,
436e3bb5 766 .statesize = sizeof(struct md5_state),
d358f1ab
JH		 767 .base = {
768 .cra_name = "md5",
769 .cra_driver_name = "img-md5",
770 .cra_priority = 300,
771 .cra_flags =
772 CRYPTO_ALG_ASYNC |
773 CRYPTO_ALG_NEED_FALLBACK,
774 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
775 .cra_ctxsize = sizeof(struct img_hash_ctx),
436e3bb5 776 .cra_init = img_hash_cra_md5_init,
d358f1ab
JH		 777 .cra_exit = img_hash_cra_exit,
778 .cra_module = THIS_MODULE,
779 }
780 }
781 },
782 {
783 .init = img_hash_init,
784 .update = img_hash_update,
785 .final = img_hash_final,
786 .finup = img_hash_finup,
436e3bb5
JH		 787 .export = img_hash_export,
788 .import = img_hash_import,
d358f1ab
JH		 789 .digest = img_hash_digest,
790 .halg = {
791 .digestsize = SHA1_DIGEST_SIZE,
436e3bb5 792 .statesize = sizeof(struct sha1_state),
d358f1ab
JH		 793 .base = {
794 .cra_name = "sha1",
795 .cra_driver_name = "img-sha1",
796 .cra_priority = 300,
797 .cra_flags =
798 CRYPTO_ALG_ASYNC |
799 CRYPTO_ALG_NEED_FALLBACK,
800 .cra_blocksize = SHA1_BLOCK_SIZE,
801 .cra_ctxsize = sizeof(struct img_hash_ctx),
436e3bb5 802 .cra_init = img_hash_cra_sha1_init,
d358f1ab
JH		 803 .cra_exit = img_hash_cra_exit,
804 .cra_module = THIS_MODULE,
805 }
806 }
807 },
808 {
809 .init = img_hash_init,
810 .update = img_hash_update,
811 .final = img_hash_final,
812 .finup = img_hash_finup,
436e3bb5
JH		 813 .export = img_hash_export,
814 .import = img_hash_import,
d358f1ab
JH		 815 .digest = img_hash_digest,
816 .halg = {
817 .digestsize = SHA224_DIGEST_SIZE,
436e3bb5 818 .statesize = sizeof(struct sha256_state),
d358f1ab
JH		 819 .base = {
820 .cra_name = "sha224",
821 .cra_driver_name = "img-sha224",
822 .cra_priority = 300,
823 .cra_flags =
824 CRYPTO_ALG_ASYNC |
825 CRYPTO_ALG_NEED_FALLBACK,
826 .cra_blocksize = SHA224_BLOCK_SIZE,
827 .cra_ctxsize = sizeof(struct img_hash_ctx),
436e3bb5 828 .cra_init = img_hash_cra_sha224_init,
d358f1ab
JH		 829 .cra_exit = img_hash_cra_exit,
830 .cra_module = THIS_MODULE,
831 }
832 }
833 },
834 {
835 .init = img_hash_init,
836 .update = img_hash_update,
837 .final = img_hash_final,
838 .finup = img_hash_finup,
436e3bb5
JH		 839 .export = img_hash_export,
840 .import = img_hash_import,
d358f1ab
JH		 841 .digest = img_hash_digest,
842 .halg = {
843 .digestsize = SHA256_DIGEST_SIZE,
436e3bb5 844 .statesize = sizeof(struct sha256_state),
d358f1ab
JH		 845 .base = {
846 .cra_name = "sha256",
847 .cra_driver_name = "img-sha256",
848 .cra_priority = 300,
849 .cra_flags =
850 CRYPTO_ALG_ASYNC |
851 CRYPTO_ALG_NEED_FALLBACK,
852 .cra_blocksize = SHA256_BLOCK_SIZE,
853 .cra_ctxsize = sizeof(struct img_hash_ctx),
436e3bb5 854 .cra_init = img_hash_cra_sha256_init,
d358f1ab
JH		 855 .cra_exit = img_hash_cra_exit,
856 .cra_module = THIS_MODULE,
857 }
858 }
859 }
860};
861
862static int img_register_algs(struct img_hash_dev *hdev)
863{
864 int i, err;
865
866 for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
867 err = crypto_register_ahash(&img_algs[i]);
868 if (err)
869 goto err_reg;
870 }
871 return 0;
872
873err_reg:
874 for (; i--; )
875 crypto_unregister_ahash(&img_algs[i]);
876
877 return err;
878}
879
880static int img_unregister_algs(struct img_hash_dev *hdev)
881{
882 int i;
883
884 for (i = 0; i < ARRAY_SIZE(img_algs); i++)
885 crypto_unregister_ahash(&img_algs[i]);
886 return 0;
887}
888
889static void img_hash_done_task(unsigned long data)
890{
891 struct img_hash_dev *hdev = (struct img_hash_dev *)data;
892 int err = 0;
893
894 if (hdev->err == -EINVAL) {
895 err = hdev->err;
896 goto finish;
897 }
898
899 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
900 img_hash_handle_queue(hdev, NULL);
901 return;
902 }
903
904 if (DRIVER_FLAGS_CPU & hdev->flags) {
905 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
906 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
907 goto finish;
908 }
909 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
910 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
911 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
912 img_hash_write_via_dma_stop(hdev);
913 if (hdev->err) {
914 err = hdev->err;
915 goto finish;
916 }
917 }
918 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
919 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
920 DRIVER_FLAGS_OUTPUT_READY);
921 goto finish;
922 }
923 }
924 return;
925
926finish:
927 img_hash_finish_req(hdev->req, err);
928}
929
930static const struct of_device_id img_hash_match[] = {
931 { .compatible = "img,hash-accelerator" },
932 {}
933};
7094e8ea 934MODULE_DEVICE_TABLE(of, img_hash_match);
d358f1ab
JH		 935
936static int img_hash_probe(struct platform_device *pdev)
937{
938 struct img_hash_dev *hdev;
939 struct device *dev = &pdev->dev;
940 struct resource *hash_res;
941 int irq;
942 int err;
943
944 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
945 if (hdev == NULL)
946 return -ENOMEM;
947
948 spin_lock_init(&hdev->lock);
949
950 hdev->dev = dev;
951
952 platform_set_drvdata(pdev, hdev);
953
954 INIT_LIST_HEAD(&hdev->list);
955
956 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
957 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
958
959 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
960
961 /* Register bank */
f78c7123 962 hdev->io_base = devm_platform_ioremap_resource(pdev, 0);
d358f1ab
JH		 963 if (IS_ERR(hdev->io_base)) {
964 err = PTR_ERR(hdev->io_base);
965 dev_err(dev, "can't ioremap, returned %d\n", err);
966
967 goto res_err;
968 }
969
970 /* Write port (DMA or CPU) */
971 hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
972 hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
973 if (IS_ERR(hdev->cpu_addr)) {
974 dev_err(dev, "can't ioremap write port\n");
975 err = PTR_ERR(hdev->cpu_addr);
976 goto res_err;
977 }
978 hdev->bus_addr = hash_res->start;
979
980 irq = platform_get_irq(pdev, 0);
981 if (irq < 0) {
d358f1ab
JH		 982 err = irq;
983 goto res_err;
984 }
985
986 err = devm_request_irq(dev, irq, img_irq_handler, 0,
987 dev_name(dev), hdev);
988 if (err) {
989 dev_err(dev, "unable to request irq\n");
990 goto res_err;
991 }
992 dev_dbg(dev, "using IRQ channel %d\n", irq);
993
994 hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
995 if (IS_ERR(hdev->hash_clk)) {
996 dev_err(dev, "clock initialization failed.\n");
997 err = PTR_ERR(hdev->hash_clk);
998 goto res_err;
999 }
1000
1001 hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
1002 if (IS_ERR(hdev->sys_clk)) {
1003 dev_err(dev, "clock initialization failed.\n");
1004 err = PTR_ERR(hdev->sys_clk);
1005 goto res_err;
1006 }
1007
1008 err = clk_prepare_enable(hdev->hash_clk);
1009 if (err)
1010 goto res_err;
1011
1012 err = clk_prepare_enable(hdev->sys_clk);
1013 if (err)
1014 goto clk_err;
1015
1016 err = img_hash_dma_init(hdev);
1017 if (err)
1018 goto dma_err;
1019
1020 dev_dbg(dev, "using %s for DMA transfers\n",
1021 dma_chan_name(hdev->dma_lch));
1022
1023 spin_lock(&img_hash.lock);
1024 list_add_tail(&hdev->list, &img_hash.dev_list);
1025 spin_unlock(&img_hash.lock);
1026
1027 err = img_register_algs(hdev);
1028 if (err)
1029 goto err_algs;
fb67740e 1030 dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
d358f1ab
JH		 1031
1032 return 0;
1033
1034err_algs:
1035 spin_lock(&img_hash.lock);
1036 list_del(&hdev->list);
1037 spin_unlock(&img_hash.lock);
1038 dma_release_channel(hdev->dma_lch);
1039dma_err:
1040 clk_disable_unprepare(hdev->sys_clk);
1041clk_err:
1042 clk_disable_unprepare(hdev->hash_clk);
1043res_err:
1044 tasklet_kill(&hdev->done_task);
1045 tasklet_kill(&hdev->dma_task);
1046
1047 return err;
1048}
1049
1050static int img_hash_remove(struct platform_device *pdev)
1051{
1ce5b2f3 1052 struct img_hash_dev *hdev;
d358f1ab
JH		 1053
1054 hdev = platform_get_drvdata(pdev);
1055 spin_lock(&img_hash.lock);
1056 list_del(&hdev->list);
1057 spin_unlock(&img_hash.lock);
1058
1059 img_unregister_algs(hdev);
1060
1061 tasklet_kill(&hdev->done_task);
1062 tasklet_kill(&hdev->dma_task);
1063
1064 dma_release_channel(hdev->dma_lch);
1065
1066 clk_disable_unprepare(hdev->hash_clk);
1067 clk_disable_unprepare(hdev->sys_clk);
1068
1069 return 0;
1070}
1071
d084e13a
GR		 1072#ifdef CONFIG_PM_SLEEP
1073static int img_hash_suspend(struct device *dev)
1074{
1075 struct img_hash_dev *hdev = dev_get_drvdata(dev);
1076
1077 clk_disable_unprepare(hdev->hash_clk);
1078 clk_disable_unprepare(hdev->sys_clk);
1079
1080 return 0;
1081}
1082
1083static int img_hash_resume(struct device *dev)
1084{
1085 struct img_hash_dev *hdev = dev_get_drvdata(dev);
5f052c9c 1086 int ret;
d084e13a 1087
5f052c9c
AY		 1088 ret = clk_prepare_enable(hdev->hash_clk);
1089 if (ret)
1090 return ret;
1091
1092 ret = clk_prepare_enable(hdev->sys_clk);
1093 if (ret) {
1094 clk_disable_unprepare(hdev->hash_clk);
1095 return ret;
1096 }
d084e13a
GR		 1097
1098 return 0;
1099}
1100#endif /* CONFIG_PM_SLEEP */
1101
1102static const struct dev_pm_ops img_hash_pm_ops = {
1103 SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)
1104};
1105
d358f1ab
JH		 1106static struct platform_driver img_hash_driver = {
1107 .probe = img_hash_probe,
1108 .remove = img_hash_remove,
1109 .driver = {
1110 .name = "img-hash-accelerator",
d084e13a 1111 .pm = &img_hash_pm_ops,
d358f1ab
JH		 1112 .of_match_table = of_match_ptr(img_hash_match),
1113 }
1114};
1115module_platform_driver(img_hash_driver);
1116
1117MODULE_LICENSE("GPL v2");
1118MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1119MODULE_AUTHOR("Will Thomas.");
1120MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");
Linux 6.x block layer and io_uring tree(s)