projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'mm-hotfixes-stable-2025-07-11-16-16' of git://git.kernel.org/pub/scm/linux...
[linux-2.6-block.git] / drivers / crypto / ccp / ccp-ops.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * AMD Cryptographic Coprocessor (CCP) driver
4 *
5 * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6 *
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
9 */
10
11#include <crypto/des.h>
12#include <crypto/scatterwalk.h>
13#include <crypto/utils.h>
14#include <linux/ccp.h>
15#include <linux/dma-mapping.h>
16#include <linux/errno.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19
20#include "ccp-dev.h"
21
22/* SHA initial context values */
23static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
24 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
25 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
26 cpu_to_be32(SHA1_H4),
27};
28
29static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
30 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
31 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
32 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
33 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
34};
35
36static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
37 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
38 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
39 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
40 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
41};
42
43static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
44 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
45 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
46 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
47 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
48};
49
50static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
51 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
52 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
53 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
54 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
55};
56
57#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
58 ccp_gen_jobid(ccp) : 0)
59
60static u32 ccp_gen_jobid(struct ccp_device *ccp)
61{
62 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
63}
64
65static void ccp_sg_free(struct ccp_sg_workarea *wa)
66{
67 if (wa->dma_count)
68 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
69
70 wa->dma_count = 0;
71}
72
73static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
74 struct scatterlist *sg, u64 len,
75 enum dma_data_direction dma_dir)
76{
77 memset(wa, 0, sizeof(*wa));
78
79 wa->sg = sg;
80 if (!sg)
81 return 0;
82
83 wa->nents = sg_nents_for_len(sg, len);
84 if (wa->nents < 0)
85 return wa->nents;
86
87 wa->bytes_left = len;
88 wa->sg_used = 0;
89
90 if (len == 0)
91 return 0;
92
93 if (dma_dir == DMA_NONE)
94 return 0;
95
96 wa->dma_sg = sg;
97 wa->dma_sg_head = sg;
98 wa->dma_dev = dev;
99 wa->dma_dir = dma_dir;
100 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
101 if (!wa->dma_count)
102 return -ENOMEM;
103
104 return 0;
105}
106
107static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
108{
109 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
110 unsigned int sg_combined_len = 0;
111
112 if (!wa->sg)
113 return;
114
115 wa->sg_used += nbytes;
116 wa->bytes_left -= nbytes;
117 if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
118 /* Advance to the next DMA scatterlist entry */
119 wa->dma_sg = sg_next(wa->dma_sg);
120
121 /* In the case that the DMA mapped scatterlist has entries
122 * that have been merged, the non-DMA mapped scatterlist
123 * must be advanced multiple times for each merged entry.
124 * This ensures that the current non-DMA mapped entry
125 * corresponds to the current DMA mapped entry.
126 */
127 do {
128 sg_combined_len += wa->sg->length;
129 wa->sg = sg_next(wa->sg);
130 } while (wa->sg_used > sg_combined_len);
131
132 wa->sg_used = 0;
133 }
134}
135
136static void ccp_dm_free(struct ccp_dm_workarea *wa)
137{
138 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
139 if (wa->address)
140 dma_pool_free(wa->dma_pool, wa->address,
141 wa->dma.address);
142 } else {
143 if (wa->dma.address)
144 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
145 wa->dma.dir);
146 kfree(wa->address);
147 }
148
149 wa->address = NULL;
150 wa->dma.address = 0;
151}
152
153static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
154 struct ccp_cmd_queue *cmd_q,
155 unsigned int len,
156 enum dma_data_direction dir)
157{
158 memset(wa, 0, sizeof(*wa));
159
160 if (!len)
161 return 0;
162
163 wa->dev = cmd_q->ccp->dev;
164 wa->length = len;
165
166 if (len <= CCP_DMAPOOL_MAX_SIZE) {
167 wa->dma_pool = cmd_q->dma_pool;
168
169 wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
170 &wa->dma.address);
171 if (!wa->address)
172 return -ENOMEM;
173
174 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
175
176 } else {
177 wa->address = kzalloc(len, GFP_KERNEL);
178 if (!wa->address)
179 return -ENOMEM;
180
181 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
182 dir);
183 if (dma_mapping_error(wa->dev, wa->dma.address)) {
184 kfree(wa->address);
185 wa->address = NULL;
186 return -ENOMEM;
187 }
188
189 wa->dma.length = len;
190 }
191 wa->dma.dir = dir;
192
193 return 0;
194}
195
196static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
197 struct scatterlist *sg, unsigned int sg_offset,
198 unsigned int len)
199{
200 WARN_ON(!wa->address);
201
202 if (len > (wa->length - wa_offset))
203 return -EINVAL;
204
205 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
206 0);
207 return 0;
208}
209
210static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
211 struct scatterlist *sg, unsigned int sg_offset,
212 unsigned int len)
213{
214 WARN_ON(!wa->address);
215
216 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
217 1);
218}
219
220static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
221 unsigned int wa_offset,
222 struct scatterlist *sg,
223 unsigned int sg_offset,
224 unsigned int len)
225{
226 u8 *p, *q;
227 int rc;
228
229 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
230 if (rc)
231 return rc;
232
233 p = wa->address + wa_offset;
234 q = p + len - 1;
235 while (p < q) {
236 *p = *p ^ *q;
237 *q = *p ^ *q;
238 *p = *p ^ *q;
239 p++;
240 q--;
241 }
242 return 0;
243}
244
245static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
246 unsigned int wa_offset,
247 struct scatterlist *sg,
248 unsigned int sg_offset,
249 unsigned int len)
250{
251 u8 *p, *q;
252
253 p = wa->address + wa_offset;
254 q = p + len - 1;
255 while (p < q) {
256 *p = *p ^ *q;
257 *q = *p ^ *q;
258 *p = *p ^ *q;
259 p++;
260 q--;
261 }
262
263 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
264}
265
266static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
267{
268 ccp_dm_free(&data->dm_wa);
269 ccp_sg_free(&data->sg_wa);
270}
271
272static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
273 struct scatterlist *sg, u64 sg_len,
274 unsigned int dm_len,
275 enum dma_data_direction dir)
276{
277 int ret;
278
279 memset(data, 0, sizeof(*data));
280
281 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
282 dir);
283 if (ret)
284 goto e_err;
285
286 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
287 if (ret)
288 goto e_err;
289
290 return 0;
291
292e_err:
293 ccp_free_data(data, cmd_q);
294
295 return ret;
296}
297
298static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
299{
300 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
301 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
302 unsigned int buf_count, nbytes;
303
304 /* Clear the buffer if setting it */
305 if (!from)
306 memset(dm_wa->address, 0, dm_wa->length);
307
308 if (!sg_wa->sg)
309 return 0;
310
311 /* Perform the copy operation
312 * nbytes will always be <= UINT_MAX because dm_wa->length is
313 * an unsigned int
314 */
315 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
316 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
317 nbytes, from);
318
319 /* Update the structures and generate the count */
320 buf_count = 0;
321 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
322 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
323 dm_wa->length - buf_count);
324 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
325
326 buf_count += nbytes;
327 ccp_update_sg_workarea(sg_wa, nbytes);
328 }
329
330 return buf_count;
331}
332
333static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
334{
335 return ccp_queue_buf(data, 0);
336}
337
338static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
339{
340 return ccp_queue_buf(data, 1);
341}
342
343static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
344 struct ccp_op *op, unsigned int block_size,
345 bool blocksize_op)
346{
347 unsigned int sg_src_len, sg_dst_len, op_len;
348
349 /* The CCP can only DMA from/to one address each per operation. This
350 * requires that we find the smallest DMA area between the source
351 * and destination. The resulting len values will always be <= UINT_MAX
352 * because the dma length is an unsigned int.
353 */
354 sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
355 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
356
357 if (dst) {
358 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
359 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
360 op_len = min(sg_src_len, sg_dst_len);
361 } else {
362 op_len = sg_src_len;
363 }
364
365 /* The data operation length will be at least block_size in length
366 * or the smaller of available sg room remaining for the source or
367 * the destination
368 */
369 op_len = max(op_len, block_size);
370
371 /* Unless we have to buffer data, there's no reason to wait */
372 op->soc = 0;
373
374 if (sg_src_len < block_size) {
375 /* Not enough data in the sg element, so it
376 * needs to be buffered into a blocksize chunk
377 */
378 int cp_len = ccp_fill_queue_buf(src);
379
380 op->soc = 1;
381 op->src.u.dma.address = src->dm_wa.dma.address;
382 op->src.u.dma.offset = 0;
383 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
384 } else {
385 /* Enough data in the sg element, but we need to
386 * adjust for any previously copied data
387 */
388 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
389 op->src.u.dma.offset = src->sg_wa.sg_used;
390 op->src.u.dma.length = op_len & ~(block_size - 1);
391
392 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
393 }
394
395 if (dst) {
396 if (sg_dst_len < block_size) {
397 /* Not enough room in the sg element or we're on the
398 * last piece of data (when using padding), so the
399 * output needs to be buffered into a blocksize chunk
400 */
401 op->soc = 1;
402 op->dst.u.dma.address = dst->dm_wa.dma.address;
403 op->dst.u.dma.offset = 0;
404 op->dst.u.dma.length = op->src.u.dma.length;
405 } else {
406 /* Enough room in the sg element, but we need to
407 * adjust for any previously used area
408 */
409 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
410 op->dst.u.dma.offset = dst->sg_wa.sg_used;
411 op->dst.u.dma.length = op->src.u.dma.length;
412 }
413 }
414}
415
416static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
417 struct ccp_op *op)
418{
419 op->init = 0;
420
421 if (dst) {
422 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
423 ccp_empty_queue_buf(dst);
424 else
425 ccp_update_sg_workarea(&dst->sg_wa,
426 op->dst.u.dma.length);
427 }
428}
429
430static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
431 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
432 u32 byte_swap, bool from)
433{
434 struct ccp_op op;
435
436 memset(&op, 0, sizeof(op));
437
438 op.cmd_q = cmd_q;
439 op.jobid = jobid;
440 op.eom = 1;
441
442 if (from) {
443 op.soc = 1;
444 op.src.type = CCP_MEMTYPE_SB;
445 op.src.u.sb = sb;
446 op.dst.type = CCP_MEMTYPE_SYSTEM;
447 op.dst.u.dma.address = wa->dma.address;
448 op.dst.u.dma.length = wa->length;
449 } else {
450 op.src.type = CCP_MEMTYPE_SYSTEM;
451 op.src.u.dma.address = wa->dma.address;
452 op.src.u.dma.length = wa->length;
453 op.dst.type = CCP_MEMTYPE_SB;
454 op.dst.u.sb = sb;
455 }
456
457 op.u.passthru.byte_swap = byte_swap;
458
459 return cmd_q->ccp->vdata->perform->passthru(&op);
460}
461
462static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
463 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
464 u32 byte_swap)
465{
466 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
467}
468
469static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
470 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
471 u32 byte_swap)
472{
473 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
474}
475
476static noinline_for_stack int
477ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
478{
479 struct ccp_aes_engine *aes = &cmd->u.aes;
480 struct ccp_dm_workarea key, ctx;
481 struct ccp_data src;
482 struct ccp_op op;
483 unsigned int dm_offset;
484 int ret;
485
486 if (!((aes->key_len == AES_KEYSIZE_128) ||
487 (aes->key_len == AES_KEYSIZE_192) ||
488 (aes->key_len == AES_KEYSIZE_256)))
489 return -EINVAL;
490
491 if (aes->src_len & (AES_BLOCK_SIZE - 1))
492 return -EINVAL;
493
494 if (aes->iv_len != AES_BLOCK_SIZE)
495 return -EINVAL;
496
497 if (!aes->key || !aes->iv || !aes->src)
498 return -EINVAL;
499
500 if (aes->cmac_final) {
501 if (aes->cmac_key_len != AES_BLOCK_SIZE)
502 return -EINVAL;
503
504 if (!aes->cmac_key)
505 return -EINVAL;
506 }
507
508 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
509 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
510
511 ret = -EIO;
512 memset(&op, 0, sizeof(op));
513 op.cmd_q = cmd_q;
514 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
515 op.sb_key = cmd_q->sb_key;
516 op.sb_ctx = cmd_q->sb_ctx;
517 op.init = 1;
518 op.u.aes.type = aes->type;
519 op.u.aes.mode = aes->mode;
520 op.u.aes.action = aes->action;
521
522 /* All supported key sizes fit in a single (32-byte) SB entry
523 * and must be in little endian format. Use the 256-bit byte
524 * swap passthru option to convert from big endian to little
525 * endian.
526 */
527 ret = ccp_init_dm_workarea(&key, cmd_q,
528 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
529 DMA_TO_DEVICE);
530 if (ret)
531 return ret;
532
533 dm_offset = CCP_SB_BYTES - aes->key_len;
534 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
535 if (ret)
536 goto e_key;
537 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
538 CCP_PASSTHRU_BYTESWAP_256BIT);
539 if (ret) {
540 cmd->engine_error = cmd_q->cmd_error;
541 goto e_key;
542 }
543
544 /* The AES context fits in a single (32-byte) SB entry and
545 * must be in little endian format. Use the 256-bit byte swap
546 * passthru option to convert from big endian to little endian.
547 */
548 ret = ccp_init_dm_workarea(&ctx, cmd_q,
549 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
550 DMA_BIDIRECTIONAL);
551 if (ret)
552 goto e_key;
553
554 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
555 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
556 if (ret)
557 goto e_ctx;
558 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
559 CCP_PASSTHRU_BYTESWAP_256BIT);
560 if (ret) {
561 cmd->engine_error = cmd_q->cmd_error;
562 goto e_ctx;
563 }
564
565 /* Send data to the CCP AES engine */
566 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
567 AES_BLOCK_SIZE, DMA_TO_DEVICE);
568 if (ret)
569 goto e_ctx;
570
571 while (src.sg_wa.bytes_left) {
572 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
573 if (aes->cmac_final && !src.sg_wa.bytes_left) {
574 op.eom = 1;
575
576 /* Push the K1/K2 key to the CCP now */
577 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
578 op.sb_ctx,
579 CCP_PASSTHRU_BYTESWAP_256BIT);
580 if (ret) {
581 cmd->engine_error = cmd_q->cmd_error;
582 goto e_src;
583 }
584
585 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
586 aes->cmac_key_len);
587 if (ret)
588 goto e_src;
589 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
590 CCP_PASSTHRU_BYTESWAP_256BIT);
591 if (ret) {
592 cmd->engine_error = cmd_q->cmd_error;
593 goto e_src;
594 }
595 }
596
597 ret = cmd_q->ccp->vdata->perform->aes(&op);
598 if (ret) {
599 cmd->engine_error = cmd_q->cmd_error;
600 goto e_src;
601 }
602
603 ccp_process_data(&src, NULL, &op);
604 }
605
606 /* Retrieve the AES context - convert from LE to BE using
607 * 32-byte (256-bit) byteswapping
608 */
609 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
610 CCP_PASSTHRU_BYTESWAP_256BIT);
611 if (ret) {
612 cmd->engine_error = cmd_q->cmd_error;
613 goto e_src;
614 }
615
616 /* ...but we only need AES_BLOCK_SIZE bytes */
617 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
618 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
619
620e_src:
621 ccp_free_data(&src, cmd_q);
622
623e_ctx:
624 ccp_dm_free(&ctx);
625
626e_key:
627 ccp_dm_free(&key);
628
629 return ret;
630}
631
632static noinline_for_stack int
633ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
634{
635 struct ccp_aes_engine *aes = &cmd->u.aes;
636 struct ccp_dm_workarea key, ctx, final_wa, tag;
637 struct ccp_data src, dst;
638 struct ccp_data aad;
639 struct ccp_op op;
640 unsigned int dm_offset;
641 unsigned int authsize;
642 unsigned int jobid;
643 unsigned int ilen;
644 bool in_place = true; /* Default value */
645 __be64 *final;
646 int ret;
647
648 struct scatterlist *p_inp, sg_inp[2];
649 struct scatterlist *p_tag, sg_tag[2];
650 struct scatterlist *p_outp, sg_outp[2];
651 struct scatterlist *p_aad;
652
653 if (!aes->iv)
654 return -EINVAL;
655
656 if (!((aes->key_len == AES_KEYSIZE_128) ||
657 (aes->key_len == AES_KEYSIZE_192) ||
658 (aes->key_len == AES_KEYSIZE_256)))
659 return -EINVAL;
660
661 if (!aes->key) /* Gotta have a key SGL */
662 return -EINVAL;
663
664 /* Zero defaults to 16 bytes, the maximum size */
665 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
666 switch (authsize) {
667 case 16:
668 case 15:
669 case 14:
670 case 13:
671 case 12:
672 case 8:
673 case 4:
674 break;
675 default:
676 return -EINVAL;
677 }
678
679 /* First, decompose the source buffer into AAD & PT,
680 * and the destination buffer into AAD, CT & tag, or
681 * the input into CT & tag.
682 * It is expected that the input and output SGs will
683 * be valid, even if the AAD and input lengths are 0.
684 */
685 p_aad = aes->src;
686 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
687 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
688 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
689 ilen = aes->src_len;
690 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
691 } else {
692 /* Input length for decryption includes tag */
693 ilen = aes->src_len - authsize;
694 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
695 }
696
697 jobid = CCP_NEW_JOBID(cmd_q->ccp);
698
699 memset(&op, 0, sizeof(op));
700 op.cmd_q = cmd_q;
701 op.jobid = jobid;
702 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
703 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
704 op.init = 1;
705 op.u.aes.type = aes->type;
706
707 /* Copy the key to the LSB */
708 ret = ccp_init_dm_workarea(&key, cmd_q,
709 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
710 DMA_TO_DEVICE);
711 if (ret)
712 return ret;
713
714 dm_offset = CCP_SB_BYTES - aes->key_len;
715 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
716 if (ret)
717 goto e_key;
718 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
719 CCP_PASSTHRU_BYTESWAP_256BIT);
720 if (ret) {
721 cmd->engine_error = cmd_q->cmd_error;
722 goto e_key;
723 }
724
725 /* Copy the context (IV) to the LSB.
726 * There is an assumption here that the IV is 96 bits in length, plus
727 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
728 */
729 ret = ccp_init_dm_workarea(&ctx, cmd_q,
730 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
731 DMA_BIDIRECTIONAL);
732 if (ret)
733 goto e_key;
734
735 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
736 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
737 if (ret)
738 goto e_ctx;
739
740 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
741 CCP_PASSTHRU_BYTESWAP_256BIT);
742 if (ret) {
743 cmd->engine_error = cmd_q->cmd_error;
744 goto e_ctx;
745 }
746
747 op.init = 1;
748 if (aes->aad_len > 0) {
749 /* Step 1: Run a GHASH over the Additional Authenticated Data */
750 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
751 AES_BLOCK_SIZE,
752 DMA_TO_DEVICE);
753 if (ret)
754 goto e_ctx;
755
756 op.u.aes.mode = CCP_AES_MODE_GHASH;
757 op.u.aes.action = CCP_AES_GHASHAAD;
758
759 while (aad.sg_wa.bytes_left) {
760 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
761
762 ret = cmd_q->ccp->vdata->perform->aes(&op);
763 if (ret) {
764 cmd->engine_error = cmd_q->cmd_error;
765 goto e_aad;
766 }
767
768 ccp_process_data(&aad, NULL, &op);
769 op.init = 0;
770 }
771 }
772
773 op.u.aes.mode = CCP_AES_MODE_GCTR;
774 op.u.aes.action = aes->action;
775
776 if (ilen > 0) {
777 /* Step 2: Run a GCTR over the plaintext */
778 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
779
780 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
781 AES_BLOCK_SIZE,
782 in_place ? DMA_BIDIRECTIONAL
783 : DMA_TO_DEVICE);
784 if (ret)
785 goto e_aad;
786
787 if (in_place) {
788 dst = src;
789 } else {
790 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
791 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
792 if (ret)
793 goto e_src;
794 }
795
796 op.soc = 0;
797 op.eom = 0;
798 op.init = 1;
799 while (src.sg_wa.bytes_left) {
800 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
801 if (!src.sg_wa.bytes_left) {
802 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
803
804 if (nbytes) {
805 op.eom = 1;
806 op.u.aes.size = (nbytes * 8) - 1;
807 }
808 }
809
810 ret = cmd_q->ccp->vdata->perform->aes(&op);
811 if (ret) {
812 cmd->engine_error = cmd_q->cmd_error;
813 goto e_dst;
814 }
815
816 ccp_process_data(&src, &dst, &op);
817 op.init = 0;
818 }
819 }
820
821 /* Step 3: Update the IV portion of the context with the original IV */
822 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
823 CCP_PASSTHRU_BYTESWAP_256BIT);
824 if (ret) {
825 cmd->engine_error = cmd_q->cmd_error;
826 goto e_dst;
827 }
828
829 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
830 if (ret)
831 goto e_dst;
832
833 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
834 CCP_PASSTHRU_BYTESWAP_256BIT);
835 if (ret) {
836 cmd->engine_error = cmd_q->cmd_error;
837 goto e_dst;
838 }
839
840 /* Step 4: Concatenate the lengths of the AAD and source, and
841 * hash that 16 byte buffer.
842 */
843 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
844 DMA_BIDIRECTIONAL);
845 if (ret)
846 goto e_dst;
847 final = (__be64 *)final_wa.address;
848 final[0] = cpu_to_be64(aes->aad_len * 8);
849 final[1] = cpu_to_be64(ilen * 8);
850
851 memset(&op, 0, sizeof(op));
852 op.cmd_q = cmd_q;
853 op.jobid = jobid;
854 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
855 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
856 op.init = 1;
857 op.u.aes.type = aes->type;
858 op.u.aes.mode = CCP_AES_MODE_GHASH;
859 op.u.aes.action = CCP_AES_GHASHFINAL;
860 op.src.type = CCP_MEMTYPE_SYSTEM;
861 op.src.u.dma.address = final_wa.dma.address;
862 op.src.u.dma.length = AES_BLOCK_SIZE;
863 op.dst.type = CCP_MEMTYPE_SYSTEM;
864 op.dst.u.dma.address = final_wa.dma.address;
865 op.dst.u.dma.length = AES_BLOCK_SIZE;
866 op.eom = 1;
867 op.u.aes.size = 0;
868 ret = cmd_q->ccp->vdata->perform->aes(&op);
869 if (ret)
870 goto e_final_wa;
871
872 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
873 /* Put the ciphered tag after the ciphertext. */
874 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
875 } else {
876 /* Does this ciphered tag match the input? */
877 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
878 DMA_BIDIRECTIONAL);
879 if (ret)
880 goto e_final_wa;
881 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
882 if (ret) {
883 ccp_dm_free(&tag);
884 goto e_final_wa;
885 }
886
887 ret = crypto_memneq(tag.address, final_wa.address,
888 authsize) ? -EBADMSG : 0;
889 ccp_dm_free(&tag);
890 }
891
892e_final_wa:
893 ccp_dm_free(&final_wa);
894
895e_dst:
896 if (ilen > 0 && !in_place)
897 ccp_free_data(&dst, cmd_q);
898
899e_src:
900 if (ilen > 0)
901 ccp_free_data(&src, cmd_q);
902
903e_aad:
904 if (aes->aad_len)
905 ccp_free_data(&aad, cmd_q);
906
907e_ctx:
908 ccp_dm_free(&ctx);
909
910e_key:
911 ccp_dm_free(&key);
912
913 return ret;
914}
915
916static noinline_for_stack int
917ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
918{
919 struct ccp_aes_engine *aes = &cmd->u.aes;
920 struct ccp_dm_workarea key, ctx;
921 struct ccp_data src, dst;
922 struct ccp_op op;
923 unsigned int dm_offset;
924 bool in_place = false;
925 int ret;
926
927 if (!((aes->key_len == AES_KEYSIZE_128) ||
928 (aes->key_len == AES_KEYSIZE_192) ||
929 (aes->key_len == AES_KEYSIZE_256)))
930 return -EINVAL;
931
932 if (((aes->mode == CCP_AES_MODE_ECB) ||
933 (aes->mode == CCP_AES_MODE_CBC)) &&
934 (aes->src_len & (AES_BLOCK_SIZE - 1)))
935 return -EINVAL;
936
937 if (!aes->key || !aes->src || !aes->dst)
938 return -EINVAL;
939
940 if (aes->mode != CCP_AES_MODE_ECB) {
941 if (aes->iv_len != AES_BLOCK_SIZE)
942 return -EINVAL;
943
944 if (!aes->iv)
945 return -EINVAL;
946 }
947
948 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
949 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
950
951 ret = -EIO;
952 memset(&op, 0, sizeof(op));
953 op.cmd_q = cmd_q;
954 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
955 op.sb_key = cmd_q->sb_key;
956 op.sb_ctx = cmd_q->sb_ctx;
957 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
958 op.u.aes.type = aes->type;
959 op.u.aes.mode = aes->mode;
960 op.u.aes.action = aes->action;
961
962 /* All supported key sizes fit in a single (32-byte) SB entry
963 * and must be in little endian format. Use the 256-bit byte
964 * swap passthru option to convert from big endian to little
965 * endian.
966 */
967 ret = ccp_init_dm_workarea(&key, cmd_q,
968 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
969 DMA_TO_DEVICE);
970 if (ret)
971 return ret;
972
973 dm_offset = CCP_SB_BYTES - aes->key_len;
974 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
975 if (ret)
976 goto e_key;
977 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
978 CCP_PASSTHRU_BYTESWAP_256BIT);
979 if (ret) {
980 cmd->engine_error = cmd_q->cmd_error;
981 goto e_key;
982 }
983
984 /* The AES context fits in a single (32-byte) SB entry and
985 * must be in little endian format. Use the 256-bit byte swap
986 * passthru option to convert from big endian to little endian.
987 */
988 ret = ccp_init_dm_workarea(&ctx, cmd_q,
989 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
990 DMA_BIDIRECTIONAL);
991 if (ret)
992 goto e_key;
993
994 if (aes->mode != CCP_AES_MODE_ECB) {
995 /* Load the AES context - convert to LE */
996 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
997 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
998 if (ret)
999 goto e_ctx;
1000 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1001 CCP_PASSTHRU_BYTESWAP_256BIT);
1002 if (ret) {
1003 cmd->engine_error = cmd_q->cmd_error;
1004 goto e_ctx;
1005 }
1006 }
1007 switch (aes->mode) {
1008 case CCP_AES_MODE_CFB: /* CFB128 only */
1009 case CCP_AES_MODE_CTR:
1010 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1011 break;
1012 default:
1013 op.u.aes.size = 0;
1014 }
1015
1016 /* Prepare the input and output data workareas. For in-place
1017 * operations we need to set the dma direction to BIDIRECTIONAL
1018 * and copy the src workarea to the dst workarea.
1019 */
1020 if (sg_virt(aes->src) == sg_virt(aes->dst))
1021 in_place = true;
1022
1023 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1024 AES_BLOCK_SIZE,
1025 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1026 if (ret)
1027 goto e_ctx;
1028
1029 if (in_place) {
1030 dst = src;
1031 } else {
1032 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1033 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1034 if (ret)
1035 goto e_src;
1036 }
1037
1038 /* Send data to the CCP AES engine */
1039 while (src.sg_wa.bytes_left) {
1040 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1041 if (!src.sg_wa.bytes_left) {
1042 op.eom = 1;
1043
1044 /* Since we don't retrieve the AES context in ECB
1045 * mode we have to wait for the operation to complete
1046 * on the last piece of data
1047 */
1048 if (aes->mode == CCP_AES_MODE_ECB)
1049 op.soc = 1;
1050 }
1051
1052 ret = cmd_q->ccp->vdata->perform->aes(&op);
1053 if (ret) {
1054 cmd->engine_error = cmd_q->cmd_error;
1055 goto e_dst;
1056 }
1057
1058 ccp_process_data(&src, &dst, &op);
1059 }
1060
1061 if (aes->mode != CCP_AES_MODE_ECB) {
1062 /* Retrieve the AES context - convert from LE to BE using
1063 * 32-byte (256-bit) byteswapping
1064 */
1065 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1066 CCP_PASSTHRU_BYTESWAP_256BIT);
1067 if (ret) {
1068 cmd->engine_error = cmd_q->cmd_error;
1069 goto e_dst;
1070 }
1071
1072 /* ...but we only need AES_BLOCK_SIZE bytes */
1073 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1074 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1075 }
1076
1077e_dst:
1078 if (!in_place)
1079 ccp_free_data(&dst, cmd_q);
1080
1081e_src:
1082 ccp_free_data(&src, cmd_q);
1083
1084e_ctx:
1085 ccp_dm_free(&ctx);
1086
1087e_key:
1088 ccp_dm_free(&key);
1089
1090 return ret;
1091}
1092
1093static noinline_for_stack int
1094ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1095{
1096 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1097 struct ccp_dm_workarea key, ctx;
1098 struct ccp_data src, dst;
1099 struct ccp_op op;
1100 unsigned int unit_size, dm_offset;
1101 bool in_place = false;
1102 unsigned int sb_count;
1103 enum ccp_aes_type aestype;
1104 int ret;
1105
1106 switch (xts->unit_size) {
1107 case CCP_XTS_AES_UNIT_SIZE_16:
1108 unit_size = 16;
1109 break;
1110 case CCP_XTS_AES_UNIT_SIZE_512:
1111 unit_size = 512;
1112 break;
1113 case CCP_XTS_AES_UNIT_SIZE_1024:
1114 unit_size = 1024;
1115 break;
1116 case CCP_XTS_AES_UNIT_SIZE_2048:
1117 unit_size = 2048;
1118 break;
1119 case CCP_XTS_AES_UNIT_SIZE_4096:
1120 unit_size = 4096;
1121 break;
1122
1123 default:
1124 return -EINVAL;
1125 }
1126
1127 if (xts->key_len == AES_KEYSIZE_128)
1128 aestype = CCP_AES_TYPE_128;
1129 else if (xts->key_len == AES_KEYSIZE_256)
1130 aestype = CCP_AES_TYPE_256;
1131 else
1132 return -EINVAL;
1133
1134 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1135 return -EINVAL;
1136
1137 if (xts->iv_len != AES_BLOCK_SIZE)
1138 return -EINVAL;
1139
1140 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1141 return -EINVAL;
1142
1143 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1144 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1145
1146 ret = -EIO;
1147 memset(&op, 0, sizeof(op));
1148 op.cmd_q = cmd_q;
1149 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1150 op.sb_key = cmd_q->sb_key;
1151 op.sb_ctx = cmd_q->sb_ctx;
1152 op.init = 1;
1153 op.u.xts.type = aestype;
1154 op.u.xts.action = xts->action;
1155 op.u.xts.unit_size = xts->unit_size;
1156
1157 /* A version 3 device only supports 128-bit keys, which fits into a
1158 * single SB entry. A version 5 device uses a 512-bit vector, so two
1159 * SB entries.
1160 */
1161 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1162 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1163 else
1164 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1165 ret = ccp_init_dm_workarea(&key, cmd_q,
1166 sb_count * CCP_SB_BYTES,
1167 DMA_TO_DEVICE);
1168 if (ret)
1169 return ret;
1170
1171 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1172 /* All supported key sizes must be in little endian format.
1173 * Use the 256-bit byte swap passthru option to convert from
1174 * big endian to little endian.
1175 */
1176 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1177 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1178 if (ret)
1179 goto e_key;
1180 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1181 if (ret)
1182 goto e_key;
1183 } else {
1184 /* Version 5 CCPs use a 512-bit space for the key: each portion
1185 * occupies 256 bits, or one entire slot, and is zero-padded.
1186 */
1187 unsigned int pad;
1188
1189 dm_offset = CCP_SB_BYTES;
1190 pad = dm_offset - xts->key_len;
1191 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1192 if (ret)
1193 goto e_key;
1194 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1195 xts->key_len, xts->key_len);
1196 if (ret)
1197 goto e_key;
1198 }
1199 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1200 CCP_PASSTHRU_BYTESWAP_256BIT);
1201 if (ret) {
1202 cmd->engine_error = cmd_q->cmd_error;
1203 goto e_key;
1204 }
1205
1206 /* The AES context fits in a single (32-byte) SB entry and
1207 * for XTS is already in little endian format so no byte swapping
1208 * is needed.
1209 */
1210 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1211 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1212 DMA_BIDIRECTIONAL);
1213 if (ret)
1214 goto e_key;
1215
1216 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1217 if (ret)
1218 goto e_ctx;
1219 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1220 CCP_PASSTHRU_BYTESWAP_NOOP);
1221 if (ret) {
1222 cmd->engine_error = cmd_q->cmd_error;
1223 goto e_ctx;
1224 }
1225
1226 /* Prepare the input and output data workareas. For in-place
1227 * operations we need to set the dma direction to BIDIRECTIONAL
1228 * and copy the src workarea to the dst workarea.
1229 */
1230 if (sg_virt(xts->src) == sg_virt(xts->dst))
1231 in_place = true;
1232
1233 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1234 unit_size,
1235 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1236 if (ret)
1237 goto e_ctx;
1238
1239 if (in_place) {
1240 dst = src;
1241 } else {
1242 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1243 unit_size, DMA_FROM_DEVICE);
1244 if (ret)
1245 goto e_src;
1246 }
1247
1248 /* Send data to the CCP AES engine */
1249 while (src.sg_wa.bytes_left) {
1250 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1251 if (!src.sg_wa.bytes_left)
1252 op.eom = 1;
1253
1254 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1255 if (ret) {
1256 cmd->engine_error = cmd_q->cmd_error;
1257 goto e_dst;
1258 }
1259
1260 ccp_process_data(&src, &dst, &op);
1261 }
1262
1263 /* Retrieve the AES context - convert from LE to BE using
1264 * 32-byte (256-bit) byteswapping
1265 */
1266 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1267 CCP_PASSTHRU_BYTESWAP_256BIT);
1268 if (ret) {
1269 cmd->engine_error = cmd_q->cmd_error;
1270 goto e_dst;
1271 }
1272
1273 /* ...but we only need AES_BLOCK_SIZE bytes */
1274 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1275 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1276
1277e_dst:
1278 if (!in_place)
1279 ccp_free_data(&dst, cmd_q);
1280
1281e_src:
1282 ccp_free_data(&src, cmd_q);
1283
1284e_ctx:
1285 ccp_dm_free(&ctx);
1286
1287e_key:
1288 ccp_dm_free(&key);
1289
1290 return ret;
1291}
1292
1293static noinline_for_stack int
1294ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1295{
1296 struct ccp_des3_engine *des3 = &cmd->u.des3;
1297
1298 struct ccp_dm_workarea key, ctx;
1299 struct ccp_data src, dst;
1300 struct ccp_op op;
1301 unsigned int dm_offset;
1302 unsigned int len_singlekey;
1303 bool in_place = false;
1304 int ret;
1305
1306 /* Error checks */
1307 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1308 return -EINVAL;
1309
1310 if (!cmd_q->ccp->vdata->perform->des3)
1311 return -EINVAL;
1312
1313 if (des3->key_len != DES3_EDE_KEY_SIZE)
1314 return -EINVAL;
1315
1316 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1317 (des3->mode == CCP_DES3_MODE_CBC)) &&
1318 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1319 return -EINVAL;
1320
1321 if (!des3->key || !des3->src || !des3->dst)
1322 return -EINVAL;
1323
1324 if (des3->mode != CCP_DES3_MODE_ECB) {
1325 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1326 return -EINVAL;
1327
1328 if (!des3->iv)
1329 return -EINVAL;
1330 }
1331
1332 /* Zero out all the fields of the command desc */
1333 memset(&op, 0, sizeof(op));
1334
1335 /* Set up the Function field */
1336 op.cmd_q = cmd_q;
1337 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1338 op.sb_key = cmd_q->sb_key;
1339
1340 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1341 op.u.des3.type = des3->type;
1342 op.u.des3.mode = des3->mode;
1343 op.u.des3.action = des3->action;
1344
1345 /*
1346 * All supported key sizes fit in a single (32-byte) KSB entry and
1347 * (like AES) must be in little endian format. Use the 256-bit byte
1348 * swap passthru option to convert from big endian to little endian.
1349 */
1350 ret = ccp_init_dm_workarea(&key, cmd_q,
1351 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1352 DMA_TO_DEVICE);
1353 if (ret)
1354 return ret;
1355
1356 /*
1357 * The contents of the key triplet are in the reverse order of what
1358 * is required by the engine. Copy the 3 pieces individually to put
1359 * them where they belong.
1360 */
1361 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1362
1363 len_singlekey = des3->key_len / 3;
1364 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1365 des3->key, 0, len_singlekey);
1366 if (ret)
1367 goto e_key;
1368 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1369 des3->key, len_singlekey, len_singlekey);
1370 if (ret)
1371 goto e_key;
1372 ret = ccp_set_dm_area(&key, dm_offset,
1373 des3->key, 2 * len_singlekey, len_singlekey);
1374 if (ret)
1375 goto e_key;
1376
1377 /* Copy the key to the SB */
1378 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1379 CCP_PASSTHRU_BYTESWAP_256BIT);
1380 if (ret) {
1381 cmd->engine_error = cmd_q->cmd_error;
1382 goto e_key;
1383 }
1384
1385 /*
1386 * The DES3 context fits in a single (32-byte) KSB entry and
1387 * must be in little endian format. Use the 256-bit byte swap
1388 * passthru option to convert from big endian to little endian.
1389 */
1390 if (des3->mode != CCP_DES3_MODE_ECB) {
1391 op.sb_ctx = cmd_q->sb_ctx;
1392
1393 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1394 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1395 DMA_BIDIRECTIONAL);
1396 if (ret)
1397 goto e_key;
1398
1399 /* Load the context into the LSB */
1400 dm_offset = CCP_SB_BYTES - des3->iv_len;
1401 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1402 des3->iv_len);
1403 if (ret)
1404 goto e_ctx;
1405
1406 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1407 CCP_PASSTHRU_BYTESWAP_256BIT);
1408 if (ret) {
1409 cmd->engine_error = cmd_q->cmd_error;
1410 goto e_ctx;
1411 }
1412 }
1413
1414 /*
1415 * Prepare the input and output data workareas. For in-place
1416 * operations we need to set the dma direction to BIDIRECTIONAL
1417 * and copy the src workarea to the dst workarea.
1418 */
1419 if (sg_virt(des3->src) == sg_virt(des3->dst))
1420 in_place = true;
1421
1422 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1423 DES3_EDE_BLOCK_SIZE,
1424 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1425 if (ret)
1426 goto e_ctx;
1427
1428 if (in_place)
1429 dst = src;
1430 else {
1431 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1432 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1433 if (ret)
1434 goto e_src;
1435 }
1436
1437 /* Send data to the CCP DES3 engine */
1438 while (src.sg_wa.bytes_left) {
1439 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1440 if (!src.sg_wa.bytes_left) {
1441 op.eom = 1;
1442
1443 /* Since we don't retrieve the context in ECB mode
1444 * we have to wait for the operation to complete
1445 * on the last piece of data
1446 */
1447 op.soc = 0;
1448 }
1449
1450 ret = cmd_q->ccp->vdata->perform->des3(&op);
1451 if (ret) {
1452 cmd->engine_error = cmd_q->cmd_error;
1453 goto e_dst;
1454 }
1455
1456 ccp_process_data(&src, &dst, &op);
1457 }
1458
1459 if (des3->mode != CCP_DES3_MODE_ECB) {
1460 /* Retrieve the context and make BE */
1461 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1462 CCP_PASSTHRU_BYTESWAP_256BIT);
1463 if (ret) {
1464 cmd->engine_error = cmd_q->cmd_error;
1465 goto e_dst;
1466 }
1467
1468 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1469 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1470 DES3_EDE_BLOCK_SIZE);
1471 }
1472e_dst:
1473 if (!in_place)
1474 ccp_free_data(&dst, cmd_q);
1475
1476e_src:
1477 ccp_free_data(&src, cmd_q);
1478
1479e_ctx:
1480 if (des3->mode != CCP_DES3_MODE_ECB)
1481 ccp_dm_free(&ctx);
1482
1483e_key:
1484 ccp_dm_free(&key);
1485
1486 return ret;
1487}
1488
1489static noinline_for_stack int
1490ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1491{
1492 struct ccp_sha_engine *sha = &cmd->u.sha;
1493 struct ccp_dm_workarea ctx;
1494 struct ccp_data src;
1495 struct ccp_op op;
1496 unsigned int ioffset, ooffset;
1497 unsigned int digest_size;
1498 int sb_count;
1499 const void *init;
1500 u64 block_size;
1501 int ctx_size;
1502 int ret;
1503
1504 switch (sha->type) {
1505 case CCP_SHA_TYPE_1:
1506 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1507 return -EINVAL;
1508 block_size = SHA1_BLOCK_SIZE;
1509 break;
1510 case CCP_SHA_TYPE_224:
1511 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1512 return -EINVAL;
1513 block_size = SHA224_BLOCK_SIZE;
1514 break;
1515 case CCP_SHA_TYPE_256:
1516 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1517 return -EINVAL;
1518 block_size = SHA256_BLOCK_SIZE;
1519 break;
1520 case CCP_SHA_TYPE_384:
1521 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1522 || sha->ctx_len < SHA384_DIGEST_SIZE)
1523 return -EINVAL;
1524 block_size = SHA384_BLOCK_SIZE;
1525 break;
1526 case CCP_SHA_TYPE_512:
1527 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1528 || sha->ctx_len < SHA512_DIGEST_SIZE)
1529 return -EINVAL;
1530 block_size = SHA512_BLOCK_SIZE;
1531 break;
1532 default:
1533 return -EINVAL;
1534 }
1535
1536 if (!sha->ctx)
1537 return -EINVAL;
1538
1539 if (!sha->final && (sha->src_len & (block_size - 1)))
1540 return -EINVAL;
1541
1542 /* The version 3 device can't handle zero-length input */
1543 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1544
1545 if (!sha->src_len) {
1546 unsigned int digest_len;
1547 const u8 *sha_zero;
1548
1549 /* Not final, just return */
1550 if (!sha->final)
1551 return 0;
1552
1553 /* CCP can't do a zero length sha operation so the
1554 * caller must buffer the data.
1555 */
1556 if (sha->msg_bits)
1557 return -EINVAL;
1558
1559 /* The CCP cannot perform zero-length sha operations
1560 * so the caller is required to buffer data for the
1561 * final operation. However, a sha operation for a
1562 * message with a total length of zero is valid so
1563 * known values are required to supply the result.
1564 */
1565 switch (sha->type) {
1566 case CCP_SHA_TYPE_1:
1567 sha_zero = sha1_zero_message_hash;
1568 digest_len = SHA1_DIGEST_SIZE;
1569 break;
1570 case CCP_SHA_TYPE_224:
1571 sha_zero = sha224_zero_message_hash;
1572 digest_len = SHA224_DIGEST_SIZE;
1573 break;
1574 case CCP_SHA_TYPE_256:
1575 sha_zero = sha256_zero_message_hash;
1576 digest_len = SHA256_DIGEST_SIZE;
1577 break;
1578 default:
1579 return -EINVAL;
1580 }
1581
1582 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1583 digest_len, 1);
1584
1585 return 0;
1586 }
1587 }
1588
1589 /* Set variables used throughout */
1590 switch (sha->type) {
1591 case CCP_SHA_TYPE_1:
1592 digest_size = SHA1_DIGEST_SIZE;
1593 init = (void *) ccp_sha1_init;
1594 ctx_size = SHA1_DIGEST_SIZE;
1595 sb_count = 1;
1596 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1597 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1598 else
1599 ooffset = ioffset = 0;
1600 break;
1601 case CCP_SHA_TYPE_224:
1602 digest_size = SHA224_DIGEST_SIZE;
1603 init = (void *) ccp_sha224_init;
1604 ctx_size = SHA256_DIGEST_SIZE;
1605 sb_count = 1;
1606 ioffset = 0;
1607 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1608 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1609 else
1610 ooffset = 0;
1611 break;
1612 case CCP_SHA_TYPE_256:
1613 digest_size = SHA256_DIGEST_SIZE;
1614 init = (void *) ccp_sha256_init;
1615 ctx_size = SHA256_DIGEST_SIZE;
1616 sb_count = 1;
1617 ooffset = ioffset = 0;
1618 break;
1619 case CCP_SHA_TYPE_384:
1620 digest_size = SHA384_DIGEST_SIZE;
1621 init = (void *) ccp_sha384_init;
1622 ctx_size = SHA512_DIGEST_SIZE;
1623 sb_count = 2;
1624 ioffset = 0;
1625 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1626 break;
1627 case CCP_SHA_TYPE_512:
1628 digest_size = SHA512_DIGEST_SIZE;
1629 init = (void *) ccp_sha512_init;
1630 ctx_size = SHA512_DIGEST_SIZE;
1631 sb_count = 2;
1632 ooffset = ioffset = 0;
1633 break;
1634 default:
1635 ret = -EINVAL;
1636 goto e_data;
1637 }
1638
1639 /* For zero-length plaintext the src pointer is ignored;
1640 * otherwise both parts must be valid
1641 */
1642 if (sha->src_len && !sha->src)
1643 return -EINVAL;
1644
1645 memset(&op, 0, sizeof(op));
1646 op.cmd_q = cmd_q;
1647 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1648 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1649 op.u.sha.type = sha->type;
1650 op.u.sha.msg_bits = sha->msg_bits;
1651
1652 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1653 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1654 * first slot, and the left half in the second. Each portion must then
1655 * be in little endian format: use the 256-bit byte swap option.
1656 */
1657 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1658 DMA_BIDIRECTIONAL);
1659 if (ret)
1660 return ret;
1661 if (sha->first) {
1662 switch (sha->type) {
1663 case CCP_SHA_TYPE_1:
1664 case CCP_SHA_TYPE_224:
1665 case CCP_SHA_TYPE_256:
1666 memcpy(ctx.address + ioffset, init, ctx_size);
1667 break;
1668 case CCP_SHA_TYPE_384:
1669 case CCP_SHA_TYPE_512:
1670 memcpy(ctx.address + ctx_size / 2, init,
1671 ctx_size / 2);
1672 memcpy(ctx.address, init + ctx_size / 2,
1673 ctx_size / 2);
1674 break;
1675 default:
1676 ret = -EINVAL;
1677 goto e_ctx;
1678 }
1679 } else {
1680 /* Restore the context */
1681 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1682 sb_count * CCP_SB_BYTES);
1683 if (ret)
1684 goto e_ctx;
1685 }
1686
1687 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1688 CCP_PASSTHRU_BYTESWAP_256BIT);
1689 if (ret) {
1690 cmd->engine_error = cmd_q->cmd_error;
1691 goto e_ctx;
1692 }
1693
1694 if (sha->src) {
1695 /* Send data to the CCP SHA engine; block_size is set above */
1696 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1697 block_size, DMA_TO_DEVICE);
1698 if (ret)
1699 goto e_ctx;
1700
1701 while (src.sg_wa.bytes_left) {
1702 ccp_prepare_data(&src, NULL, &op, block_size, false);
1703 if (sha->final && !src.sg_wa.bytes_left)
1704 op.eom = 1;
1705
1706 ret = cmd_q->ccp->vdata->perform->sha(&op);
1707 if (ret) {
1708 cmd->engine_error = cmd_q->cmd_error;
1709 goto e_data;
1710 }
1711
1712 ccp_process_data(&src, NULL, &op);
1713 }
1714 } else {
1715 op.eom = 1;
1716 ret = cmd_q->ccp->vdata->perform->sha(&op);
1717 if (ret) {
1718 cmd->engine_error = cmd_q->cmd_error;
1719 goto e_data;
1720 }
1721 }
1722
1723 /* Retrieve the SHA context - convert from LE to BE using
1724 * 32-byte (256-bit) byteswapping to BE
1725 */
1726 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1727 CCP_PASSTHRU_BYTESWAP_256BIT);
1728 if (ret) {
1729 cmd->engine_error = cmd_q->cmd_error;
1730 goto e_data;
1731 }
1732
1733 if (sha->final) {
1734 /* Finishing up, so get the digest */
1735 switch (sha->type) {
1736 case CCP_SHA_TYPE_1:
1737 case CCP_SHA_TYPE_224:
1738 case CCP_SHA_TYPE_256:
1739 ccp_get_dm_area(&ctx, ooffset,
1740 sha->ctx, 0,
1741 digest_size);
1742 break;
1743 case CCP_SHA_TYPE_384:
1744 case CCP_SHA_TYPE_512:
1745 ccp_get_dm_area(&ctx, 0,
1746 sha->ctx, LSB_ITEM_SIZE - ooffset,
1747 LSB_ITEM_SIZE);
1748 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1749 sha->ctx, 0,
1750 LSB_ITEM_SIZE - ooffset);
1751 break;
1752 default:
1753 ret = -EINVAL;
1754 goto e_data;
1755 }
1756 } else {
1757 /* Stash the context */
1758 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1759 sb_count * CCP_SB_BYTES);
1760 }
1761
1762 if (sha->final && sha->opad) {
1763 /* HMAC operation, recursively perform final SHA */
1764 struct ccp_cmd hmac_cmd;
1765 struct scatterlist sg;
1766 u8 *hmac_buf;
1767
1768 if (sha->opad_len != block_size) {
1769 ret = -EINVAL;
1770 goto e_data;
1771 }
1772
1773 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1774 if (!hmac_buf) {
1775 ret = -ENOMEM;
1776 goto e_data;
1777 }
1778 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1779
1780 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1781 switch (sha->type) {
1782 case CCP_SHA_TYPE_1:
1783 case CCP_SHA_TYPE_224:
1784 case CCP_SHA_TYPE_256:
1785 memcpy(hmac_buf + block_size,
1786 ctx.address + ooffset,
1787 digest_size);
1788 break;
1789 case CCP_SHA_TYPE_384:
1790 case CCP_SHA_TYPE_512:
1791 memcpy(hmac_buf + block_size,
1792 ctx.address + LSB_ITEM_SIZE + ooffset,
1793 LSB_ITEM_SIZE);
1794 memcpy(hmac_buf + block_size +
1795 (LSB_ITEM_SIZE - ooffset),
1796 ctx.address,
1797 LSB_ITEM_SIZE);
1798 break;
1799 default:
1800 kfree(hmac_buf);
1801 ret = -EINVAL;
1802 goto e_data;
1803 }
1804
1805 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1806 hmac_cmd.engine = CCP_ENGINE_SHA;
1807 hmac_cmd.u.sha.type = sha->type;
1808 hmac_cmd.u.sha.ctx = sha->ctx;
1809 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1810 hmac_cmd.u.sha.src = &sg;
1811 hmac_cmd.u.sha.src_len = block_size + digest_size;
1812 hmac_cmd.u.sha.opad = NULL;
1813 hmac_cmd.u.sha.opad_len = 0;
1814 hmac_cmd.u.sha.first = 1;
1815 hmac_cmd.u.sha.final = 1;
1816 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1817
1818 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1819 if (ret)
1820 cmd->engine_error = hmac_cmd.engine_error;
1821
1822 kfree(hmac_buf);
1823 }
1824
1825e_data:
1826 if (sha->src)
1827 ccp_free_data(&src, cmd_q);
1828
1829e_ctx:
1830 ccp_dm_free(&ctx);
1831
1832 return ret;
1833}
1834
1835static noinline_for_stack int
1836ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1837{
1838 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1839 struct ccp_dm_workarea exp, src, dst;
1840 struct ccp_op op;
1841 unsigned int sb_count, i_len, o_len;
1842 int ret;
1843
1844 /* Check against the maximum allowable size, in bits */
1845 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1846 return -EINVAL;
1847
1848 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1849 return -EINVAL;
1850
1851 memset(&op, 0, sizeof(op));
1852 op.cmd_q = cmd_q;
1853 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1854
1855 /* The RSA modulus must precede the message being acted upon, so
1856 * it must be copied to a DMA area where the message and the
1857 * modulus can be concatenated. Therefore the input buffer
1858 * length required is twice the output buffer length (which
1859 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1860 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1861 * required.
1862 */
1863 o_len = 32 * ((rsa->key_size + 255) / 256);
1864 i_len = o_len * 2;
1865
1866 sb_count = 0;
1867 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1868 /* sb_count is the number of storage block slots required
1869 * for the modulus.
1870 */
1871 sb_count = o_len / CCP_SB_BYTES;
1872 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1873 sb_count);
1874 if (!op.sb_key)
1875 return -EIO;
1876 } else {
1877 /* A version 5 device allows a modulus size that will not fit
1878 * in the LSB, so the command will transfer it from memory.
1879 * Set the sb key to the default, even though it's not used.
1880 */
1881 op.sb_key = cmd_q->sb_key;
1882 }
1883
1884 /* The RSA exponent must be in little endian format. Reverse its
1885 * byte order.
1886 */
1887 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1888 if (ret)
1889 goto e_sb;
1890
1891 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1892 if (ret)
1893 goto e_exp;
1894
1895 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1896 /* Copy the exponent to the local storage block, using
1897 * as many 32-byte blocks as were allocated above. It's
1898 * already little endian, so no further change is required.
1899 */
1900 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1901 CCP_PASSTHRU_BYTESWAP_NOOP);
1902 if (ret) {
1903 cmd->engine_error = cmd_q->cmd_error;
1904 goto e_exp;
1905 }
1906 } else {
1907 /* The exponent can be retrieved from memory via DMA. */
1908 op.exp.u.dma.address = exp.dma.address;
1909 op.exp.u.dma.offset = 0;
1910 }
1911
1912 /* Concatenate the modulus and the message. Both the modulus and
1913 * the operands must be in little endian format. Since the input
1914 * is in big endian format it must be converted.
1915 */
1916 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1917 if (ret)
1918 goto e_exp;
1919
1920 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1921 if (ret)
1922 goto e_src;
1923 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1924 if (ret)
1925 goto e_src;
1926
1927 /* Prepare the output area for the operation */
1928 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1929 if (ret)
1930 goto e_src;
1931
1932 op.soc = 1;
1933 op.src.u.dma.address = src.dma.address;
1934 op.src.u.dma.offset = 0;
1935 op.src.u.dma.length = i_len;
1936 op.dst.u.dma.address = dst.dma.address;
1937 op.dst.u.dma.offset = 0;
1938 op.dst.u.dma.length = o_len;
1939
1940 op.u.rsa.mod_size = rsa->key_size;
1941 op.u.rsa.input_len = i_len;
1942
1943 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1944 if (ret) {
1945 cmd->engine_error = cmd_q->cmd_error;
1946 goto e_dst;
1947 }
1948
1949 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1950
1951e_dst:
1952 ccp_dm_free(&dst);
1953
1954e_src:
1955 ccp_dm_free(&src);
1956
1957e_exp:
1958 ccp_dm_free(&exp);
1959
1960e_sb:
1961 if (sb_count)
1962 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1963
1964 return ret;
1965}
1966
1967static noinline_for_stack int
1968ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1969{
1970 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1971 struct ccp_dm_workarea mask;
1972 struct ccp_data src, dst;
1973 struct ccp_op op;
1974 bool in_place = false;
1975 unsigned int i;
1976 int ret = 0;
1977
1978 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1979 return -EINVAL;
1980
1981 if (!pt->src || !pt->dst)
1982 return -EINVAL;
1983
1984 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1985 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1986 return -EINVAL;
1987 if (!pt->mask)
1988 return -EINVAL;
1989 }
1990
1991 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1992
1993 memset(&op, 0, sizeof(op));
1994 op.cmd_q = cmd_q;
1995 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1996
1997 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1998 /* Load the mask */
1999 op.sb_key = cmd_q->sb_key;
2000
2001 ret = ccp_init_dm_workarea(&mask, cmd_q,
2002 CCP_PASSTHRU_SB_COUNT *
2003 CCP_SB_BYTES,
2004 DMA_TO_DEVICE);
2005 if (ret)
2006 return ret;
2007
2008 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2009 if (ret)
2010 goto e_mask;
2011 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2012 CCP_PASSTHRU_BYTESWAP_NOOP);
2013 if (ret) {
2014 cmd->engine_error = cmd_q->cmd_error;
2015 goto e_mask;
2016 }
2017 }
2018
2019 /* Prepare the input and output data workareas. For in-place
2020 * operations we need to set the dma direction to BIDIRECTIONAL
2021 * and copy the src workarea to the dst workarea.
2022 */
2023 if (sg_virt(pt->src) == sg_virt(pt->dst))
2024 in_place = true;
2025
2026 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2027 CCP_PASSTHRU_MASKSIZE,
2028 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2029 if (ret)
2030 goto e_mask;
2031
2032 if (in_place) {
2033 dst = src;
2034 } else {
2035 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2036 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2037 if (ret)
2038 goto e_src;
2039 }
2040
2041 /* Send data to the CCP Passthru engine
2042 * Because the CCP engine works on a single source and destination
2043 * dma address at a time, each entry in the source scatterlist
2044 * (after the dma_map_sg call) must be less than or equal to the
2045 * (remaining) length in the destination scatterlist entry and the
2046 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2047 */
2048 dst.sg_wa.sg_used = 0;
2049 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2050 if (!dst.sg_wa.sg ||
2051 (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2052 ret = -EINVAL;
2053 goto e_dst;
2054 }
2055
2056 if (i == src.sg_wa.dma_count) {
2057 op.eom = 1;
2058 op.soc = 1;
2059 }
2060
2061 op.src.type = CCP_MEMTYPE_SYSTEM;
2062 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2063 op.src.u.dma.offset = 0;
2064 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2065
2066 op.dst.type = CCP_MEMTYPE_SYSTEM;
2067 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2068 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2069 op.dst.u.dma.length = op.src.u.dma.length;
2070
2071 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2072 if (ret) {
2073 cmd->engine_error = cmd_q->cmd_error;
2074 goto e_dst;
2075 }
2076
2077 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2078 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2079 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2080 dst.sg_wa.sg_used = 0;
2081 }
2082 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2083 }
2084
2085e_dst:
2086 if (!in_place)
2087 ccp_free_data(&dst, cmd_q);
2088
2089e_src:
2090 ccp_free_data(&src, cmd_q);
2091
2092e_mask:
2093 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2094 ccp_dm_free(&mask);
2095
2096 return ret;
2097}
2098
2099static noinline_for_stack int
2100ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2101 struct ccp_cmd *cmd)
2102{
2103 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2104 struct ccp_dm_workarea mask;
2105 struct ccp_op op;
2106 int ret;
2107
2108 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2109 return -EINVAL;
2110
2111 if (!pt->src_dma || !pt->dst_dma)
2112 return -EINVAL;
2113
2114 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2115 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2116 return -EINVAL;
2117 if (!pt->mask)
2118 return -EINVAL;
2119 }
2120
2121 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2122
2123 memset(&op, 0, sizeof(op));
2124 op.cmd_q = cmd_q;
2125 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2126
2127 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2128 /* Load the mask */
2129 op.sb_key = cmd_q->sb_key;
2130
2131 mask.length = pt->mask_len;
2132 mask.dma.address = pt->mask;
2133 mask.dma.length = pt->mask_len;
2134
2135 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2136 CCP_PASSTHRU_BYTESWAP_NOOP);
2137 if (ret) {
2138 cmd->engine_error = cmd_q->cmd_error;
2139 return ret;
2140 }
2141 }
2142
2143 /* Send data to the CCP Passthru engine */
2144 op.eom = 1;
2145 op.soc = 1;
2146
2147 op.src.type = CCP_MEMTYPE_SYSTEM;
2148 op.src.u.dma.address = pt->src_dma;
2149 op.src.u.dma.offset = 0;
2150 op.src.u.dma.length = pt->src_len;
2151
2152 op.dst.type = CCP_MEMTYPE_SYSTEM;
2153 op.dst.u.dma.address = pt->dst_dma;
2154 op.dst.u.dma.offset = 0;
2155 op.dst.u.dma.length = pt->src_len;
2156
2157 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2158 if (ret)
2159 cmd->engine_error = cmd_q->cmd_error;
2160
2161 return ret;
2162}
2163
2164static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2165{
2166 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2167 struct ccp_dm_workarea src, dst;
2168 struct ccp_op op;
2169 int ret;
2170 u8 *save;
2171
2172 if (!ecc->u.mm.operand_1 ||
2173 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2174 return -EINVAL;
2175
2176 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2177 if (!ecc->u.mm.operand_2 ||
2178 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2179 return -EINVAL;
2180
2181 if (!ecc->u.mm.result ||
2182 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2183 return -EINVAL;
2184
2185 memset(&op, 0, sizeof(op));
2186 op.cmd_q = cmd_q;
2187 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2188
2189 /* Concatenate the modulus and the operands. Both the modulus and
2190 * the operands must be in little endian format. Since the input
2191 * is in big endian format it must be converted and placed in a
2192 * fixed length buffer.
2193 */
2194 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2195 DMA_TO_DEVICE);
2196 if (ret)
2197 return ret;
2198
2199 /* Save the workarea address since it is updated in order to perform
2200 * the concatenation
2201 */
2202 save = src.address;
2203
2204 /* Copy the ECC modulus */
2205 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2206 if (ret)
2207 goto e_src;
2208 src.address += CCP_ECC_OPERAND_SIZE;
2209
2210 /* Copy the first operand */
2211 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2212 ecc->u.mm.operand_1_len);
2213 if (ret)
2214 goto e_src;
2215 src.address += CCP_ECC_OPERAND_SIZE;
2216
2217 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2218 /* Copy the second operand */
2219 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2220 ecc->u.mm.operand_2_len);
2221 if (ret)
2222 goto e_src;
2223 src.address += CCP_ECC_OPERAND_SIZE;
2224 }
2225
2226 /* Restore the workarea address */
2227 src.address = save;
2228
2229 /* Prepare the output area for the operation */
2230 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2231 DMA_FROM_DEVICE);
2232 if (ret)
2233 goto e_src;
2234
2235 op.soc = 1;
2236 op.src.u.dma.address = src.dma.address;
2237 op.src.u.dma.offset = 0;
2238 op.src.u.dma.length = src.length;
2239 op.dst.u.dma.address = dst.dma.address;
2240 op.dst.u.dma.offset = 0;
2241 op.dst.u.dma.length = dst.length;
2242
2243 op.u.ecc.function = cmd->u.ecc.function;
2244
2245 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2246 if (ret) {
2247 cmd->engine_error = cmd_q->cmd_error;
2248 goto e_dst;
2249 }
2250
2251 ecc->ecc_result = le16_to_cpup(
2252 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2253 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2254 ret = -EIO;
2255 goto e_dst;
2256 }
2257
2258 /* Save the ECC result */
2259 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2260 CCP_ECC_MODULUS_BYTES);
2261
2262e_dst:
2263 ccp_dm_free(&dst);
2264
2265e_src:
2266 ccp_dm_free(&src);
2267
2268 return ret;
2269}
2270
2271static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2272{
2273 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2274 struct ccp_dm_workarea src, dst;
2275 struct ccp_op op;
2276 int ret;
2277 u8 *save;
2278
2279 if (!ecc->u.pm.point_1.x ||
2280 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2281 !ecc->u.pm.point_1.y ||
2282 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2283 return -EINVAL;
2284
2285 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2286 if (!ecc->u.pm.point_2.x ||
2287 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2288 !ecc->u.pm.point_2.y ||
2289 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2290 return -EINVAL;
2291 } else {
2292 if (!ecc->u.pm.domain_a ||
2293 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2294 return -EINVAL;
2295
2296 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2297 if (!ecc->u.pm.scalar ||
2298 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2299 return -EINVAL;
2300 }
2301
2302 if (!ecc->u.pm.result.x ||
2303 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2304 !ecc->u.pm.result.y ||
2305 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2306 return -EINVAL;
2307
2308 memset(&op, 0, sizeof(op));
2309 op.cmd_q = cmd_q;
2310 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2311
2312 /* Concatenate the modulus and the operands. Both the modulus and
2313 * the operands must be in little endian format. Since the input
2314 * is in big endian format it must be converted and placed in a
2315 * fixed length buffer.
2316 */
2317 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2318 DMA_TO_DEVICE);
2319 if (ret)
2320 return ret;
2321
2322 /* Save the workarea address since it is updated in order to perform
2323 * the concatenation
2324 */
2325 save = src.address;
2326
2327 /* Copy the ECC modulus */
2328 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2329 if (ret)
2330 goto e_src;
2331 src.address += CCP_ECC_OPERAND_SIZE;
2332
2333 /* Copy the first point X and Y coordinate */
2334 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2335 ecc->u.pm.point_1.x_len);
2336 if (ret)
2337 goto e_src;
2338 src.address += CCP_ECC_OPERAND_SIZE;
2339 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2340 ecc->u.pm.point_1.y_len);
2341 if (ret)
2342 goto e_src;
2343 src.address += CCP_ECC_OPERAND_SIZE;
2344
2345 /* Set the first point Z coordinate to 1 */
2346 *src.address = 0x01;
2347 src.address += CCP_ECC_OPERAND_SIZE;
2348
2349 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2350 /* Copy the second point X and Y coordinate */
2351 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2352 ecc->u.pm.point_2.x_len);
2353 if (ret)
2354 goto e_src;
2355 src.address += CCP_ECC_OPERAND_SIZE;
2356 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2357 ecc->u.pm.point_2.y_len);
2358 if (ret)
2359 goto e_src;
2360 src.address += CCP_ECC_OPERAND_SIZE;
2361
2362 /* Set the second point Z coordinate to 1 */
2363 *src.address = 0x01;
2364 src.address += CCP_ECC_OPERAND_SIZE;
2365 } else {
2366 /* Copy the Domain "a" parameter */
2367 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2368 ecc->u.pm.domain_a_len);
2369 if (ret)
2370 goto e_src;
2371 src.address += CCP_ECC_OPERAND_SIZE;
2372
2373 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2374 /* Copy the scalar value */
2375 ret = ccp_reverse_set_dm_area(&src, 0,
2376 ecc->u.pm.scalar, 0,
2377 ecc->u.pm.scalar_len);
2378 if (ret)
2379 goto e_src;
2380 src.address += CCP_ECC_OPERAND_SIZE;
2381 }
2382 }
2383
2384 /* Restore the workarea address */
2385 src.address = save;
2386
2387 /* Prepare the output area for the operation */
2388 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2389 DMA_FROM_DEVICE);
2390 if (ret)
2391 goto e_src;
2392
2393 op.soc = 1;
2394 op.src.u.dma.address = src.dma.address;
2395 op.src.u.dma.offset = 0;
2396 op.src.u.dma.length = src.length;
2397 op.dst.u.dma.address = dst.dma.address;
2398 op.dst.u.dma.offset = 0;
2399 op.dst.u.dma.length = dst.length;
2400
2401 op.u.ecc.function = cmd->u.ecc.function;
2402
2403 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2404 if (ret) {
2405 cmd->engine_error = cmd_q->cmd_error;
2406 goto e_dst;
2407 }
2408
2409 ecc->ecc_result = le16_to_cpup(
2410 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2411 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2412 ret = -EIO;
2413 goto e_dst;
2414 }
2415
2416 /* Save the workarea address since it is updated as we walk through
2417 * to copy the point math result
2418 */
2419 save = dst.address;
2420
2421 /* Save the ECC result X and Y coordinates */
2422 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2423 CCP_ECC_MODULUS_BYTES);
2424 dst.address += CCP_ECC_OUTPUT_SIZE;
2425 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2426 CCP_ECC_MODULUS_BYTES);
2427
2428 /* Restore the workarea address */
2429 dst.address = save;
2430
2431e_dst:
2432 ccp_dm_free(&dst);
2433
2434e_src:
2435 ccp_dm_free(&src);
2436
2437 return ret;
2438}
2439
2440static noinline_for_stack int
2441ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2442{
2443 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2444
2445 ecc->ecc_result = 0;
2446
2447 if (!ecc->mod ||
2448 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2449 return -EINVAL;
2450
2451 switch (ecc->function) {
2452 case CCP_ECC_FUNCTION_MMUL_384BIT:
2453 case CCP_ECC_FUNCTION_MADD_384BIT:
2454 case CCP_ECC_FUNCTION_MINV_384BIT:
2455 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2456
2457 case CCP_ECC_FUNCTION_PADD_384BIT:
2458 case CCP_ECC_FUNCTION_PMUL_384BIT:
2459 case CCP_ECC_FUNCTION_PDBL_384BIT:
2460 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2461
2462 default:
2463 return -EINVAL;
2464 }
2465}
2466
2467int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2468{
2469 int ret;
2470
2471 cmd->engine_error = 0;
2472 cmd_q->cmd_error = 0;
2473 cmd_q->int_rcvd = 0;
2474 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2475
2476 switch (cmd->engine) {
2477 case CCP_ENGINE_AES:
2478 switch (cmd->u.aes.mode) {
2479 case CCP_AES_MODE_CMAC:
2480 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2481 break;
2482 case CCP_AES_MODE_GCM:
2483 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2484 break;
2485 default:
2486 ret = ccp_run_aes_cmd(cmd_q, cmd);
2487 break;
2488 }
2489 break;
2490 case CCP_ENGINE_XTS_AES_128:
2491 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2492 break;
2493 case CCP_ENGINE_DES3:
2494 ret = ccp_run_des3_cmd(cmd_q, cmd);
2495 break;
2496 case CCP_ENGINE_SHA:
2497 ret = ccp_run_sha_cmd(cmd_q, cmd);
2498 break;
2499 case CCP_ENGINE_RSA:
2500 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2501 break;
2502 case CCP_ENGINE_PASSTHRU:
2503 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2504 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2505 else
2506 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2507 break;
2508 case CCP_ENGINE_ECC:
2509 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2510 break;
2511 default:
2512 ret = -EINVAL;
2513 }
2514
2515 return ret;
2516}
Linux 6.x block layer and io_uring tree(s)