projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'kvm-s390-master-4.5-1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / drivers / crypto / ccp / ccp-ops.c
CommitLineData
63b94509
TL		 1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/pci.h>
16#include <linux/pci_ids.h>
17#include <linux/kthread.h>
18#include <linux/sched.h>
19#include <linux/interrupt.h>
20#include <linux/spinlock.h>
21#include <linux/mutex.h>
22#include <linux/delay.h>
23#include <linux/ccp.h>
24#include <linux/scatterlist.h>
25#include <crypto/scatterwalk.h>
c11baa02 26#include <crypto/sha.h>
63b94509
TL		 27
28#include "ccp-dev.h"
29
63b94509
TL		 30enum ccp_memtype {
31 CCP_MEMTYPE_SYSTEM = 0,
32 CCP_MEMTYPE_KSB,
33 CCP_MEMTYPE_LOCAL,
34 CCP_MEMTYPE__LAST,
35};
36
37struct ccp_dma_info {
38 dma_addr_t address;
39 unsigned int offset;
40 unsigned int length;
41 enum dma_data_direction dir;
42};
43
44struct ccp_dm_workarea {
45 struct device *dev;
46 struct dma_pool *dma_pool;
47 unsigned int length;
48
49 u8 *address;
50 struct ccp_dma_info dma;
51};
52
53struct ccp_sg_workarea {
54 struct scatterlist *sg;
fb43f694 55 int nents;
63b94509
TL		 56
57 struct scatterlist *dma_sg;
58 struct device *dma_dev;
59 unsigned int dma_count;
60 enum dma_data_direction dma_dir;
61
81a59f00 62 unsigned int sg_used;
63b94509 63
81a59f00 64 u64 bytes_left;
63b94509
TL		 65};
66
67struct ccp_data {
68 struct ccp_sg_workarea sg_wa;
69 struct ccp_dm_workarea dm_wa;
70};
71
72struct ccp_mem {
73 enum ccp_memtype type;
74 union {
75 struct ccp_dma_info dma;
76 u32 ksb;
77 } u;
78};
79
80struct ccp_aes_op {
81 enum ccp_aes_type type;
82 enum ccp_aes_mode mode;
83 enum ccp_aes_action action;
84};
85
86struct ccp_xts_aes_op {
87 enum ccp_aes_action action;
88 enum ccp_xts_aes_unit_size unit_size;
89};
90
91struct ccp_sha_op {
92 enum ccp_sha_type type;
93 u64 msg_bits;
94};
95
96struct ccp_rsa_op {
97 u32 mod_size;
98 u32 input_len;
99};
100
101struct ccp_passthru_op {
102 enum ccp_passthru_bitwise bit_mod;
103 enum ccp_passthru_byteswap byte_swap;
104};
105
106struct ccp_ecc_op {
107 enum ccp_ecc_function function;
108};
109
110struct ccp_op {
111 struct ccp_cmd_queue *cmd_q;
112
113 u32 jobid;
114 u32 ioc;
115 u32 soc;
116 u32 ksb_key;
117 u32 ksb_ctx;
118 u32 init;
119 u32 eom;
120
121 struct ccp_mem src;
122 struct ccp_mem dst;
123
124 union {
125 struct ccp_aes_op aes;
126 struct ccp_xts_aes_op xts;
127 struct ccp_sha_op sha;
128 struct ccp_rsa_op rsa;
129 struct ccp_passthru_op passthru;
130 struct ccp_ecc_op ecc;
131 } u;
132};
133
c11baa02
TL		 134/* SHA initial context values */
135static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
136 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
137 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
138 cpu_to_be32(SHA1_H4), 0, 0, 0,
139};
140
141static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
142 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
143 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
144 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
145 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
146};
147
148static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
149 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
150 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
151 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
152 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
153};
154
63b94509
TL		 155static u32 ccp_addr_lo(struct ccp_dma_info *info)
156{
157 return lower_32_bits(info->address + info->offset);
158}
159
160static u32 ccp_addr_hi(struct ccp_dma_info *info)
161{
162 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
163}
164
165static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
166{
167 struct ccp_cmd_queue *cmd_q = op->cmd_q;
168 struct ccp_device *ccp = cmd_q->ccp;
169 void __iomem *cr_addr;
170 u32 cr0, cmd;
171 unsigned int i;
172 int ret = 0;
173
174 /* We could read a status register to see how many free slots
175 * are actually available, but reading that register resets it
176 * and you could lose some error information.
177 */
178 cmd_q->free_slots--;
179
180 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
181 | (op->jobid << REQ0_JOBID_SHIFT)
182 | REQ0_WAIT_FOR_WRITE;
183
184 if (op->soc)
185 cr0 |= REQ0_STOP_ON_COMPLETE
186 | REQ0_INT_ON_COMPLETE;
187
188 if (op->ioc || !cmd_q->free_slots)
189 cr0 |= REQ0_INT_ON_COMPLETE;
190
191 /* Start at CMD_REQ1 */
192 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
193
194 mutex_lock(&ccp->req_mutex);
195
196 /* Write CMD_REQ1 through CMD_REQx first */
197 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
198 iowrite32(*(cr + i), cr_addr);
199
200 /* Tell the CCP to start */
201 wmb();
202 iowrite32(cr0, ccp->io_regs + CMD_REQ0);
203
204 mutex_unlock(&ccp->req_mutex);
205
206 if (cr0 & REQ0_INT_ON_COMPLETE) {
207 /* Wait for the job to complete */
208 ret = wait_event_interruptible(cmd_q->int_queue,
209 cmd_q->int_rcvd);
210 if (ret || cmd_q->cmd_error) {
211 /* On error delete all related jobs from the queue */
212 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
213 | op->jobid;
214
215 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
216
217 if (!ret)
218 ret = -EIO;
219 } else if (op->soc) {
220 /* Delete just head job from the queue on SoC */
221 cmd = DEL_Q_ACTIVE
222 | (cmd_q->id << DEL_Q_ID_SHIFT)
223 | op->jobid;
224
225 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
226 }
227
228 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
229
230 cmd_q->int_rcvd = 0;
231 }
232
233 return ret;
234}
235
236static int ccp_perform_aes(struct ccp_op *op)
237{
238 u32 cr[6];
239
240 /* Fill out the register contents for REQ1 through REQ6 */
241 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
242 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
243 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
244 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
245 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
246 cr[1] = op->src.u.dma.length - 1;
247 cr[2] = ccp_addr_lo(&op->src.u.dma);
248 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
249 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
250 | ccp_addr_hi(&op->src.u.dma);
251 cr[4] = ccp_addr_lo(&op->dst.u.dma);
252 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
253 | ccp_addr_hi(&op->dst.u.dma);
254
255 if (op->u.aes.mode == CCP_AES_MODE_CFB)
256 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
257
258 if (op->eom)
259 cr[0] |= REQ1_EOM;
260
261 if (op->init)
262 cr[0] |= REQ1_INIT;
263
264 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
265}
266
267static int ccp_perform_xts_aes(struct ccp_op *op)
268{
269 u32 cr[6];
270
271 /* Fill out the register contents for REQ1 through REQ6 */
272 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
273 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
274 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
275 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
276 cr[1] = op->src.u.dma.length - 1;
277 cr[2] = ccp_addr_lo(&op->src.u.dma);
278 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
279 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
280 | ccp_addr_hi(&op->src.u.dma);
281 cr[4] = ccp_addr_lo(&op->dst.u.dma);
282 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
283 | ccp_addr_hi(&op->dst.u.dma);
284
285 if (op->eom)
286 cr[0] |= REQ1_EOM;
287
288 if (op->init)
289 cr[0] |= REQ1_INIT;
290
291 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
292}
293
294static int ccp_perform_sha(struct ccp_op *op)
295{
296 u32 cr[6];
297
298 /* Fill out the register contents for REQ1 through REQ6 */
299 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
300 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
301 | REQ1_INIT;
302 cr[1] = op->src.u.dma.length - 1;
303 cr[2] = ccp_addr_lo(&op->src.u.dma);
304 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
305 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
306 | ccp_addr_hi(&op->src.u.dma);
307
308 if (op->eom) {
309 cr[0] |= REQ1_EOM;
310 cr[4] = lower_32_bits(op->u.sha.msg_bits);
311 cr[5] = upper_32_bits(op->u.sha.msg_bits);
312 } else {
313 cr[4] = 0;
314 cr[5] = 0;
315 }
316
317 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
318}
319
320static int ccp_perform_rsa(struct ccp_op *op)
321{
322 u32 cr[6];
323
324 /* Fill out the register contents for REQ1 through REQ6 */
325 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
326 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
327 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
328 | REQ1_EOM;
329 cr[1] = op->u.rsa.input_len - 1;
330 cr[2] = ccp_addr_lo(&op->src.u.dma);
331 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
332 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
333 | ccp_addr_hi(&op->src.u.dma);
334 cr[4] = ccp_addr_lo(&op->dst.u.dma);
335 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
336 | ccp_addr_hi(&op->dst.u.dma);
337
338 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
339}
340
341static int ccp_perform_passthru(struct ccp_op *op)
342{
343 u32 cr[6];
344
345 /* Fill out the register contents for REQ1 through REQ6 */
346 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
347 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
348 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
349
350 if (op->src.type == CCP_MEMTYPE_SYSTEM)
351 cr[1] = op->src.u.dma.length - 1;
352 else
353 cr[1] = op->dst.u.dma.length - 1;
354
355 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
356 cr[2] = ccp_addr_lo(&op->src.u.dma);
357 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
358 | ccp_addr_hi(&op->src.u.dma);
359
360 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
361 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
362 } else {
363 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
364 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
365 }
366
367 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
368 cr[4] = ccp_addr_lo(&op->dst.u.dma);
369 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
370 | ccp_addr_hi(&op->dst.u.dma);
371 } else {
372 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
373 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
374 }
375
376 if (op->eom)
377 cr[0] |= REQ1_EOM;
378
379 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
380}
381
382static int ccp_perform_ecc(struct ccp_op *op)
383{
384 u32 cr[6];
385
386 /* Fill out the register contents for REQ1 through REQ6 */
387 cr[0] = REQ1_ECC_AFFINE_CONVERT
388 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
389 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
390 | REQ1_EOM;
391 cr[1] = op->src.u.dma.length - 1;
392 cr[2] = ccp_addr_lo(&op->src.u.dma);
393 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
394 | ccp_addr_hi(&op->src.u.dma);
395 cr[4] = ccp_addr_lo(&op->dst.u.dma);
396 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
397 | ccp_addr_hi(&op->dst.u.dma);
398
399 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
400}
401
402static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
403{
404 int start;
405
406 for (;;) {
407 mutex_lock(&ccp->ksb_mutex);
408
409 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
410 ccp->ksb_count,
411 ccp->ksb_start,
412 count, 0);
413 if (start <= ccp->ksb_count) {
414 bitmap_set(ccp->ksb, start, count);
415
416 mutex_unlock(&ccp->ksb_mutex);
417 break;
418 }
419
420 ccp->ksb_avail = 0;
421
422 mutex_unlock(&ccp->ksb_mutex);
423
424 /* Wait for KSB entries to become available */
425 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
426 return 0;
427 }
428
429 return KSB_START + start;
430}
431
432static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
433 unsigned int count)
434{
435 if (!start)
436 return;
437
438 mutex_lock(&ccp->ksb_mutex);
439
440 bitmap_clear(ccp->ksb, start - KSB_START, count);
441
442 ccp->ksb_avail = 1;
443
444 mutex_unlock(&ccp->ksb_mutex);
445
446 wake_up_interruptible_all(&ccp->ksb_queue);
447}
448
449static u32 ccp_gen_jobid(struct ccp_device *ccp)
450{
451 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
452}
453
454static void ccp_sg_free(struct ccp_sg_workarea *wa)
455{
456 if (wa->dma_count)
457 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
458
459 wa->dma_count = 0;
460}
461
462static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
81a59f00 463 struct scatterlist *sg, u64 len,
63b94509
TL		 464 enum dma_data_direction dma_dir)
465{
466 memset(wa, 0, sizeof(*wa));
467
468 wa->sg = sg;
469 if (!sg)
470 return 0;
471
fb43f694
TL		 472 wa->nents = sg_nents_for_len(sg, len);
473 if (wa->nents < 0)
474 return wa->nents;
475
63b94509
TL		 476 wa->bytes_left = len;
477 wa->sg_used = 0;
478
479 if (len == 0)
480 return 0;
481
482 if (dma_dir == DMA_NONE)
483 return 0;
484
485 wa->dma_sg = sg;
486 wa->dma_dev = dev;
487 wa->dma_dir = dma_dir;
488 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
489 if (!wa->dma_count)
490 return -ENOMEM;
491
63b94509
TL		 492 return 0;
493}
494
495static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
496{
81a59f00 497 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
63b94509
TL		 498
499 if (!wa->sg)
500 return;
501
502 wa->sg_used += nbytes;
503 wa->bytes_left -= nbytes;
504 if (wa->sg_used == wa->sg->length) {
505 wa->sg = sg_next(wa->sg);
506 wa->sg_used = 0;
507 }
508}
509
510static void ccp_dm_free(struct ccp_dm_workarea *wa)
511{
512 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
513 if (wa->address)
514 dma_pool_free(wa->dma_pool, wa->address,
515 wa->dma.address);
516 } else {
517 if (wa->dma.address)
518 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
519 wa->dma.dir);
520 kfree(wa->address);
521 }
522
523 wa->address = NULL;
524 wa->dma.address = 0;
525}
526
527static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
528 struct ccp_cmd_queue *cmd_q,
529 unsigned int len,
530 enum dma_data_direction dir)
531{
532 memset(wa, 0, sizeof(*wa));
533
534 if (!len)
535 return 0;
536
537 wa->dev = cmd_q->ccp->dev;
538 wa->length = len;
539
540 if (len <= CCP_DMAPOOL_MAX_SIZE) {
541 wa->dma_pool = cmd_q->dma_pool;
542
543 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
544 &wa->dma.address);
545 if (!wa->address)
546 return -ENOMEM;
547
548 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
549
550 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
551 } else {
552 wa->address = kzalloc(len, GFP_KERNEL);
553 if (!wa->address)
554 return -ENOMEM;
555
556 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
557 dir);
558 if (!wa->dma.address)
559 return -ENOMEM;
560
561 wa->dma.length = len;
562 }
563 wa->dma.dir = dir;
564
565 return 0;
566}
567
568static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
569 struct scatterlist *sg, unsigned int sg_offset,
570 unsigned int len)
571{
572 WARN_ON(!wa->address);
573
574 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
575 0);
576}
577
578static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
579 struct scatterlist *sg, unsigned int sg_offset,
580 unsigned int len)
581{
582 WARN_ON(!wa->address);
583
584 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
585 1);
586}
587
355eba5d
TL		 588static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
589 struct scatterlist *sg,
590 unsigned int len, unsigned int se_len,
591 bool sign_extend)
63b94509
TL		 592{
593 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
594 u8 buffer[CCP_REVERSE_BUF_SIZE];
595
355eba5d
TL		 596 if (WARN_ON(se_len > sizeof(buffer)))
597 return -EINVAL;
63b94509
TL		 598
599 sg_offset = len;
600 dm_offset = 0;
601 nbytes = len;
602 while (nbytes) {
603 ksb_len = min_t(unsigned int, nbytes, se_len);
604 sg_offset -= ksb_len;
605
606 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
607 for (i = 0; i < ksb_len; i++)
608 wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
609
610 dm_offset += ksb_len;
611 nbytes -= ksb_len;
612
613 if ((ksb_len != se_len) && sign_extend) {
614 /* Must sign-extend to nearest sign-extend length */
615 if (wa->address[dm_offset - 1] & 0x80)
616 memset(wa->address + dm_offset, 0xff,
617 se_len - ksb_len);
618 }
619 }
355eba5d
TL		 620
621 return 0;
63b94509
TL		 622}
623
624static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
625 struct scatterlist *sg,
626 unsigned int len)
627{
628 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
629 u8 buffer[CCP_REVERSE_BUF_SIZE];
630
631 sg_offset = 0;
632 dm_offset = len;
633 nbytes = len;
634 while (nbytes) {
635 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
636 dm_offset -= ksb_len;
637
638 for (i = 0; i < ksb_len; i++)
639 buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
640 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
641
642 sg_offset += ksb_len;
643 nbytes -= ksb_len;
644 }
645}
646
647static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
648{
649 ccp_dm_free(&data->dm_wa);
650 ccp_sg_free(&data->sg_wa);
651}
652
653static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
81a59f00 654 struct scatterlist *sg, u64 sg_len,
63b94509
TL		 655 unsigned int dm_len,
656 enum dma_data_direction dir)
657{
658 int ret;
659
660 memset(data, 0, sizeof(*data));
661
662 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
663 dir);
664 if (ret)
665 goto e_err;
666
667 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
668 if (ret)
669 goto e_err;
670
671 return 0;
672
673e_err:
674 ccp_free_data(data, cmd_q);
675
676 return ret;
677}
678
679static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
680{
681 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
682 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
683 unsigned int buf_count, nbytes;
684
685 /* Clear the buffer if setting it */
686 if (!from)
687 memset(dm_wa->address, 0, dm_wa->length);
688
689 if (!sg_wa->sg)
690 return 0;
691
81a59f00
TL		 692 /* Perform the copy operation
693 * nbytes will always be <= UINT_MAX because dm_wa->length is
694 * an unsigned int
695 */
696 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
63b94509
TL		 697 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
698 nbytes, from);
699
700 /* Update the structures and generate the count */
701 buf_count = 0;
702 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
81a59f00
TL		 703 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
704 dm_wa->length - buf_count);
705 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
63b94509
TL		 706
707 buf_count += nbytes;
708 ccp_update_sg_workarea(sg_wa, nbytes);
709 }
710
711 return buf_count;
712}
713
714static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
715{
716 return ccp_queue_buf(data, 0);
717}
718
719static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
720{
721 return ccp_queue_buf(data, 1);
722}
723
724static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
725 struct ccp_op *op, unsigned int block_size,
726 bool blocksize_op)
727{
728 unsigned int sg_src_len, sg_dst_len, op_len;
729
730 /* The CCP can only DMA from/to one address each per operation. This
731 * requires that we find the smallest DMA area between the source
81a59f00
TL		 732 * and destination. The resulting len values will always be <= UINT_MAX
733 * because the dma length is an unsigned int.
63b94509 734 */
81a59f00
TL		 735 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
736 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
63b94509
TL		 737
738 if (dst) {
81a59f00
TL		 739 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
740 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
63b94509 741 op_len = min(sg_src_len, sg_dst_len);
8db88467 742 } else {
63b94509 743 op_len = sg_src_len;
8db88467 744 }
63b94509
TL		 745
746 /* The data operation length will be at least block_size in length
747 * or the smaller of available sg room remaining for the source or
748 * the destination
749 */
750 op_len = max(op_len, block_size);
751
752 /* Unless we have to buffer data, there's no reason to wait */
753 op->soc = 0;
754
755 if (sg_src_len < block_size) {
756 /* Not enough data in the sg element, so it
757 * needs to be buffered into a blocksize chunk
758 */
759 int cp_len = ccp_fill_queue_buf(src);
760
761 op->soc = 1;
762 op->src.u.dma.address = src->dm_wa.dma.address;
763 op->src.u.dma.offset = 0;
764 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
765 } else {
766 /* Enough data in the sg element, but we need to
767 * adjust for any previously copied data
768 */
769 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
770 op->src.u.dma.offset = src->sg_wa.sg_used;
771 op->src.u.dma.length = op_len & ~(block_size - 1);
772
773 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
774 }
775
776 if (dst) {
777 if (sg_dst_len < block_size) {
778 /* Not enough room in the sg element or we're on the
779 * last piece of data (when using padding), so the
780 * output needs to be buffered into a blocksize chunk
781 */
782 op->soc = 1;
783 op->dst.u.dma.address = dst->dm_wa.dma.address;
784 op->dst.u.dma.offset = 0;
785 op->dst.u.dma.length = op->src.u.dma.length;
786 } else {
787 /* Enough room in the sg element, but we need to
788 * adjust for any previously used area
789 */
790 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
791 op->dst.u.dma.offset = dst->sg_wa.sg_used;
792 op->dst.u.dma.length = op->src.u.dma.length;
793 }
794 }
795}
796
797static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
798 struct ccp_op *op)
799{
800 op->init = 0;
801
802 if (dst) {
803 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
804 ccp_empty_queue_buf(dst);
805 else
806 ccp_update_sg_workarea(&dst->sg_wa,
807 op->dst.u.dma.length);
808 }
809}
810
811static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
812 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
813 u32 byte_swap, bool from)
814{
815 struct ccp_op op;
816
817 memset(&op, 0, sizeof(op));
818
819 op.cmd_q = cmd_q;
820 op.jobid = jobid;
821 op.eom = 1;
822
823 if (from) {
824 op.soc = 1;
825 op.src.type = CCP_MEMTYPE_KSB;
826 op.src.u.ksb = ksb;
827 op.dst.type = CCP_MEMTYPE_SYSTEM;
828 op.dst.u.dma.address = wa->dma.address;
829 op.dst.u.dma.length = wa->length;
830 } else {
831 op.src.type = CCP_MEMTYPE_SYSTEM;
832 op.src.u.dma.address = wa->dma.address;
833 op.src.u.dma.length = wa->length;
834 op.dst.type = CCP_MEMTYPE_KSB;
835 op.dst.u.ksb = ksb;
836 }
837
838 op.u.passthru.byte_swap = byte_swap;
839
840 return ccp_perform_passthru(&op);
841}
842
843static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
844 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
845 u32 byte_swap)
846{
847 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
848}
849
850static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
851 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
852 u32 byte_swap)
853{
854 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
855}
856
857static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
858 struct ccp_cmd *cmd)
859{
860 struct ccp_aes_engine *aes = &cmd->u.aes;
861 struct ccp_dm_workarea key, ctx;
862 struct ccp_data src;
863 struct ccp_op op;
864 unsigned int dm_offset;
865 int ret;
866
867 if (!((aes->key_len == AES_KEYSIZE_128) ||
868 (aes->key_len == AES_KEYSIZE_192) ||
869 (aes->key_len == AES_KEYSIZE_256)))
870 return -EINVAL;
871
872 if (aes->src_len & (AES_BLOCK_SIZE - 1))
873 return -EINVAL;
874
875 if (aes->iv_len != AES_BLOCK_SIZE)
876 return -EINVAL;
877
878 if (!aes->key || !aes->iv || !aes->src)
879 return -EINVAL;
880
881 if (aes->cmac_final) {
882 if (aes->cmac_key_len != AES_BLOCK_SIZE)
883 return -EINVAL;
884
885 if (!aes->cmac_key)
886 return -EINVAL;
887 }
888
889 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
890 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
891
892 ret = -EIO;
893 memset(&op, 0, sizeof(op));
894 op.cmd_q = cmd_q;
895 op.jobid = ccp_gen_jobid(cmd_q->ccp);
896 op.ksb_key = cmd_q->ksb_key;
897 op.ksb_ctx = cmd_q->ksb_ctx;
898 op.init = 1;
899 op.u.aes.type = aes->type;
900 op.u.aes.mode = aes->mode;
901 op.u.aes.action = aes->action;
902
903 /* All supported key sizes fit in a single (32-byte) KSB entry
904 * and must be in little endian format. Use the 256-bit byte
905 * swap passthru option to convert from big endian to little
906 * endian.
907 */
908 ret = ccp_init_dm_workarea(&key, cmd_q,
909 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
910 DMA_TO_DEVICE);
911 if (ret)
912 return ret;
913
914 dm_offset = CCP_KSB_BYTES - aes->key_len;
915 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
916 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
917 CCP_PASSTHRU_BYTESWAP_256BIT);
918 if (ret) {
919 cmd->engine_error = cmd_q->cmd_error;
920 goto e_key;
921 }
922
923 /* The AES context fits in a single (32-byte) KSB entry and
924 * must be in little endian format. Use the 256-bit byte swap
925 * passthru option to convert from big endian to little endian.
926 */
927 ret = ccp_init_dm_workarea(&ctx, cmd_q,
928 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
929 DMA_BIDIRECTIONAL);
930 if (ret)
931 goto e_key;
932
933 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
934 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
935 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
936 CCP_PASSTHRU_BYTESWAP_256BIT);
937 if (ret) {
938 cmd->engine_error = cmd_q->cmd_error;
939 goto e_ctx;
940 }
941
942 /* Send data to the CCP AES engine */
943 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
944 AES_BLOCK_SIZE, DMA_TO_DEVICE);
945 if (ret)
946 goto e_ctx;
947
948 while (src.sg_wa.bytes_left) {
949 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
950 if (aes->cmac_final && !src.sg_wa.bytes_left) {
951 op.eom = 1;
952
953 /* Push the K1/K2 key to the CCP now */
954 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
955 op.ksb_ctx,
956 CCP_PASSTHRU_BYTESWAP_256BIT);
957 if (ret) {
958 cmd->engine_error = cmd_q->cmd_error;
959 goto e_src;
960 }
961
962 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
963 aes->cmac_key_len);
964 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
965 CCP_PASSTHRU_BYTESWAP_256BIT);
966 if (ret) {
967 cmd->engine_error = cmd_q->cmd_error;
968 goto e_src;
969 }
970 }
971
972 ret = ccp_perform_aes(&op);
973 if (ret) {
974 cmd->engine_error = cmd_q->cmd_error;
975 goto e_src;
976 }
977
978 ccp_process_data(&src, NULL, &op);
979 }
980
981 /* Retrieve the AES context - convert from LE to BE using
982 * 32-byte (256-bit) byteswapping
983 */
984 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
985 CCP_PASSTHRU_BYTESWAP_256BIT);
986 if (ret) {
987 cmd->engine_error = cmd_q->cmd_error;
988 goto e_src;
989 }
990
991 /* ...but we only need AES_BLOCK_SIZE bytes */
992 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
993 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
994
995e_src:
996 ccp_free_data(&src, cmd_q);
997
998e_ctx:
999 ccp_dm_free(&ctx);
1000
1001e_key:
1002 ccp_dm_free(&key);
1003
1004 return ret;
1005}
1006
1007static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1008{
1009 struct ccp_aes_engine *aes = &cmd->u.aes;
1010 struct ccp_dm_workarea key, ctx;
1011 struct ccp_data src, dst;
1012 struct ccp_op op;
1013 unsigned int dm_offset;
1014 bool in_place = false;
1015 int ret;
1016
1017 if (aes->mode == CCP_AES_MODE_CMAC)
1018 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1019
1020 if (!((aes->key_len == AES_KEYSIZE_128) ||
1021 (aes->key_len == AES_KEYSIZE_192) ||
1022 (aes->key_len == AES_KEYSIZE_256)))
1023 return -EINVAL;
1024
1025 if (((aes->mode == CCP_AES_MODE_ECB) ||
1026 (aes->mode == CCP_AES_MODE_CBC) ||
1027 (aes->mode == CCP_AES_MODE_CFB)) &&
1028 (aes->src_len & (AES_BLOCK_SIZE - 1)))
1029 return -EINVAL;
1030
1031 if (!aes->key || !aes->src || !aes->dst)
1032 return -EINVAL;
1033
1034 if (aes->mode != CCP_AES_MODE_ECB) {
1035 if (aes->iv_len != AES_BLOCK_SIZE)
1036 return -EINVAL;
1037
1038 if (!aes->iv)
1039 return -EINVAL;
1040 }
1041
1042 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1043 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1044
1045 ret = -EIO;
1046 memset(&op, 0, sizeof(op));
1047 op.cmd_q = cmd_q;
1048 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1049 op.ksb_key = cmd_q->ksb_key;
1050 op.ksb_ctx = cmd_q->ksb_ctx;
1051 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1052 op.u.aes.type = aes->type;
1053 op.u.aes.mode = aes->mode;
1054 op.u.aes.action = aes->action;
1055
1056 /* All supported key sizes fit in a single (32-byte) KSB entry
1057 * and must be in little endian format. Use the 256-bit byte
1058 * swap passthru option to convert from big endian to little
1059 * endian.
1060 */
1061 ret = ccp_init_dm_workarea(&key, cmd_q,
1062 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1063 DMA_TO_DEVICE);
1064 if (ret)
1065 return ret;
1066
1067 dm_offset = CCP_KSB_BYTES - aes->key_len;
1068 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1069 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1070 CCP_PASSTHRU_BYTESWAP_256BIT);
1071 if (ret) {
1072 cmd->engine_error = cmd_q->cmd_error;
1073 goto e_key;
1074 }
1075
1076 /* The AES context fits in a single (32-byte) KSB entry and
1077 * must be in little endian format. Use the 256-bit byte swap
1078 * passthru option to convert from big endian to little endian.
1079 */
1080 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1081 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1082 DMA_BIDIRECTIONAL);
1083 if (ret)
1084 goto e_key;
1085
1086 if (aes->mode != CCP_AES_MODE_ECB) {
1087 /* Load the AES context - conver to LE */
1088 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1089 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1090 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1091 CCP_PASSTHRU_BYTESWAP_256BIT);
1092 if (ret) {
1093 cmd->engine_error = cmd_q->cmd_error;
1094 goto e_ctx;
1095 }
1096 }
1097
1098 /* Prepare the input and output data workareas. For in-place
1099 * operations we need to set the dma direction to BIDIRECTIONAL
1100 * and copy the src workarea to the dst workarea.
1101 */
1102 if (sg_virt(aes->src) == sg_virt(aes->dst))
1103 in_place = true;
1104
1105 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1106 AES_BLOCK_SIZE,
1107 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1108 if (ret)
1109 goto e_ctx;
1110
8db88467 1111 if (in_place) {
63b94509 1112 dst = src;
8db88467 1113 } else {
63b94509
TL		 1114 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1115 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1116 if (ret)
1117 goto e_src;
1118 }
1119
1120 /* Send data to the CCP AES engine */
1121 while (src.sg_wa.bytes_left) {
1122 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1123 if (!src.sg_wa.bytes_left) {
1124 op.eom = 1;
1125
1126 /* Since we don't retrieve the AES context in ECB
1127 * mode we have to wait for the operation to complete
1128 * on the last piece of data
1129 */
1130 if (aes->mode == CCP_AES_MODE_ECB)
1131 op.soc = 1;
1132 }
1133
1134 ret = ccp_perform_aes(&op);
1135 if (ret) {
1136 cmd->engine_error = cmd_q->cmd_error;
1137 goto e_dst;
1138 }
1139
1140 ccp_process_data(&src, &dst, &op);
1141 }
1142
1143 if (aes->mode != CCP_AES_MODE_ECB) {
1144 /* Retrieve the AES context - convert from LE to BE using
1145 * 32-byte (256-bit) byteswapping
1146 */
1147 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1148 CCP_PASSTHRU_BYTESWAP_256BIT);
1149 if (ret) {
1150 cmd->engine_error = cmd_q->cmd_error;
1151 goto e_dst;
1152 }
1153
1154 /* ...but we only need AES_BLOCK_SIZE bytes */
1155 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1156 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1157 }
1158
1159e_dst:
1160 if (!in_place)
1161 ccp_free_data(&dst, cmd_q);
1162
1163e_src:
1164 ccp_free_data(&src, cmd_q);
1165
1166e_ctx:
1167 ccp_dm_free(&ctx);
1168
1169e_key:
1170 ccp_dm_free(&key);
1171
1172 return ret;
1173}
1174
1175static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1176 struct ccp_cmd *cmd)
1177{
1178 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1179 struct ccp_dm_workarea key, ctx;
1180 struct ccp_data src, dst;
1181 struct ccp_op op;
1182 unsigned int unit_size, dm_offset;
1183 bool in_place = false;
1184 int ret;
1185
1186 switch (xts->unit_size) {
1187 case CCP_XTS_AES_UNIT_SIZE_16:
1188 unit_size = 16;
1189 break;
1190 case CCP_XTS_AES_UNIT_SIZE_512:
1191 unit_size = 512;
1192 break;
1193 case CCP_XTS_AES_UNIT_SIZE_1024:
1194 unit_size = 1024;
1195 break;
1196 case CCP_XTS_AES_UNIT_SIZE_2048:
1197 unit_size = 2048;
1198 break;
1199 case CCP_XTS_AES_UNIT_SIZE_4096:
1200 unit_size = 4096;
1201 break;
1202
1203 default:
1204 return -EINVAL;
1205 }
1206
1207 if (xts->key_len != AES_KEYSIZE_128)
1208 return -EINVAL;
1209
1210 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1211 return -EINVAL;
1212
1213 if (xts->iv_len != AES_BLOCK_SIZE)
1214 return -EINVAL;
1215
1216 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1217 return -EINVAL;
1218
1219 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1220 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1221
1222 ret = -EIO;
1223 memset(&op, 0, sizeof(op));
1224 op.cmd_q = cmd_q;
1225 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1226 op.ksb_key = cmd_q->ksb_key;
1227 op.ksb_ctx = cmd_q->ksb_ctx;
1228 op.init = 1;
1229 op.u.xts.action = xts->action;
1230 op.u.xts.unit_size = xts->unit_size;
1231
1232 /* All supported key sizes fit in a single (32-byte) KSB entry
1233 * and must be in little endian format. Use the 256-bit byte
1234 * swap passthru option to convert from big endian to little
1235 * endian.
1236 */
1237 ret = ccp_init_dm_workarea(&key, cmd_q,
1238 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1239 DMA_TO_DEVICE);
1240 if (ret)
1241 return ret;
1242
1243 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1244 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1245 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1246 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1247 CCP_PASSTHRU_BYTESWAP_256BIT);
1248 if (ret) {
1249 cmd->engine_error = cmd_q->cmd_error;
1250 goto e_key;
1251 }
1252
1253 /* The AES context fits in a single (32-byte) KSB entry and
1254 * for XTS is already in little endian format so no byte swapping
1255 * is needed.
1256 */
1257 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1258 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1259 DMA_BIDIRECTIONAL);
1260 if (ret)
1261 goto e_key;
1262
1263 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1264 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1265 CCP_PASSTHRU_BYTESWAP_NOOP);
1266 if (ret) {
1267 cmd->engine_error = cmd_q->cmd_error;
1268 goto e_ctx;
1269 }
1270
1271 /* Prepare the input and output data workareas. For in-place
1272 * operations we need to set the dma direction to BIDIRECTIONAL
1273 * and copy the src workarea to the dst workarea.
1274 */
1275 if (sg_virt(xts->src) == sg_virt(xts->dst))
1276 in_place = true;
1277
1278 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1279 unit_size,
1280 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1281 if (ret)
1282 goto e_ctx;
1283
8db88467 1284 if (in_place) {
63b94509 1285 dst = src;
8db88467 1286 } else {
63b94509
TL		 1287 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1288 unit_size, DMA_FROM_DEVICE);
1289 if (ret)
1290 goto e_src;
1291 }
1292
1293 /* Send data to the CCP AES engine */
1294 while (src.sg_wa.bytes_left) {
1295 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1296 if (!src.sg_wa.bytes_left)
1297 op.eom = 1;
1298
1299 ret = ccp_perform_xts_aes(&op);
1300 if (ret) {
1301 cmd->engine_error = cmd_q->cmd_error;
1302 goto e_dst;
1303 }
1304
1305 ccp_process_data(&src, &dst, &op);
1306 }
1307
1308 /* Retrieve the AES context - convert from LE to BE using
1309 * 32-byte (256-bit) byteswapping
1310 */
1311 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1312 CCP_PASSTHRU_BYTESWAP_256BIT);
1313 if (ret) {
1314 cmd->engine_error = cmd_q->cmd_error;
1315 goto e_dst;
1316 }
1317
1318 /* ...but we only need AES_BLOCK_SIZE bytes */
1319 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1320 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1321
1322e_dst:
1323 if (!in_place)
1324 ccp_free_data(&dst, cmd_q);
1325
1326e_src:
1327 ccp_free_data(&src, cmd_q);
1328
1329e_ctx:
1330 ccp_dm_free(&ctx);
1331
1332e_key:
1333 ccp_dm_free(&key);
1334
1335 return ret;
1336}
1337
1338static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1339{
1340 struct ccp_sha_engine *sha = &cmd->u.sha;
1341 struct ccp_dm_workarea ctx;
1342 struct ccp_data src;
1343 struct ccp_op op;
1344 int ret;
1345
1346 if (sha->ctx_len != CCP_SHA_CTXSIZE)
1347 return -EINVAL;
1348
1349 if (!sha->ctx)
1350 return -EINVAL;
1351
1352 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1353 return -EINVAL;
1354
1355 if (!sha->src_len) {
1356 const u8 *sha_zero;
1357
1358 /* Not final, just return */
1359 if (!sha->final)
1360 return 0;
1361
1362 /* CCP can't do a zero length sha operation so the caller
1363 * must buffer the data.
1364 */
1365 if (sha->msg_bits)
1366 return -EINVAL;
1367
bdd75064
LC		 1368 /* The CCP cannot perform zero-length sha operations so the
1369 * caller is required to buffer data for the final operation.
1370 * However, a sha operation for a message with a total length
1371 * of zero is valid so known values are required to supply
1372 * the result.
63b94509
TL		 1373 */
1374 switch (sha->type) {
1375 case CCP_SHA_TYPE_1:
bdd75064 1376 sha_zero = sha1_zero_message_hash;
63b94509
TL		 1377 break;
1378 case CCP_SHA_TYPE_224:
bdd75064 1379 sha_zero = sha224_zero_message_hash;
63b94509
TL		 1380 break;
1381 case CCP_SHA_TYPE_256:
bdd75064 1382 sha_zero = sha256_zero_message_hash;
63b94509
TL		 1383 break;
1384 default:
1385 return -EINVAL;
1386 }
1387
1388 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1389 sha->ctx_len, 1);
1390
1391 return 0;
1392 }
1393
1394 if (!sha->src)
1395 return -EINVAL;
1396
1397 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1398
1399 memset(&op, 0, sizeof(op));
1400 op.cmd_q = cmd_q;
1401 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1402 op.ksb_ctx = cmd_q->ksb_ctx;
1403 op.u.sha.type = sha->type;
1404 op.u.sha.msg_bits = sha->msg_bits;
1405
1406 /* The SHA context fits in a single (32-byte) KSB entry and
1407 * must be in little endian format. Use the 256-bit byte swap
1408 * passthru option to convert from big endian to little endian.
1409 */
1410 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1411 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1412 DMA_BIDIRECTIONAL);
1413 if (ret)
1414 return ret;
1415
c11baa02
TL		 1416 if (sha->first) {
1417 const __be32 *init;
1418
1419 switch (sha->type) {
1420 case CCP_SHA_TYPE_1:
1421 init = ccp_sha1_init;
1422 break;
1423 case CCP_SHA_TYPE_224:
1424 init = ccp_sha224_init;
1425 break;
1426 case CCP_SHA_TYPE_256:
1427 init = ccp_sha256_init;
1428 break;
1429 default:
1430 ret = -EINVAL;
1431 goto e_ctx;
1432 }
1433 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
8db88467 1434 } else {
c11baa02 1435 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
8db88467 1436 }
c11baa02 1437
63b94509
TL		 1438 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1439 CCP_PASSTHRU_BYTESWAP_256BIT);
1440 if (ret) {
1441 cmd->engine_error = cmd_q->cmd_error;
1442 goto e_ctx;
1443 }
1444
1445 /* Send data to the CCP SHA engine */
1446 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1447 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1448 if (ret)
1449 goto e_ctx;
1450
1451 while (src.sg_wa.bytes_left) {
1452 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1453 if (sha->final && !src.sg_wa.bytes_left)
1454 op.eom = 1;
1455
1456 ret = ccp_perform_sha(&op);
1457 if (ret) {
1458 cmd->engine_error = cmd_q->cmd_error;
1459 goto e_data;
1460 }
1461
1462 ccp_process_data(&src, NULL, &op);
1463 }
1464
1465 /* Retrieve the SHA context - convert from LE to BE using
1466 * 32-byte (256-bit) byteswapping to BE
1467 */
1468 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1469 CCP_PASSTHRU_BYTESWAP_256BIT);
1470 if (ret) {
1471 cmd->engine_error = cmd_q->cmd_error;
1472 goto e_data;
1473 }
1474
1475 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1476
c11baa02
TL		 1477 if (sha->final && sha->opad) {
1478 /* HMAC operation, recursively perform final SHA */
1479 struct ccp_cmd hmac_cmd;
1480 struct scatterlist sg;
1481 u64 block_size, digest_size;
1482 u8 *hmac_buf;
1483
1484 switch (sha->type) {
1485 case CCP_SHA_TYPE_1:
1486 block_size = SHA1_BLOCK_SIZE;
1487 digest_size = SHA1_DIGEST_SIZE;
1488 break;
1489 case CCP_SHA_TYPE_224:
1490 block_size = SHA224_BLOCK_SIZE;
1491 digest_size = SHA224_DIGEST_SIZE;
1492 break;
1493 case CCP_SHA_TYPE_256:
1494 block_size = SHA256_BLOCK_SIZE;
1495 digest_size = SHA256_DIGEST_SIZE;
1496 break;
1497 default:
1498 ret = -EINVAL;
1499 goto e_data;
1500 }
1501
1502 if (sha->opad_len != block_size) {
1503 ret = -EINVAL;
1504 goto e_data;
1505 }
1506
1507 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1508 if (!hmac_buf) {
1509 ret = -ENOMEM;
1510 goto e_data;
1511 }
1512 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1513
1514 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1515 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1516
1517 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1518 hmac_cmd.engine = CCP_ENGINE_SHA;
1519 hmac_cmd.u.sha.type = sha->type;
1520 hmac_cmd.u.sha.ctx = sha->ctx;
1521 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1522 hmac_cmd.u.sha.src = &sg;
1523 hmac_cmd.u.sha.src_len = block_size + digest_size;
1524 hmac_cmd.u.sha.opad = NULL;
1525 hmac_cmd.u.sha.opad_len = 0;
1526 hmac_cmd.u.sha.first = 1;
1527 hmac_cmd.u.sha.final = 1;
1528 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1529
1530 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1531 if (ret)
1532 cmd->engine_error = hmac_cmd.engine_error;
1533
1534 kfree(hmac_buf);
1535 }
1536
63b94509
TL		 1537e_data:
1538 ccp_free_data(&src, cmd_q);
1539
1540e_ctx:
1541 ccp_dm_free(&ctx);
1542
1543 return ret;
1544}
1545
1546static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1547{
1548 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1549 struct ccp_dm_workarea exp, src;
1550 struct ccp_data dst;
1551 struct ccp_op op;
1552 unsigned int ksb_count, i_len, o_len;
1553 int ret;
1554
1555 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1556 return -EINVAL;
1557
1558 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1559 return -EINVAL;
1560
1561 /* The RSA modulus must precede the message being acted upon, so
1562 * it must be copied to a DMA area where the message and the
1563 * modulus can be concatenated. Therefore the input buffer
1564 * length required is twice the output buffer length (which
1565 * must be a multiple of 256-bits).
1566 */
1567 o_len = ((rsa->key_size + 255) / 256) * 32;
1568 i_len = o_len * 2;
1569
1570 ksb_count = o_len / CCP_KSB_BYTES;
1571
1572 memset(&op, 0, sizeof(op));
1573 op.cmd_q = cmd_q;
1574 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1575 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1576 if (!op.ksb_key)
1577 return -EIO;
1578
1579 /* The RSA exponent may span multiple (32-byte) KSB entries and must
1580 * be in little endian format. Reverse copy each 32-byte chunk
1581 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1582 * and each byte within that chunk and do not perform any byte swap
1583 * operations on the passthru operation.
1584 */
1585 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1586 if (ret)
1587 goto e_ksb;
1588
355eba5d
TL		 1589 ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1590 CCP_KSB_BYTES, false);
1591 if (ret)
1592 goto e_exp;
63b94509
TL		 1593 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1594 CCP_PASSTHRU_BYTESWAP_NOOP);
1595 if (ret) {
1596 cmd->engine_error = cmd_q->cmd_error;
1597 goto e_exp;
1598 }
1599
1600 /* Concatenate the modulus and the message. Both the modulus and
1601 * the operands must be in little endian format. Since the input
1602 * is in big endian format it must be converted.
1603 */
1604 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1605 if (ret)
1606 goto e_exp;
1607
355eba5d
TL		 1608 ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1609 CCP_KSB_BYTES, false);
1610 if (ret)
1611 goto e_src;
63b94509 1612 src.address += o_len; /* Adjust the address for the copy operation */
355eba5d
TL		 1613 ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1614 CCP_KSB_BYTES, false);
1615 if (ret)
1616 goto e_src;
63b94509
TL		 1617 src.address -= o_len; /* Reset the address to original value */
1618
1619 /* Prepare the output area for the operation */
1620 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1621 o_len, DMA_FROM_DEVICE);
1622 if (ret)
1623 goto e_src;
1624
1625 op.soc = 1;
1626 op.src.u.dma.address = src.dma.address;
1627 op.src.u.dma.offset = 0;
1628 op.src.u.dma.length = i_len;
1629 op.dst.u.dma.address = dst.dm_wa.dma.address;
1630 op.dst.u.dma.offset = 0;
1631 op.dst.u.dma.length = o_len;
1632
1633 op.u.rsa.mod_size = rsa->key_size;
1634 op.u.rsa.input_len = i_len;
1635
1636 ret = ccp_perform_rsa(&op);
1637 if (ret) {
1638 cmd->engine_error = cmd_q->cmd_error;
1639 goto e_dst;
1640 }
1641
1642 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1643
1644e_dst:
1645 ccp_free_data(&dst, cmd_q);
1646
1647e_src:
1648 ccp_dm_free(&src);
1649
1650e_exp:
1651 ccp_dm_free(&exp);
1652
1653e_ksb:
1654 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1655
1656 return ret;
1657}
1658
1659static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1660 struct ccp_cmd *cmd)
1661{
1662 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1663 struct ccp_dm_workarea mask;
1664 struct ccp_data src, dst;
1665 struct ccp_op op;
1666 bool in_place = false;
1667 unsigned int i;
1668 int ret;
1669
1670 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1671 return -EINVAL;
1672
1673 if (!pt->src || !pt->dst)
1674 return -EINVAL;
1675
1676 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1677 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1678 return -EINVAL;
1679 if (!pt->mask)
1680 return -EINVAL;
1681 }
1682
1683 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1684
1685 memset(&op, 0, sizeof(op));
1686 op.cmd_q = cmd_q;
1687 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1688
1689 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1690 /* Load the mask */
1691 op.ksb_key = cmd_q->ksb_key;
1692
1693 ret = ccp_init_dm_workarea(&mask, cmd_q,
1694 CCP_PASSTHRU_KSB_COUNT *
1695 CCP_KSB_BYTES,
1696 DMA_TO_DEVICE);
1697 if (ret)
1698 return ret;
1699
1700 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1701 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1702 CCP_PASSTHRU_BYTESWAP_NOOP);
1703 if (ret) {
1704 cmd->engine_error = cmd_q->cmd_error;
1705 goto e_mask;
1706 }
1707 }
1708
1709 /* Prepare the input and output data workareas. For in-place
1710 * operations we need to set the dma direction to BIDIRECTIONAL
1711 * and copy the src workarea to the dst workarea.
1712 */
1713 if (sg_virt(pt->src) == sg_virt(pt->dst))
1714 in_place = true;
1715
1716 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1717 CCP_PASSTHRU_MASKSIZE,
1718 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1719 if (ret)
1720 goto e_mask;
1721
8db88467 1722 if (in_place) {
63b94509 1723 dst = src;
8db88467 1724 } else {
63b94509
TL		 1725 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1726 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1727 if (ret)
1728 goto e_src;
1729 }
1730
1731 /* Send data to the CCP Passthru engine
1732 * Because the CCP engine works on a single source and destination
1733 * dma address at a time, each entry in the source scatterlist
1734 * (after the dma_map_sg call) must be less than or equal to the
1735 * (remaining) length in the destination scatterlist entry and the
1736 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1737 */
1738 dst.sg_wa.sg_used = 0;
1739 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1740 if (!dst.sg_wa.sg ||
1741 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1742 ret = -EINVAL;
1743 goto e_dst;
1744 }
1745
1746 if (i == src.sg_wa.dma_count) {
1747 op.eom = 1;
1748 op.soc = 1;
1749 }
1750
1751 op.src.type = CCP_MEMTYPE_SYSTEM;
1752 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1753 op.src.u.dma.offset = 0;
1754 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1755
1756 op.dst.type = CCP_MEMTYPE_SYSTEM;
1757 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
80e84c16
DJ		 1758 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1759 op.dst.u.dma.length = op.src.u.dma.length;
63b94509
TL		 1760
1761 ret = ccp_perform_passthru(&op);
1762 if (ret) {
1763 cmd->engine_error = cmd_q->cmd_error;
1764 goto e_dst;
1765 }
1766
1767 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1768 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1769 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1770 dst.sg_wa.sg_used = 0;
1771 }
1772 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1773 }
1774
1775e_dst:
1776 if (!in_place)
1777 ccp_free_data(&dst, cmd_q);
1778
1779e_src:
1780 ccp_free_data(&src, cmd_q);
1781
1782e_mask:
1783 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1784 ccp_dm_free(&mask);
1785
1786 return ret;
1787}
1788
1789static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1790{
1791 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1792 struct ccp_dm_workarea src, dst;
1793 struct ccp_op op;
1794 int ret;
1795 u8 *save;
1796
1797 if (!ecc->u.mm.operand_1 ||
1798 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1799 return -EINVAL;
1800
1801 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1802 if (!ecc->u.mm.operand_2 ||
1803 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1804 return -EINVAL;
1805
1806 if (!ecc->u.mm.result ||
1807 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1808 return -EINVAL;
1809
1810 memset(&op, 0, sizeof(op));
1811 op.cmd_q = cmd_q;
1812 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1813
1814 /* Concatenate the modulus and the operands. Both the modulus and
1815 * the operands must be in little endian format. Since the input
1816 * is in big endian format it must be converted and placed in a
1817 * fixed length buffer.
1818 */
1819 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1820 DMA_TO_DEVICE);
1821 if (ret)
1822 return ret;
1823
1824 /* Save the workarea address since it is updated in order to perform
1825 * the concatenation
1826 */
1827 save = src.address;
1828
1829 /* Copy the ECC modulus */
355eba5d
TL		 1830 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1831 CCP_ECC_OPERAND_SIZE, false);
1832 if (ret)
1833 goto e_src;
63b94509
TL		 1834 src.address += CCP_ECC_OPERAND_SIZE;
1835
1836 /* Copy the first operand */
355eba5d
TL		 1837 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1838 ecc->u.mm.operand_1_len,
1839 CCP_ECC_OPERAND_SIZE, false);
1840 if (ret)
1841 goto e_src;
63b94509
TL		 1842 src.address += CCP_ECC_OPERAND_SIZE;
1843
1844 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1845 /* Copy the second operand */
355eba5d
TL		 1846 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1847 ecc->u.mm.operand_2_len,
1848 CCP_ECC_OPERAND_SIZE, false);
1849 if (ret)
1850 goto e_src;
63b94509
TL		 1851 src.address += CCP_ECC_OPERAND_SIZE;
1852 }
1853
1854 /* Restore the workarea address */
1855 src.address = save;
1856
1857 /* Prepare the output area for the operation */
1858 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1859 DMA_FROM_DEVICE);
1860 if (ret)
1861 goto e_src;
1862
1863 op.soc = 1;
1864 op.src.u.dma.address = src.dma.address;
1865 op.src.u.dma.offset = 0;
1866 op.src.u.dma.length = src.length;
1867 op.dst.u.dma.address = dst.dma.address;
1868 op.dst.u.dma.offset = 0;
1869 op.dst.u.dma.length = dst.length;
1870
1871 op.u.ecc.function = cmd->u.ecc.function;
1872
1873 ret = ccp_perform_ecc(&op);
1874 if (ret) {
1875 cmd->engine_error = cmd_q->cmd_error;
1876 goto e_dst;
1877 }
1878
1879 ecc->ecc_result = le16_to_cpup(
1880 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1881 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1882 ret = -EIO;
1883 goto e_dst;
1884 }
1885
1886 /* Save the ECC result */
1887 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1888
1889e_dst:
1890 ccp_dm_free(&dst);
1891
1892e_src:
1893 ccp_dm_free(&src);
1894
1895 return ret;
1896}
1897
1898static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1899{
1900 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1901 struct ccp_dm_workarea src, dst;
1902 struct ccp_op op;
1903 int ret;
1904 u8 *save;
1905
1906 if (!ecc->u.pm.point_1.x ||
1907 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1908 !ecc->u.pm.point_1.y ||
1909 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1910 return -EINVAL;
1911
1912 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1913 if (!ecc->u.pm.point_2.x ||
1914 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1915 !ecc->u.pm.point_2.y ||
1916 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1917 return -EINVAL;
1918 } else {
1919 if (!ecc->u.pm.domain_a ||
1920 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1921 return -EINVAL;
1922
1923 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1924 if (!ecc->u.pm.scalar ||
1925 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1926 return -EINVAL;
1927 }
1928
1929 if (!ecc->u.pm.result.x ||
1930 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1931 !ecc->u.pm.result.y ||
1932 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1933 return -EINVAL;
1934
1935 memset(&op, 0, sizeof(op));
1936 op.cmd_q = cmd_q;
1937 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1938
1939 /* Concatenate the modulus and the operands. Both the modulus and
1940 * the operands must be in little endian format. Since the input
1941 * is in big endian format it must be converted and placed in a
1942 * fixed length buffer.
1943 */
1944 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1945 DMA_TO_DEVICE);
1946 if (ret)
1947 return ret;
1948
1949 /* Save the workarea address since it is updated in order to perform
1950 * the concatenation
1951 */
1952 save = src.address;
1953
1954 /* Copy the ECC modulus */
355eba5d
TL		 1955 ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1956 CCP_ECC_OPERAND_SIZE, false);
1957 if (ret)
1958 goto e_src;
63b94509
TL		 1959 src.address += CCP_ECC_OPERAND_SIZE;
1960
1961 /* Copy the first point X and Y coordinate */
355eba5d
TL		 1962 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1963 ecc->u.pm.point_1.x_len,
1964 CCP_ECC_OPERAND_SIZE, false);
1965 if (ret)
1966 goto e_src;
63b94509 1967 src.address += CCP_ECC_OPERAND_SIZE;
355eba5d
TL		 1968 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1969 ecc->u.pm.point_1.y_len,
1970 CCP_ECC_OPERAND_SIZE, false);
1971 if (ret)
1972 goto e_src;
63b94509
TL		 1973 src.address += CCP_ECC_OPERAND_SIZE;
1974
1975 /* Set the first point Z coordianate to 1 */
8db88467 1976 *src.address = 0x01;
63b94509
TL		 1977 src.address += CCP_ECC_OPERAND_SIZE;
1978
1979 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1980 /* Copy the second point X and Y coordinate */
355eba5d
TL		 1981 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1982 ecc->u.pm.point_2.x_len,
1983 CCP_ECC_OPERAND_SIZE, false);
1984 if (ret)
1985 goto e_src;
63b94509 1986 src.address += CCP_ECC_OPERAND_SIZE;
355eba5d
TL		 1987 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1988 ecc->u.pm.point_2.y_len,
1989 CCP_ECC_OPERAND_SIZE, false);
1990 if (ret)
1991 goto e_src;
63b94509
TL		 1992 src.address += CCP_ECC_OPERAND_SIZE;
1993
1994 /* Set the second point Z coordianate to 1 */
8db88467 1995 *src.address = 0x01;
63b94509
TL		 1996 src.address += CCP_ECC_OPERAND_SIZE;
1997 } else {
1998 /* Copy the Domain "a" parameter */
355eba5d
TL		 1999 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
2000 ecc->u.pm.domain_a_len,
2001 CCP_ECC_OPERAND_SIZE, false);
2002 if (ret)
2003 goto e_src;
63b94509
TL		 2004 src.address += CCP_ECC_OPERAND_SIZE;
2005
2006 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2007 /* Copy the scalar value */
355eba5d
TL		 2008 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
2009 ecc->u.pm.scalar_len,
2010 CCP_ECC_OPERAND_SIZE,
2011 false);
2012 if (ret)
2013 goto e_src;
63b94509
TL		 2014 src.address += CCP_ECC_OPERAND_SIZE;
2015 }
2016 }
2017
2018 /* Restore the workarea address */
2019 src.address = save;
2020
2021 /* Prepare the output area for the operation */
2022 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2023 DMA_FROM_DEVICE);
2024 if (ret)
2025 goto e_src;
2026
2027 op.soc = 1;
2028 op.src.u.dma.address = src.dma.address;
2029 op.src.u.dma.offset = 0;
2030 op.src.u.dma.length = src.length;
2031 op.dst.u.dma.address = dst.dma.address;
2032 op.dst.u.dma.offset = 0;
2033 op.dst.u.dma.length = dst.length;
2034
2035 op.u.ecc.function = cmd->u.ecc.function;
2036
2037 ret = ccp_perform_ecc(&op);
2038 if (ret) {
2039 cmd->engine_error = cmd_q->cmd_error;
2040 goto e_dst;
2041 }
2042
2043 ecc->ecc_result = le16_to_cpup(
2044 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2045 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2046 ret = -EIO;
2047 goto e_dst;
2048 }
2049
2050 /* Save the workarea address since it is updated as we walk through
2051 * to copy the point math result
2052 */
2053 save = dst.address;
2054
2055 /* Save the ECC result X and Y coordinates */
2056 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
2057 CCP_ECC_MODULUS_BYTES);
2058 dst.address += CCP_ECC_OUTPUT_SIZE;
2059 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
2060 CCP_ECC_MODULUS_BYTES);
2061 dst.address += CCP_ECC_OUTPUT_SIZE;
2062
2063 /* Restore the workarea address */
2064 dst.address = save;
2065
2066e_dst:
2067 ccp_dm_free(&dst);
2068
2069e_src:
2070 ccp_dm_free(&src);
2071
2072 return ret;
2073}
2074
2075static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2076{
2077 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2078
2079 ecc->ecc_result = 0;
2080
2081 if (!ecc->mod ||
2082 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2083 return -EINVAL;
2084
2085 switch (ecc->function) {
2086 case CCP_ECC_FUNCTION_MMUL_384BIT:
2087 case CCP_ECC_FUNCTION_MADD_384BIT:
2088 case CCP_ECC_FUNCTION_MINV_384BIT:
2089 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2090
2091 case CCP_ECC_FUNCTION_PADD_384BIT:
2092 case CCP_ECC_FUNCTION_PMUL_384BIT:
2093 case CCP_ECC_FUNCTION_PDBL_384BIT:
2094 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2095
2096 default:
2097 return -EINVAL;
2098 }
2099}
2100
2101int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2102{
2103 int ret;
2104
2105 cmd->engine_error = 0;
2106 cmd_q->cmd_error = 0;
2107 cmd_q->int_rcvd = 0;
2108 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
2109
2110 switch (cmd->engine) {
2111 case CCP_ENGINE_AES:
2112 ret = ccp_run_aes_cmd(cmd_q, cmd);
2113 break;
2114 case CCP_ENGINE_XTS_AES_128:
2115 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2116 break;
2117 case CCP_ENGINE_SHA:
2118 ret = ccp_run_sha_cmd(cmd_q, cmd);
2119 break;
2120 case CCP_ENGINE_RSA:
2121 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2122 break;
2123 case CCP_ENGINE_PASSTHRU:
2124 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2125 break;
2126 case CCP_ENGINE_ECC:
2127 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2128 break;
2129 default:
2130 ret = -EINVAL;
2131 }
2132
2133 return ret;
2134}
Linux 6.x block layer and io_uring tree(s)