projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'next-general' of git://git.kernel.org/pub/scm/linux/kernel/git/jmorris...
[linux-2.6-block.git] / drivers / crypto / axis / artpec6_crypto.c
CommitLineData
a21eb94f
LP		 1/*
2 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
3 *
4 * Copyright (C) 2014-2017 Axis Communications AB
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/bitfield.h>
9#include <linux/crypto.h>
10#include <linux/debugfs.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/fault-inject.h>
14#include <linux/init.h>
15#include <linux/interrupt.h>
16#include <linux/kernel.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/platform_device.h>
21#include <linux/scatterlist.h>
22#include <linux/slab.h>
23
24#include <crypto/aes.h>
6d6e71fe 25#include <crypto/gcm.h>
a21eb94f
LP		 26#include <crypto/internal/aead.h>
27#include <crypto/internal/hash.h>
28#include <crypto/internal/skcipher.h>
29#include <crypto/scatterwalk.h>
30#include <crypto/sha.h>
31#include <crypto/xts.h>
32
33/* Max length of a line in all cache levels for Artpec SoCs. */
34#define ARTPEC_CACHE_LINE_MAX 32
35
36#define PDMA_OUT_CFG 0x0000
37#define PDMA_OUT_BUF_CFG 0x0004
38#define PDMA_OUT_CMD 0x0008
39#define PDMA_OUT_DESCRQ_PUSH 0x0010
40#define PDMA_OUT_DESCRQ_STAT 0x0014
41
42#define A6_PDMA_IN_CFG 0x0028
43#define A6_PDMA_IN_BUF_CFG 0x002c
44#define A6_PDMA_IN_CMD 0x0030
45#define A6_PDMA_IN_STATQ_PUSH 0x0038
46#define A6_PDMA_IN_DESCRQ_PUSH 0x0044
47#define A6_PDMA_IN_DESCRQ_STAT 0x0048
48#define A6_PDMA_INTR_MASK 0x0068
49#define A6_PDMA_ACK_INTR 0x006c
50#define A6_PDMA_MASKED_INTR 0x0074
51
52#define A7_PDMA_IN_CFG 0x002c
53#define A7_PDMA_IN_BUF_CFG 0x0030
54#define A7_PDMA_IN_CMD 0x0034
55#define A7_PDMA_IN_STATQ_PUSH 0x003c
56#define A7_PDMA_IN_DESCRQ_PUSH 0x0048
57#define A7_PDMA_IN_DESCRQ_STAT 0x004C
58#define A7_PDMA_INTR_MASK 0x006c
59#define A7_PDMA_ACK_INTR 0x0070
60#define A7_PDMA_MASKED_INTR 0x0078
61
62#define PDMA_OUT_CFG_EN BIT(0)
63
64#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
65#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
66
67#define PDMA_OUT_CMD_START BIT(0)
68#define A6_PDMA_OUT_CMD_STOP BIT(3)
69#define A7_PDMA_OUT_CMD_STOP BIT(2)
70
71#define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0)
72#define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6)
73
74#define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0)
75#define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4)
76
77#define PDMA_IN_CFG_EN BIT(0)
78
79#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
80#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
81#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10)
82
83#define PDMA_IN_CMD_START BIT(0)
84#define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2)
85#define A6_PDMA_IN_CMD_STOP BIT(3)
86#define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1)
87#define A7_PDMA_IN_CMD_STOP BIT(2)
88
89#define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0)
90#define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6)
91
92#define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0)
93#define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6)
94
95#define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0)
96#define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4)
97
98#define A6_PDMA_INTR_MASK_IN_DATA BIT(2)
99#define A6_PDMA_INTR_MASK_IN_EOP BIT(3)
100#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4)
101
102#define A7_PDMA_INTR_MASK_IN_DATA BIT(3)
103#define A7_PDMA_INTR_MASK_IN_EOP BIT(4)
104#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5)
105
106#define A6_CRY_MD_OPER GENMASK(19, 16)
107
108#define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20)
109#define A6_CRY_MD_HASH_HMAC_FIN BIT(23)
110
111#define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20)
112#define A6_CRY_MD_CIPHER_DECR BIT(22)
113#define A6_CRY_MD_CIPHER_TWEAK BIT(23)
114#define A6_CRY_MD_CIPHER_DSEQ BIT(24)
115
116#define A7_CRY_MD_OPER GENMASK(11, 8)
117
118#define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12)
119#define A7_CRY_MD_HASH_HMAC_FIN BIT(15)
120
121#define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12)
122#define A7_CRY_MD_CIPHER_DECR BIT(14)
123#define A7_CRY_MD_CIPHER_TWEAK BIT(15)
124#define A7_CRY_MD_CIPHER_DSEQ BIT(16)
125
126/* DMA metadata constants */
127#define regk_crypto_aes_cbc 0x00000002
128#define regk_crypto_aes_ctr 0x00000003
129#define regk_crypto_aes_ecb 0x00000001
130#define regk_crypto_aes_gcm 0x00000004
131#define regk_crypto_aes_xts 0x00000005
132#define regk_crypto_cache 0x00000002
133#define a6_regk_crypto_dlkey 0x0000000a
134#define a7_regk_crypto_dlkey 0x0000000e
135#define regk_crypto_ext 0x00000001
136#define regk_crypto_hmac_sha1 0x00000007
137#define regk_crypto_hmac_sha256 0x00000009
a21eb94f
LP		 138#define regk_crypto_init 0x00000000
139#define regk_crypto_key_128 0x00000000
140#define regk_crypto_key_192 0x00000001
141#define regk_crypto_key_256 0x00000002
142#define regk_crypto_null 0x00000000
143#define regk_crypto_sha1 0x00000006
144#define regk_crypto_sha256 0x00000008
a21eb94f
LP		 145
146/* DMA descriptor structures */
147struct pdma_descr_ctrl {
148 unsigned char short_descr : 1;
149 unsigned char pad1 : 1;
150 unsigned char eop : 1;
151 unsigned char intr : 1;
152 unsigned char short_len : 3;
153 unsigned char pad2 : 1;
154} __packed;
155
156struct pdma_data_descr {
157 unsigned int len : 24;
158 unsigned int buf : 32;
159} __packed;
160
161struct pdma_short_descr {
162 unsigned char data[7];
163} __packed;
164
165struct pdma_descr {
166 struct pdma_descr_ctrl ctrl;
167 union {
168 struct pdma_data_descr data;
169 struct pdma_short_descr shrt;
170 };
171};
172
173struct pdma_stat_descr {
174 unsigned char pad1 : 1;
175 unsigned char pad2 : 1;
176 unsigned char eop : 1;
177 unsigned char pad3 : 5;
178 unsigned int len : 24;
179};
180
181/* Each descriptor array can hold max 64 entries */
182#define PDMA_DESCR_COUNT 64
183
184#define MODULE_NAME "Artpec-6 CA"
185
186/* Hash modes (including HMAC variants) */
187#define ARTPEC6_CRYPTO_HASH_SHA1 1
188#define ARTPEC6_CRYPTO_HASH_SHA256 2
a21eb94f
LP		 189
190/* Crypto modes */
191#define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1
192#define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2
193#define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3
194#define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5
195
196/* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
197 * It operates on a descriptor array with up to 64 descriptor entries.
198 * The arrays must be 64 byte aligned in memory.
199 *
200 * The ciphering unit has no registers and is completely controlled by
201 * a 4-byte metadata that is inserted at the beginning of each dma packet.
202 *
203 * A dma packet is a sequence of descriptors terminated by setting the .eop
204 * field in the final descriptor of the packet.
205 *
206 * Multiple packets are used for providing context data, key data and
207 * the plain/ciphertext.
208 *
209 * PDMA Descriptors (Array)
210 * +------+------+------+~~+-------+------+----
211 * | 0 | 1 | 2 |~~| 11 EOP| 12 | ....
212 * +--+---+--+---+----+-+~~+-------+----+-+----
213 * | | | | |
214 * | | | | |
215 * __|__ +-------++-------++-------+ +----+
216 * | MD | |Payload||Payload||Payload| | MD |
217 * +-----+ +-------++-------++-------+ +----+
218 */
219
220struct artpec6_crypto_bounce_buffer {
221 struct list_head list;
222 size_t length;
223 struct scatterlist *sg;
224 size_t offset;
225 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and
226 * holds up to ARTPEC_CACHE_LINE_MAX bytes data.
227 */
228 void *buf;
229};
230
231struct artpec6_crypto_dma_map {
232 dma_addr_t dma_addr;
233 size_t size;
234 enum dma_data_direction dir;
235};
236
237struct artpec6_crypto_dma_descriptors {
238 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64);
239 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64);
240 u32 stat[PDMA_DESCR_COUNT] __aligned(64);
241 struct list_head bounce_buffers;
242 /* Enough maps for all out/in buffers, and all three descr. arrays */
243 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2];
244 dma_addr_t out_dma_addr;
245 dma_addr_t in_dma_addr;
246 dma_addr_t stat_dma_addr;
247 size_t out_cnt;
248 size_t in_cnt;
249 size_t map_count;
250};
251
252enum artpec6_crypto_variant {
253 ARTPEC6_CRYPTO,
254 ARTPEC7_CRYPTO,
255};
256
257struct artpec6_crypto {
258 void __iomem *base;
259 spinlock_t queue_lock;
260 struct list_head queue; /* waiting for pdma fifo space */
261 struct list_head pending; /* submitted to pdma fifo */
262 struct tasklet_struct task;
263 struct kmem_cache *dma_cache;
264 int pending_count;
265 struct timer_list timer;
266 enum artpec6_crypto_variant variant;
267 void *pad_buffer; /* cache-aligned block padding buffer */
268 void *zero_buffer;
269};
270
271enum artpec6_crypto_hash_flags {
272 HASH_FLAG_INIT_CTX = 2,
273 HASH_FLAG_UPDATE = 4,
274 HASH_FLAG_FINALIZE = 8,
275 HASH_FLAG_HMAC = 16,
276 HASH_FLAG_UPDATE_KEY = 32,
277};
278
279struct artpec6_crypto_req_common {
280 struct list_head list;
c34a8382 281 struct list_head complete_in_progress;
a21eb94f
LP		 282 struct artpec6_crypto_dma_descriptors *dma;
283 struct crypto_async_request *req;
284 void (*complete)(struct crypto_async_request *req);
285 gfp_t gfp_flags;
286};
287
288struct artpec6_hash_request_context {
f68deeba
LP		 289 char partial_buffer[SHA256_BLOCK_SIZE];
290 char partial_buffer_out[SHA256_BLOCK_SIZE];
291 char key_buffer[SHA256_BLOCK_SIZE];
292 char pad_buffer[SHA256_BLOCK_SIZE + 32];
293 unsigned char digeststate[SHA256_DIGEST_SIZE];
a21eb94f
LP		 294 size_t partial_bytes;
295 u64 digcnt;
296 u32 key_md;
297 u32 hash_md;
298 enum artpec6_crypto_hash_flags hash_flags;
299 struct artpec6_crypto_req_common common;
300};
301
302struct artpec6_hash_export_state {
f68deeba
LP		 303 char partial_buffer[SHA256_BLOCK_SIZE];
304 unsigned char digeststate[SHA256_DIGEST_SIZE];
a21eb94f
LP		 305 size_t partial_bytes;
306 u64 digcnt;
307 int oper;
308 unsigned int hash_flags;
309};
310
311struct artpec6_hashalg_context {
f68deeba 312 char hmac_key[SHA256_BLOCK_SIZE];
a21eb94f
LP		 313 size_t hmac_key_length;
314 struct crypto_shash *child_hash;
315};
316
317struct artpec6_crypto_request_context {
318 u32 cipher_md;
319 bool decrypt;
320 struct artpec6_crypto_req_common common;
321};
322
323struct artpec6_cryptotfm_context {
324 unsigned char aes_key[2*AES_MAX_KEY_SIZE];
325 size_t key_length;
326 u32 key_md;
327 int crypto_type;
888a649c 328 struct crypto_sync_skcipher *fallback;
a21eb94f
LP		 329};
330
331struct artpec6_crypto_aead_hw_ctx {
332 __be64 aad_length_bits;
333 __be64 text_length_bits;
334 __u8 J0[AES_BLOCK_SIZE];
335};
336
337struct artpec6_crypto_aead_req_ctx {
338 struct artpec6_crypto_aead_hw_ctx hw_ctx;
339 u32 cipher_md;
340 bool decrypt;
341 struct artpec6_crypto_req_common common;
342 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned;
343};
344
345/* The crypto framework makes it hard to avoid this global. */
346static struct device *artpec6_crypto_dev;
347
a21eb94f
LP		 348#ifdef CONFIG_FAULT_INJECTION
349static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read);
350static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full);
351#endif
352
353enum {
354 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START,
355 ARTPEC6_CRYPTO_PREPARE_HASH_START,
356};
357
358static int artpec6_crypto_prepare_aead(struct aead_request *areq);
359static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq);
360static int artpec6_crypto_prepare_hash(struct ahash_request *areq);
361
362static void
363artpec6_crypto_complete_crypto(struct crypto_async_request *req);
364static void
365artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req);
366static void
367artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req);
368static void
369artpec6_crypto_complete_aead(struct crypto_async_request *req);
370static void
371artpec6_crypto_complete_hash(struct crypto_async_request *req);
372
373static int
374artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common);
375
376static void
377artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common);
378
379struct artpec6_crypto_walk {
380 struct scatterlist *sg;
381 size_t offset;
382};
383
384static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk,
385 struct scatterlist *sg)
386{
387 awalk->sg = sg;
388 awalk->offset = 0;
389}
390
391static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk,
392 size_t nbytes)
393{
394 while (nbytes && awalk->sg) {
395 size_t piece;
396
397 WARN_ON(awalk->offset > awalk->sg->length);
398
399 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset);
400 nbytes -= piece;
401 awalk->offset += piece;
402 if (awalk->offset == awalk->sg->length) {
403 awalk->sg = sg_next(awalk->sg);
404 awalk->offset = 0;
405 }
406
407 }
408
409 return nbytes;
410}
411
412static size_t
413artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk)
414{
415 WARN_ON(awalk->sg->length == awalk->offset);
416
417 return awalk->sg->length - awalk->offset;
418}
419
420static dma_addr_t
421artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk)
422{
423 return sg_phys(awalk->sg) + awalk->offset;
424}
425
426static void
427artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common)
428{
429 struct artpec6_crypto_dma_descriptors *dma = common->dma;
430 struct artpec6_crypto_bounce_buffer *b;
431 struct artpec6_crypto_bounce_buffer *next;
432
433 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
434 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n",
435 b, b->length, b->offset, b->buf);
436 sg_pcopy_from_buffer(b->sg,
437 1,
438 b->buf,
439 b->length,
440 b->offset);
441
442 list_del(&b->list);
443 kfree(b);
444 }
445}
446
447static inline bool artpec6_crypto_busy(void)
448{
449 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
450 int fifo_count = ac->pending_count;
451
452 return fifo_count > 6;
453}
454
455static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req)
456{
457 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
458 int ret = -EBUSY;
459
460 spin_lock_bh(&ac->queue_lock);
461
462 if (!artpec6_crypto_busy()) {
463 list_add_tail(&req->list, &ac->pending);
464 artpec6_crypto_start_dma(req);
465 ret = -EINPROGRESS;
466 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
467 list_add_tail(&req->list, &ac->queue);
468 } else {
469 artpec6_crypto_common_destroy(req);
470 }
471
472 spin_unlock_bh(&ac->queue_lock);
473
474 return ret;
475}
476
477static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common)
478{
479 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
480 enum artpec6_crypto_variant variant = ac->variant;
481 void __iomem *base = ac->base;
482 struct artpec6_crypto_dma_descriptors *dma = common->dma;
483 u32 ind, statd, outd;
484
485 /* Make descriptor content visible to the DMA before starting it. */
486 wmb();
487
488 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |
489 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6);
490
491 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |
492 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6);
493
494 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |
495 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6);
496
497 if (variant == ARTPEC6_CRYPTO) {
498 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH);
499 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH);
500 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD);
501 } else {
502 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH);
503 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH);
504 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD);
505 }
506
507 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH);
508 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD);
509
510 ac->pending_count++;
511}
512
513static void
514artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common)
515{
516 struct artpec6_crypto_dma_descriptors *dma = common->dma;
517
518 dma->out_cnt = 0;
519 dma->in_cnt = 0;
520 dma->map_count = 0;
521 INIT_LIST_HEAD(&dma->bounce_buffers);
522}
523
524static bool fault_inject_dma_descr(void)
525{
526#ifdef CONFIG_FAULT_INJECTION
527 return should_fail(&artpec6_crypto_fail_dma_array_full, 1);
528#else
529 return false;
530#endif
531}
532
533/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
534 * physical address
535 *
536 * @addr: The physical address of the data buffer
537 * @len: The length of the data buffer
538 * @eop: True if this is the last buffer in the packet
539 *
540 * @return 0 on success or -ENOSPC if there are no more descriptors available
541 */
542static int
543artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common,
544 dma_addr_t addr, size_t len, bool eop)
545{
546 struct artpec6_crypto_dma_descriptors *dma = common->dma;
547 struct pdma_descr *d;
548
549 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
550 fault_inject_dma_descr()) {
551 pr_err("No free OUT DMA descriptors available!\n");
552 return -ENOSPC;
553 }
554
555 d = &dma->out[dma->out_cnt++];
556 memset(d, 0, sizeof(*d));
557
558 d->ctrl.short_descr = 0;
559 d->ctrl.eop = eop;
560 d->data.len = len;
561 d->data.buf = addr;
562 return 0;
563}
564
565/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
566 *
567 * @dst: The virtual address of the data
568 * @len: The length of the data, must be between 1 to 7 bytes
569 * @eop: True if this is the last buffer in the packet
570 *
571 * @return 0 on success
572 * -ENOSPC if no more descriptors are available
573 * -EINVAL if the data length exceeds 7 bytes
574 */
575static int
576artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common,
577 void *dst, unsigned int len, bool eop)
578{
579 struct artpec6_crypto_dma_descriptors *dma = common->dma;
580 struct pdma_descr *d;
581
582 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
583 fault_inject_dma_descr()) {
584 pr_err("No free OUT DMA descriptors available!\n");
585 return -ENOSPC;
586 } else if (len > 7 || len < 1) {
587 return -EINVAL;
588 }
589 d = &dma->out[dma->out_cnt++];
590 memset(d, 0, sizeof(*d));
591
592 d->ctrl.short_descr = 1;
593 d->ctrl.short_len = len;
594 d->ctrl.eop = eop;
595 memcpy(d->shrt.data, dst, len);
596 return 0;
597}
598
599static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common,
600 struct page *page, size_t offset,
601 size_t size,
602 enum dma_data_direction dir,
603 dma_addr_t *dma_addr_out)
604{
605 struct artpec6_crypto_dma_descriptors *dma = common->dma;
606 struct device *dev = artpec6_crypto_dev;
607 struct artpec6_crypto_dma_map *map;
608 dma_addr_t dma_addr;
609
610 *dma_addr_out = 0;
611
612 if (dma->map_count >= ARRAY_SIZE(dma->maps))
613 return -ENOMEM;
614
615 dma_addr = dma_map_page(dev, page, offset, size, dir);
616 if (dma_mapping_error(dev, dma_addr))
617 return -ENOMEM;
618
619 map = &dma->maps[dma->map_count++];
620 map->size = size;
621 map->dma_addr = dma_addr;
622 map->dir = dir;
623
624 *dma_addr_out = dma_addr;
625
626 return 0;
627}
628
629static int
630artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common,
631 void *ptr, size_t size,
632 enum dma_data_direction dir,
633 dma_addr_t *dma_addr_out)
634{
635 struct page *page = virt_to_page(ptr);
636 size_t offset = (uintptr_t)ptr & ~PAGE_MASK;
637
638 return artpec6_crypto_dma_map_page(common, page, offset, size, dir,
639 dma_addr_out);
640}
641
642static int
643artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common)
644{
645 struct artpec6_crypto_dma_descriptors *dma = common->dma;
646 int ret;
647
648 ret = artpec6_crypto_dma_map_single(common, dma->in,
649 sizeof(dma->in[0]) * dma->in_cnt,
650 DMA_TO_DEVICE, &dma->in_dma_addr);
651 if (ret)
652 return ret;
653
654 ret = artpec6_crypto_dma_map_single(common, dma->out,
655 sizeof(dma->out[0]) * dma->out_cnt,
656 DMA_TO_DEVICE, &dma->out_dma_addr);
657 if (ret)
658 return ret;
659
660 /* We only read one stat descriptor */
661 dma->stat[dma->in_cnt - 1] = 0;
662
663 /*
664 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor
665 * to be written.
666 */
667 return artpec6_crypto_dma_map_single(common,
0d1d4824
LP		 668 dma->stat,
669 sizeof(dma->stat[0]) * dma->in_cnt,
a21eb94f
LP		 670 DMA_BIDIRECTIONAL,
671 &dma->stat_dma_addr);
672}
673
674static void
675artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common)
676{
677 struct artpec6_crypto_dma_descriptors *dma = common->dma;
678 struct device *dev = artpec6_crypto_dev;
679 int i;
680
681 for (i = 0; i < dma->map_count; i++) {
682 struct artpec6_crypto_dma_map *map = &dma->maps[i];
683
684 dma_unmap_page(dev, map->dma_addr, map->size, map->dir);
685 }
686
687 dma->map_count = 0;
688}
689
690/** artpec6_crypto_setup_out_descr - Setup an out descriptor
691 *
692 * @dst: The virtual address of the data
693 * @len: The length of the data
694 * @eop: True if this is the last buffer in the packet
695 * @use_short: If this is true and the data length is 7 bytes or less then
696 * a short descriptor will be used
697 *
698 * @return 0 on success
699 * Any errors from artpec6_crypto_setup_out_descr_short() or
700 * setup_out_descr_phys()
701 */
702static int
703artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
704 void *dst, unsigned int len, bool eop,
705 bool use_short)
706{
707 if (use_short && len < 7) {
708 return artpec6_crypto_setup_out_descr_short(common, dst, len,
709 eop);
710 } else {
711 int ret;
712 dma_addr_t dma_addr;
713
714 ret = artpec6_crypto_dma_map_single(common, dst, len,
715 DMA_TO_DEVICE,
716 &dma_addr);
717 if (ret)
718 return ret;
719
720 return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
721 len, eop);
722 }
723}
724
725/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
726 * physical address
727 *
728 * @addr: The physical address of the data buffer
729 * @len: The length of the data buffer
730 * @intr: True if an interrupt should be fired after HW processing of this
731 * descriptor
732 *
733 */
734static int
735artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common,
736 dma_addr_t addr, unsigned int len, bool intr)
737{
738 struct artpec6_crypto_dma_descriptors *dma = common->dma;
739 struct pdma_descr *d;
740
741 if (dma->in_cnt >= PDMA_DESCR_COUNT ||
742 fault_inject_dma_descr()) {
743 pr_err("No free IN DMA descriptors available!\n");
744 return -ENOSPC;
745 }
746 d = &dma->in[dma->in_cnt++];
747 memset(d, 0, sizeof(*d));
748
749 d->ctrl.intr = intr;
750 d->data.len = len;
751 d->data.buf = addr;
752 return 0;
753}
754
755/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
756 *
757 * @buffer: The virtual address to of the data buffer
758 * @len: The length of the data buffer
759 * @last: If this is the last data buffer in the request (i.e. an interrupt
760 * is needed
761 *
762 * Short descriptors are not used for the in channel
763 */
764static int
765artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common,
766 void *buffer, unsigned int len, bool last)
767{
768 dma_addr_t dma_addr;
769 int ret;
770
771 ret = artpec6_crypto_dma_map_single(common, buffer, len,
772 DMA_FROM_DEVICE, &dma_addr);
773 if (ret)
774 return ret;
775
776 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last);
777}
778
779static struct artpec6_crypto_bounce_buffer *
780artpec6_crypto_alloc_bounce(gfp_t flags)
781{
782 void *base;
783 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) +
784 2 * ARTPEC_CACHE_LINE_MAX;
785 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags);
786
787 if (!bbuf)
788 return NULL;
789
790 base = bbuf + 1;
791 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX);
792 return bbuf;
793}
794
795static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common,
796 struct artpec6_crypto_walk *walk, size_t size)
797{
798 struct artpec6_crypto_bounce_buffer *bbuf;
799 int ret;
800
801 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags);
802 if (!bbuf)
803 return -ENOMEM;
804
805 bbuf->length = size;
806 bbuf->sg = walk->sg;
807 bbuf->offset = walk->offset;
808
809 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false);
810 if (ret) {
811 kfree(bbuf);
812 return ret;
813 }
814
815 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset);
816 list_add_tail(&bbuf->list, &common->dma->bounce_buffers);
817 return 0;
818}
819
820static int
821artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common,
822 struct artpec6_crypto_walk *walk,
823 size_t count)
824{
825 size_t chunk;
826 int ret;
827 dma_addr_t addr;
828
829 while (walk->sg && count) {
830 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
831 addr = artpec6_crypto_walk_chunk_phys(walk);
832
833 /* When destination buffers are not aligned to the cache line
834 * size we need bounce buffers. The DMA-API requires that the
835 * entire line is owned by the DMA buffer and this holds also
836 * for the case when coherent DMA is used.
837 */
838 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) {
839 chunk = min_t(dma_addr_t, chunk,
840 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) -
841 addr);
842
843 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
844 ret = setup_bounce_buffer_in(common, walk, chunk);
845 } else if (chunk < ARTPEC_CACHE_LINE_MAX) {
846 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
847 ret = setup_bounce_buffer_in(common, walk, chunk);
848 } else {
849 dma_addr_t dma_addr;
850
851 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);
852
853 pr_debug("CHUNK %pad:%zu\n", &addr, chunk);
854
855 ret = artpec6_crypto_dma_map_page(common,
856 sg_page(walk->sg),
857 walk->sg->offset +
858 walk->offset,
859 chunk,
860 DMA_FROM_DEVICE,
861 &dma_addr);
862 if (ret)
863 return ret;
864
865 ret = artpec6_crypto_setup_in_descr_phys(common,
866 dma_addr,
867 chunk, false);
868 }
869
870 if (ret)
871 return ret;
872
873 count = count - chunk;
874 artpec6_crypto_walk_advance(walk, chunk);
875 }
876
877 if (count)
878 pr_err("EOL unexpected %zu bytes left\n", count);
879
880 return count ? -EINVAL : 0;
881}
882
883static int
884artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common,
885 struct artpec6_crypto_walk *walk,
886 size_t count)
887{
888 size_t chunk;
889 int ret;
890 dma_addr_t addr;
891
892 while (walk->sg && count) {
893 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
894 addr = artpec6_crypto_walk_chunk_phys(walk);
895
896 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk);
897
898 if (addr & 3) {
899 char buf[3];
900
901 chunk = min_t(size_t, chunk, (4-(addr&3)));
902
903 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,
904 walk->offset);
905
906 ret = artpec6_crypto_setup_out_descr_short(common, buf,
907 chunk,
908 false);
909 } else {
910 dma_addr_t dma_addr;
911
912 ret = artpec6_crypto_dma_map_page(common,
913 sg_page(walk->sg),
914 walk->sg->offset +
915 walk->offset,
916 chunk,
917 DMA_TO_DEVICE,
918 &dma_addr);
919 if (ret)
920 return ret;
921
922 ret = artpec6_crypto_setup_out_descr_phys(common,
923 dma_addr,
924 chunk, false);
925 }
926
927 if (ret)
928 return ret;
929
930 count = count - chunk;
931 artpec6_crypto_walk_advance(walk, chunk);
932 }
933
934 if (count)
935 pr_err("EOL unexpected %zu bytes left\n", count);
936
937 return count ? -EINVAL : 0;
938}
939
940
941/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
942 *
943 * If the out descriptor list is non-empty, then the eop flag on the
944 * last used out descriptor will be set.
945 *
946 * @return 0 on success
947 * -EINVAL if the out descriptor is empty or has overflown
948 */
949static int
950artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common)
951{
952 struct artpec6_crypto_dma_descriptors *dma = common->dma;
953 struct pdma_descr *d;
954
955 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) {
956 pr_err("%s: OUT descriptor list is %s\n",
957 MODULE_NAME, dma->out_cnt ? "empty" : "full");
958 return -EINVAL;
959
960 }
961
962 d = &dma->out[dma->out_cnt-1];
963 d->ctrl.eop = 1;
964
965 return 0;
966}
967
968/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
969 * in descriptor
970 *
971 * See artpec6_crypto_terminate_out_descrs() for return values
972 */
973static int
974artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common)
975{
976 struct artpec6_crypto_dma_descriptors *dma = common->dma;
977 struct pdma_descr *d;
978
979 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) {
980 pr_err("%s: IN descriptor list is %s\n",
981 MODULE_NAME, dma->in_cnt ? "empty" : "full");
982 return -EINVAL;
983 }
984
985 d = &dma->in[dma->in_cnt-1];
986 d->ctrl.intr = 1;
987 return 0;
988}
989
990/** create_hash_pad - Create a Secure Hash conformant pad
991 *
992 * @dst: The destination buffer to write the pad. Must be at least 64 bytes
993 * @dgstlen: The total length of the hash digest in bytes
994 * @bitcount: The total length of the digest in bits
995 *
996 * @return The total number of padding bytes written to @dst
997 */
998static size_t
999create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount)
1000{
1001 unsigned int mod, target, diff, pad_bytes, size_bytes;
1002 __be64 bits = __cpu_to_be64(bitcount);
1003
1004 switch (oper) {
1005 case regk_crypto_sha1:
1006 case regk_crypto_sha256:
1007 case regk_crypto_hmac_sha1:
1008 case regk_crypto_hmac_sha256:
1009 target = 448 / 8;
1010 mod = 512 / 8;
1011 size_bytes = 8;
1012 break;
1013 default:
1014 target = 896 / 8;
1015 mod = 1024 / 8;
1016 size_bytes = 16;
1017 break;
1018 }
1019
1020 target -= 1;
1021 diff = dgstlen & (mod - 1);
1022 pad_bytes = diff > target ? target + mod - diff : target - diff;
1023
1024 memset(dst + 1, 0, pad_bytes);
1025 dst[0] = 0x80;
1026
1027 if (size_bytes == 16) {
1028 memset(dst + 1 + pad_bytes, 0, 8);
1029 memcpy(dst + 1 + pad_bytes + 8, &bits, 8);
1030 } else {
1031 memcpy(dst + 1 + pad_bytes, &bits, 8);
1032 }
1033
1034 return pad_bytes + size_bytes + 1;
1035}
1036
1037static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common,
1038 struct crypto_async_request *parent,
1039 void (*complete)(struct crypto_async_request *req),
1040 struct scatterlist *dstsg, unsigned int nbytes)
1041{
1042 gfp_t flags;
1043 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1044
1045 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1046 GFP_KERNEL : GFP_ATOMIC;
1047
1048 common->gfp_flags = flags;
1049 common->dma = kmem_cache_alloc(ac->dma_cache, flags);
1050 if (!common->dma)
1051 return -ENOMEM;
1052
1053 common->req = parent;
1054 common->complete = complete;
1055 return 0;
1056}
1057
1058static void
1059artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma)
1060{
1061 struct artpec6_crypto_bounce_buffer *b;
1062 struct artpec6_crypto_bounce_buffer *next;
1063
1064 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
1065 kfree(b);
1066 }
1067}
1068
1069static int
1070artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common)
1071{
1072 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1073
1074 artpec6_crypto_dma_unmap_all(common);
1075 artpec6_crypto_bounce_destroy(common->dma);
1076 kmem_cache_free(ac->dma_cache, common->dma);
1077 common->dma = NULL;
1078 return 0;
1079}
1080
1081/*
1082 * Ciphering functions.
1083 */
1084static int artpec6_crypto_encrypt(struct skcipher_request *req)
1085{
1086 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1087 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1088 struct artpec6_crypto_request_context *req_ctx = NULL;
1089 void (*complete)(struct crypto_async_request *req);
1090 int ret;
1091
1092 req_ctx = skcipher_request_ctx(req);
1093
1094 switch (ctx->crypto_type) {
1095 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1096 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1097 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1098 req_ctx->decrypt = 0;
1099 break;
1100 default:
1101 break;
1102 }
1103
1104 switch (ctx->crypto_type) {
1105 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1106 complete = artpec6_crypto_complete_cbc_encrypt;
1107 break;
1108 default:
1109 complete = artpec6_crypto_complete_crypto;
1110 break;
1111 }
1112
1113 ret = artpec6_crypto_common_init(&req_ctx->common,
1114 &req->base,
1115 complete,
1116 req->dst, req->cryptlen);
1117 if (ret)
1118 return ret;
1119
1120 ret = artpec6_crypto_prepare_crypto(req);
1121 if (ret) {
1122 artpec6_crypto_common_destroy(&req_ctx->common);
1123 return ret;
1124 }
1125
1126 return artpec6_crypto_submit(&req_ctx->common);
1127}
1128
1129static int artpec6_crypto_decrypt(struct skcipher_request *req)
1130{
1131 int ret;
1132 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1133 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1134 struct artpec6_crypto_request_context *req_ctx = NULL;
1135 void (*complete)(struct crypto_async_request *req);
1136
1137 req_ctx = skcipher_request_ctx(req);
1138
1139 switch (ctx->crypto_type) {
1140 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1141 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1142 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1143 req_ctx->decrypt = 1;
1144 break;
1145 default:
1146 break;
1147 }
1148
1149
1150 switch (ctx->crypto_type) {
1151 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1152 complete = artpec6_crypto_complete_cbc_decrypt;
1153 break;
1154 default:
1155 complete = artpec6_crypto_complete_crypto;
1156 break;
1157 }
1158
1159 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1160 complete,
1161 req->dst, req->cryptlen);
1162 if (ret)
1163 return ret;
1164
1165 ret = artpec6_crypto_prepare_crypto(req);
1166 if (ret) {
1167 artpec6_crypto_common_destroy(&req_ctx->common);
1168 return ret;
1169 }
1170
1171 return artpec6_crypto_submit(&req_ctx->common);
1172}
1173
1174static int
1175artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
1176{
1177 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1178 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1179 size_t iv_len = crypto_skcipher_ivsize(cipher);
1180 unsigned int counter = be32_to_cpup((__be32 *)
1181 (req->iv + iv_len - 4));
1182 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
1183 AES_BLOCK_SIZE;
1184
1185 /*
1186 * The hardware uses only the last 32-bits as the counter while the
1187 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
1188 * the whole IV is a counter. So fallback if the counter is going to
1189 * overlow.
1190 */
1191 if (counter + nblks < counter) {
1192 int ret;
1193
1194 pr_debug("counter %x will overflow (nblks %u), falling back\n",
1195 counter, counter + nblks);
1196
888a649c
KC		 1197 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key,
1198 ctx->key_length);
a21eb94f
LP		 1199 if (ret)
1200 return ret;
1201
1202 {
888a649c 1203 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
a21eb94f 1204
888a649c 1205 skcipher_request_set_sync_tfm(subreq, ctx->fallback);
a21eb94f
LP		 1206 skcipher_request_set_callback(subreq, req->base.flags,
1207 NULL, NULL);
1208 skcipher_request_set_crypt(subreq, req->src, req->dst,
1209 req->cryptlen, req->iv);
1210 ret = encrypt ? crypto_skcipher_encrypt(subreq)
1211 : crypto_skcipher_decrypt(subreq);
1212 skcipher_request_zero(subreq);
1213 }
1214 return ret;
1215 }
1216
1217 return encrypt ? artpec6_crypto_encrypt(req)
1218 : artpec6_crypto_decrypt(req);
1219}
1220
1221static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req)
1222{
1223 return artpec6_crypto_ctr_crypt(req, true);
1224}
1225
1226static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req)
1227{
1228 return artpec6_crypto_ctr_crypt(req, false);
1229}
1230
1231/*
1232 * AEAD functions
1233 */
1234static int artpec6_crypto_aead_init(struct crypto_aead *tfm)
1235{
1236 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm);
1237
1238 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
1239
1240 crypto_aead_set_reqsize(tfm,
1241 sizeof(struct artpec6_crypto_aead_req_ctx));
1242
1243 return 0;
1244}
1245
1246static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key,
1247 unsigned int len)
1248{
1249 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base);
1250
1251 if (len != 16 && len != 24 && len != 32) {
1252 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1253 return -1;
1254 }
1255
1256 ctx->key_length = len;
1257
1258 memcpy(ctx->aes_key, key, len);
1259 return 0;
1260}
1261
1262static int artpec6_crypto_aead_encrypt(struct aead_request *req)
1263{
1264 int ret;
1265 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1266
1267 req_ctx->decrypt = false;
1268 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1269 artpec6_crypto_complete_aead,
1270 NULL, 0);
1271 if (ret)
1272 return ret;
1273
1274 ret = artpec6_crypto_prepare_aead(req);
1275 if (ret) {
1276 artpec6_crypto_common_destroy(&req_ctx->common);
1277 return ret;
1278 }
1279
1280 return artpec6_crypto_submit(&req_ctx->common);
1281}
1282
1283static int artpec6_crypto_aead_decrypt(struct aead_request *req)
1284{
1285 int ret;
1286 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1287
1288 req_ctx->decrypt = true;
1289 if (req->cryptlen < AES_BLOCK_SIZE)
1290 return -EINVAL;
1291
1292 ret = artpec6_crypto_common_init(&req_ctx->common,
1293 &req->base,
1294 artpec6_crypto_complete_aead,
1295 NULL, 0);
1296 if (ret)
1297 return ret;
1298
1299 ret = artpec6_crypto_prepare_aead(req);
1300 if (ret) {
1301 artpec6_crypto_common_destroy(&req_ctx->common);
1302 return ret;
1303 }
1304
1305 return artpec6_crypto_submit(&req_ctx->common);
1306}
1307
1308static int artpec6_crypto_prepare_hash(struct ahash_request *areq)
1309{
1310 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1311 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq);
1312 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
971108c3 1313 size_t contextsize = digestsize;
a21eb94f
LP		 1314 size_t blocksize = crypto_tfm_alg_blocksize(
1315 crypto_ahash_tfm(crypto_ahash_reqtfm(areq)));
1316 struct artpec6_crypto_req_common *common = &req_ctx->common;
1317 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1318 enum artpec6_crypto_variant variant = ac->variant;
1319 u32 sel_ctx;
1320 bool ext_ctx = false;
1321 bool run_hw = false;
1322 int error = 0;
1323
1324 artpec6_crypto_init_dma_operation(common);
1325
1326 /* Upload HMAC key, must be first the first packet */
1327 if (req_ctx->hash_flags & HASH_FLAG_HMAC) {
1328 if (variant == ARTPEC6_CRYPTO) {
1329 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1330 a6_regk_crypto_dlkey);
1331 } else {
1332 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1333 a7_regk_crypto_dlkey);
1334 }
1335
1336 /* Copy and pad up the key */
1337 memcpy(req_ctx->key_buffer, ctx->hmac_key,
1338 ctx->hmac_key_length);
1339 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0,
1340 blocksize - ctx->hmac_key_length);
1341
1342 error = artpec6_crypto_setup_out_descr(common,
1343 (void *)&req_ctx->key_md,
1344 sizeof(req_ctx->key_md), false, false);
1345 if (error)
1346 return error;
1347
1348 error = artpec6_crypto_setup_out_descr(common,
1349 req_ctx->key_buffer, blocksize,
1350 true, false);
1351 if (error)
1352 return error;
1353 }
1354
1355 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) {
1356 /* Restore context */
1357 sel_ctx = regk_crypto_ext;
1358 ext_ctx = true;
1359 } else {
1360 sel_ctx = regk_crypto_init;
1361 }
1362
1363 if (variant == ARTPEC6_CRYPTO) {
1364 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX;
1365 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx);
1366
1367 /* If this is the final round, set the final flag */
1368 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1369 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN;
1370 } else {
1371 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX;
1372 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx);
1373
1374 /* If this is the final round, set the final flag */
1375 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1376 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN;
1377 }
1378
1379 /* Setup up metadata descriptors */
1380 error = artpec6_crypto_setup_out_descr(common,
1381 (void *)&req_ctx->hash_md,
1382 sizeof(req_ctx->hash_md), false, false);
1383 if (error)
1384 return error;
1385
1386 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1387 if (error)
1388 return error;
1389
1390 if (ext_ctx) {
1391 error = artpec6_crypto_setup_out_descr(common,
1392 req_ctx->digeststate,
1393 contextsize, false, false);
1394
1395 if (error)
1396 return error;
1397 }
1398
1399 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) {
1400 size_t done_bytes = 0;
1401 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes;
1402 size_t ready_bytes = round_down(total_bytes, blocksize);
1403 struct artpec6_crypto_walk walk;
1404
1405 run_hw = ready_bytes > 0;
1406 if (req_ctx->partial_bytes && ready_bytes) {
1407 /* We have a partial buffer and will at least some bytes
1408 * to the HW. Empty this partial buffer before tackling
1409 * the SG lists
1410 */
1411 memcpy(req_ctx->partial_buffer_out,
1412 req_ctx->partial_buffer,
1413 req_ctx->partial_bytes);
1414
1415 error = artpec6_crypto_setup_out_descr(common,
1416 req_ctx->partial_buffer_out,
1417 req_ctx->partial_bytes,
1418 false, true);
1419 if (error)
1420 return error;
1421
1422 /* Reset partial buffer */
1423 done_bytes += req_ctx->partial_bytes;
1424 req_ctx->partial_bytes = 0;
1425 }
1426
1427 artpec6_crypto_walk_init(&walk, areq->src);
1428
1429 error = artpec6_crypto_setup_sg_descrs_out(common, &walk,
1430 ready_bytes -
1431 done_bytes);
1432 if (error)
1433 return error;
1434
1435 if (walk.sg) {
1436 size_t sg_skip = ready_bytes - done_bytes;
1437 size_t sg_rem = areq->nbytes - sg_skip;
1438
1439 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
1440 req_ctx->partial_buffer +
1441 req_ctx->partial_bytes,
1442 sg_rem, sg_skip);
1443
1444 req_ctx->partial_bytes += sg_rem;
1445 }
1446
1447 req_ctx->digcnt += ready_bytes;
1448 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE);
1449 }
1450
1451 /* Finalize */
1452 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) {
a21eb94f
LP		 1453 size_t hash_pad_len;
1454 u64 digest_bits;
1455 u32 oper;
1456
1457 if (variant == ARTPEC6_CRYPTO)
1458 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md);
1459 else
1460 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md);
1461
1462 /* Write out the partial buffer if present */
1463 if (req_ctx->partial_bytes) {
1464 memcpy(req_ctx->partial_buffer_out,
1465 req_ctx->partial_buffer,
1466 req_ctx->partial_bytes);
1467 error = artpec6_crypto_setup_out_descr(common,
1468 req_ctx->partial_buffer_out,
1469 req_ctx->partial_bytes,
1470 false, true);
1471 if (error)
1472 return error;
1473
1474 req_ctx->digcnt += req_ctx->partial_bytes;
1475 req_ctx->partial_bytes = 0;
1476 }
1477
1478 if (req_ctx->hash_flags & HASH_FLAG_HMAC)
1479 digest_bits = 8 * (req_ctx->digcnt + blocksize);
1480 else
1481 digest_bits = 8 * req_ctx->digcnt;
1482
1483 /* Add the hash pad */
1484 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer,
1485 req_ctx->digcnt, digest_bits);
1486 error = artpec6_crypto_setup_out_descr(common,
1487 req_ctx->pad_buffer,
1488 hash_pad_len, false,
1489 true);
1490 req_ctx->digcnt = 0;
1491
1492 if (error)
1493 return error;
1494
1495 /* Descriptor for the final result */
1496 error = artpec6_crypto_setup_in_descr(common, areq->result,
1497 digestsize,
971108c3 1498 true);
a21eb94f
LP		 1499 if (error)
1500 return error;
1501
a21eb94f
LP		 1502 } else { /* This is not the final operation for this request */
1503 if (!run_hw)
1504 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START;
1505
1506 /* Save the result to the context */
1507 error = artpec6_crypto_setup_in_descr(common,
1508 req_ctx->digeststate,
1509 contextsize, false);
1510 if (error)
1511 return error;
1512 /* fall through */
1513 }
1514
1515 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE |
1516 HASH_FLAG_FINALIZE);
1517
1518 error = artpec6_crypto_terminate_in_descrs(common);
1519 if (error)
1520 return error;
1521
1522 error = artpec6_crypto_terminate_out_descrs(common);
1523 if (error)
1524 return error;
1525
1526 error = artpec6_crypto_dma_map_descs(common);
1527 if (error)
1528 return error;
1529
1530 return ARTPEC6_CRYPTO_PREPARE_HASH_START;
1531}
1532
1533
1534static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm)
1535{
1536 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1537
1538 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1539 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB;
1540
1541 return 0;
1542}
1543
1544static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
1545{
1546 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1547
888a649c
KC		 1548 ctx->fallback =
1549 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base),
1550 0, CRYPTO_ALG_NEED_FALLBACK);
a21eb94f
LP		 1551 if (IS_ERR(ctx->fallback))
1552 return PTR_ERR(ctx->fallback);
1553
1554 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1555 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR;
1556
1557 return 0;
1558}
1559
1560static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm)
1561{
1562 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1563
1564 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1565 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC;
1566
1567 return 0;
1568}
1569
1570static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm)
1571{
1572 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1573
1574 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1575 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS;
1576
1577 return 0;
1578}
1579
1580static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm)
1581{
1582 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1583
1584 memset(ctx, 0, sizeof(*ctx));
1585}
1586
1587static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
1588{
1589 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1590
888a649c 1591 crypto_free_sync_skcipher(ctx->fallback);
a21eb94f
LP		 1592 artpec6_crypto_aes_exit(tfm);
1593}
1594
1595static int
1596artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key,
1597 unsigned int keylen)
1598{
1599 struct artpec6_cryptotfm_context *ctx =
1600 crypto_skcipher_ctx(cipher);
1601
1602 switch (keylen) {
1603 case 16:
1604 case 24:
1605 case 32:
1606 break;
1607 default:
1608 crypto_skcipher_set_flags(cipher,
1609 CRYPTO_TFM_RES_BAD_KEY_LEN);
1610 return -EINVAL;
1611 }
1612
1613 memcpy(ctx->aes_key, key, keylen);
1614 ctx->key_length = keylen;
1615 return 0;
1616}
1617
1618static int
1619artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key,
1620 unsigned int keylen)
1621{
1622 struct artpec6_cryptotfm_context *ctx =
1623 crypto_skcipher_ctx(cipher);
1624 int ret;
1625
1626 ret = xts_check_key(&cipher->base, key, keylen);
1627 if (ret)
1628 return ret;
1629
1630 switch (keylen) {
1631 case 32:
1632 case 48:
1633 case 64:
1634 break;
1635 default:
1636 crypto_skcipher_set_flags(cipher,
1637 CRYPTO_TFM_RES_BAD_KEY_LEN);
1638 return -EINVAL;
1639 }
1640
1641 memcpy(ctx->aes_key, key, keylen);
1642 ctx->key_length = keylen;
1643 return 0;
1644}
1645
1646/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
1647 *
1648 * @req: The asynch request to process
1649 *
1650 * @return 0 if the dma job was successfully prepared
1651 * <0 on error
1652 *
1653 * This function sets up the PDMA descriptors for a block cipher request.
1654 *
1655 * The required padding is added for AES-CTR using a statically defined
1656 * buffer.
1657 *
1658 * The PDMA descriptor list will be as follows:
1659 *
1660 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
1661 * IN: <CIPHER_MD><data_0>...[data_n]<intr>
1662 *
1663 */
1664static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq)
1665{
1666 int ret;
1667 struct artpec6_crypto_walk walk;
1668 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1669 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1670 struct artpec6_crypto_request_context *req_ctx = NULL;
1671 size_t iv_len = crypto_skcipher_ivsize(cipher);
1672 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1673 enum artpec6_crypto_variant variant = ac->variant;
1674 struct artpec6_crypto_req_common *common;
1675 bool cipher_decr = false;
1676 size_t cipher_klen;
1677 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */
1678 u32 oper;
1679
1680 req_ctx = skcipher_request_ctx(areq);
1681 common = &req_ctx->common;
1682
1683 artpec6_crypto_init_dma_operation(common);
1684
1685 if (variant == ARTPEC6_CRYPTO)
1686 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey);
1687 else
1688 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey);
1689
1690 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1691 sizeof(ctx->key_md), false, false);
1692 if (ret)
1693 return ret;
1694
1695 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1696 ctx->key_length, true, false);
1697 if (ret)
1698 return ret;
1699
1700 req_ctx->cipher_md = 0;
1701
1702 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS)
1703 cipher_klen = ctx->key_length/2;
1704 else
1705 cipher_klen = ctx->key_length;
1706
1707 /* Metadata */
1708 switch (cipher_klen) {
1709 case 16:
1710 cipher_len = regk_crypto_key_128;
1711 break;
1712 case 24:
1713 cipher_len = regk_crypto_key_192;
1714 break;
1715 case 32:
1716 cipher_len = regk_crypto_key_256;
1717 break;
1718 default:
1719 pr_err("%s: Invalid key length %d!\n",
1720 MODULE_NAME, ctx->key_length);
1721 return -EINVAL;
1722 }
1723
1724 switch (ctx->crypto_type) {
1725 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1726 oper = regk_crypto_aes_ecb;
1727 cipher_decr = req_ctx->decrypt;
1728 break;
1729
1730 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1731 oper = regk_crypto_aes_cbc;
1732 cipher_decr = req_ctx->decrypt;
1733 break;
1734
1735 case ARTPEC6_CRYPTO_CIPHER_AES_CTR:
1736 oper = regk_crypto_aes_ctr;
1737 cipher_decr = false;
1738 break;
1739
1740 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1741 oper = regk_crypto_aes_xts;
1742 cipher_decr = req_ctx->decrypt;
1743
1744 if (variant == ARTPEC6_CRYPTO)
1745 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ;
1746 else
1747 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ;
1748 break;
1749
1750 default:
1751 pr_err("%s: Invalid cipher mode %d!\n",
1752 MODULE_NAME, ctx->crypto_type);
1753 return -EINVAL;
1754 }
1755
1756 if (variant == ARTPEC6_CRYPTO) {
1757 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper);
1758 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1759 cipher_len);
1760 if (cipher_decr)
1761 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1762 } else {
1763 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper);
1764 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1765 cipher_len);
1766 if (cipher_decr)
1767 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1768 }
1769
1770 ret = artpec6_crypto_setup_out_descr(common,
1771 &req_ctx->cipher_md,
1772 sizeof(req_ctx->cipher_md),
1773 false, false);
1774 if (ret)
1775 return ret;
1776
1777 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1778 if (ret)
1779 return ret;
1780
1781 if (iv_len) {
1782 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len,
1783 false, false);
1784 if (ret)
1785 return ret;
1786 }
1787 /* Data out */
1788 artpec6_crypto_walk_init(&walk, areq->src);
1789 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen);
1790 if (ret)
1791 return ret;
1792
1793 /* Data in */
1794 artpec6_crypto_walk_init(&walk, areq->dst);
1795 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen);
1796 if (ret)
1797 return ret;
1798
1799 /* CTR-mode padding required by the HW. */
1800 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR ||
1801 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) {
1802 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) -
1803 areq->cryptlen;
1804
1805 if (pad) {
1806 ret = artpec6_crypto_setup_out_descr(common,
1807 ac->pad_buffer,
1808 pad, false, false);
1809 if (ret)
1810 return ret;
1811
1812 ret = artpec6_crypto_setup_in_descr(common,
1813 ac->pad_buffer, pad,
1814 false);
1815 if (ret)
1816 return ret;
1817 }
1818 }
1819
1820 ret = artpec6_crypto_terminate_out_descrs(common);
1821 if (ret)
1822 return ret;
1823
1824 ret = artpec6_crypto_terminate_in_descrs(common);
1825 if (ret)
1826 return ret;
1827
1828 return artpec6_crypto_dma_map_descs(common);
1829}
1830
1831static int artpec6_crypto_prepare_aead(struct aead_request *areq)
1832{
1833 size_t count;
1834 int ret;
1835 size_t input_length;
1836 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1837 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
1838 struct crypto_aead *cipher = crypto_aead_reqtfm(areq);
1839 struct artpec6_crypto_req_common *common = &req_ctx->common;
1840 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1841 enum artpec6_crypto_variant variant = ac->variant;
1842 u32 md_cipher_len;
1843
1844 artpec6_crypto_init_dma_operation(common);
1845
1846 /* Key */
1847 if (variant == ARTPEC6_CRYPTO) {
1848 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1849 a6_regk_crypto_dlkey);
1850 } else {
1851 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1852 a7_regk_crypto_dlkey);
1853 }
1854 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1855 sizeof(ctx->key_md), false, false);
1856 if (ret)
1857 return ret;
1858
1859 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1860 ctx->key_length, true, false);
1861 if (ret)
1862 return ret;
1863
1864 req_ctx->cipher_md = 0;
1865
1866 switch (ctx->key_length) {
1867 case 16:
1868 md_cipher_len = regk_crypto_key_128;
1869 break;
1870 case 24:
1871 md_cipher_len = regk_crypto_key_192;
1872 break;
1873 case 32:
1874 md_cipher_len = regk_crypto_key_256;
1875 break;
1876 default:
1877 return -EINVAL;
1878 }
1879
1880 if (variant == ARTPEC6_CRYPTO) {
1881 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER,
1882 regk_crypto_aes_gcm);
1883 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1884 md_cipher_len);
1885 if (req_ctx->decrypt)
1886 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1887 } else {
1888 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER,
1889 regk_crypto_aes_gcm);
1890 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1891 md_cipher_len);
1892 if (req_ctx->decrypt)
1893 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1894 }
1895
1896 ret = artpec6_crypto_setup_out_descr(common,
1897 (void *) &req_ctx->cipher_md,
1898 sizeof(req_ctx->cipher_md), false,
1899 false);
1900 if (ret)
1901 return ret;
1902
1903 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1904 if (ret)
1905 return ret;
1906
1907 /* For the decryption, cryptlen includes the tag. */
1908 input_length = areq->cryptlen;
1909 if (req_ctx->decrypt)
48ef0908 1910 input_length -= crypto_aead_authsize(cipher);
a21eb94f
LP		 1911
1912 /* Prepare the context buffer */
1913 req_ctx->hw_ctx.aad_length_bits =
1914 __cpu_to_be64(8*areq->assoclen);
1915
1916 req_ctx->hw_ctx.text_length_bits =
1917 __cpu_to_be64(8*input_length);
1918
1919 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
1920 // The HW omits the initial increment of the counter field.
6d6e71fe 1921 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4);
a21eb94f
LP		 1922
1923 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
1924 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
1925 if (ret)
1926 return ret;
1927
1928 {
1929 struct artpec6_crypto_walk walk;
1930
1931 artpec6_crypto_walk_init(&walk, areq->src);
1932
1933 /* Associated data */
1934 count = areq->assoclen;
1935 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1936 if (ret)
1937 return ret;
1938
1939 if (!IS_ALIGNED(areq->assoclen, 16)) {
1940 size_t assoc_pad = 16 - (areq->assoclen % 16);
1941 /* The HW mandates zero padding here */
1942 ret = artpec6_crypto_setup_out_descr(common,
1943 ac->zero_buffer,
1944 assoc_pad, false,
1945 false);
1946 if (ret)
1947 return ret;
1948 }
1949
1950 /* Data to crypto */
1951 count = input_length;
1952 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1953 if (ret)
1954 return ret;
1955
1956 if (!IS_ALIGNED(input_length, 16)) {
1957 size_t crypto_pad = 16 - (input_length % 16);
1958 /* The HW mandates zero padding here */
1959 ret = artpec6_crypto_setup_out_descr(common,
1960 ac->zero_buffer,
1961 crypto_pad,
1962 false,
1963 false);
1964 if (ret)
1965 return ret;
1966 }
1967 }
1968
1969 /* Data from crypto */
1970 {
1971 struct artpec6_crypto_walk walk;
1972 size_t output_len = areq->cryptlen;
1973
1974 if (req_ctx->decrypt)
48ef0908 1975 output_len -= crypto_aead_authsize(cipher);
a21eb94f
LP		 1976
1977 artpec6_crypto_walk_init(&walk, areq->dst);
1978
1979 /* skip associated data in the output */
1980 count = artpec6_crypto_walk_advance(&walk, areq->assoclen);
1981 if (count)
1982 return -EINVAL;
1983
1984 count = output_len;
1985 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count);
1986 if (ret)
1987 return ret;
1988
1989 /* Put padding between the cryptotext and the auth tag */
1990 if (!IS_ALIGNED(output_len, 16)) {
1991 size_t crypto_pad = 16 - (output_len % 16);
1992
1993 ret = artpec6_crypto_setup_in_descr(common,
1994 ac->pad_buffer,
1995 crypto_pad, false);
1996 if (ret)
1997 return ret;
1998 }
1999
2000 /* The authentication tag shall follow immediately after
2001 * the output ciphertext. For decryption it is put in a context
2002 * buffer for later compare against the input tag.
2003 */
a21eb94f
LP		 2004
2005 if (req_ctx->decrypt) {
2006 ret = artpec6_crypto_setup_in_descr(common,
48ef0908 2007 req_ctx->decryption_tag, AES_BLOCK_SIZE, false);
a21eb94f
LP		 2008 if (ret)
2009 return ret;
2010
2011 } else {
48ef0908
LP		 2012 /* For encryption the requested tag size may be smaller
2013 * than the hardware's generated tag.
2014 */
2015 size_t authsize = crypto_aead_authsize(cipher);
2016
a21eb94f 2017 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk,
48ef0908 2018 authsize);
a21eb94f
LP		 2019 if (ret)
2020 return ret;
48ef0908
LP		 2021
2022 if (authsize < AES_BLOCK_SIZE) {
2023 count = AES_BLOCK_SIZE - authsize;
2024 ret = artpec6_crypto_setup_in_descr(common,
2025 ac->pad_buffer,
2026 count, false);
2027 if (ret)
2028 return ret;
2029 }
a21eb94f
LP		 2030 }
2031
2032 }
2033
2034 ret = artpec6_crypto_terminate_in_descrs(common);
2035 if (ret)
2036 return ret;
2037
2038 ret = artpec6_crypto_terminate_out_descrs(common);
2039 if (ret)
2040 return ret;
2041
2042 return artpec6_crypto_dma_map_descs(common);
2043}
2044
c34a8382
LP		 2045static void artpec6_crypto_process_queue(struct artpec6_crypto *ac,
2046 struct list_head *completions)
a21eb94f
LP		 2047{
2048 struct artpec6_crypto_req_common *req;
2049
2050 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {
2051 req = list_first_entry(&ac->queue,
2052 struct artpec6_crypto_req_common,
2053 list);
2054 list_move_tail(&req->list, &ac->pending);
2055 artpec6_crypto_start_dma(req);
2056
c34a8382 2057 list_add_tail(&req->complete_in_progress, completions);
a21eb94f
LP		 2058 }
2059
2060 /*
2061 * In some cases, the hardware can raise an in_eop_flush interrupt
2062 * before actually updating the status, so we have an timer which will
2063 * recheck the status on timeout. Since the cases are expected to be
2064 * very rare, we use a relatively large timeout value. There should be
2065 * no noticeable negative effect if we timeout spuriously.
2066 */
2067 if (ac->pending_count)
2068 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));
2069 else
2070 del_timer(&ac->timer);
2071}
2072
f34d8d50 2073static void artpec6_crypto_timeout(struct timer_list *t)
a21eb94f 2074{
f34d8d50 2075 struct artpec6_crypto *ac = from_timer(ac, t, timer);
a21eb94f
LP		 2076
2077 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n");
2078
2079 tasklet_schedule(&ac->task);
2080}
2081
2082static void artpec6_crypto_task(unsigned long data)
2083{
2084 struct artpec6_crypto *ac = (struct artpec6_crypto *)data;
2085 struct artpec6_crypto_req_common *req;
2086 struct artpec6_crypto_req_common *n;
c34a8382
LP		 2087 struct list_head complete_done;
2088 struct list_head complete_in_progress;
2089
2090 INIT_LIST_HEAD(&complete_done);
2091 INIT_LIST_HEAD(&complete_in_progress);
a21eb94f
LP		 2092
2093 if (list_empty(&ac->pending)) {
2094 pr_debug("Spurious IRQ\n");
2095 return;
2096 }
2097
2098 spin_lock_bh(&ac->queue_lock);
2099
2100 list_for_each_entry_safe(req, n, &ac->pending, list) {
2101 struct artpec6_crypto_dma_descriptors *dma = req->dma;
2102 u32 stat;
0d1d4824 2103 dma_addr_t stataddr;
a21eb94f 2104
0d1d4824
LP		 2105 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1);
2106 dma_sync_single_for_cpu(artpec6_crypto_dev,
2107 stataddr,
2108 4,
a21eb94f
LP		 2109 DMA_BIDIRECTIONAL);
2110
2111 stat = req->dma->stat[req->dma->in_cnt-1];
2112
2113 /* A non-zero final status descriptor indicates
2114 * this job has finished.
2115 */
2116 pr_debug("Request %p status is %X\n", req, stat);
2117 if (!stat)
2118 break;
2119
2120 /* Allow testing of timeout handling with fault injection */
2121#ifdef CONFIG_FAULT_INJECTION
2122 if (should_fail(&artpec6_crypto_fail_status_read, 1))
2123 continue;
2124#endif
2125
2126 pr_debug("Completing request %p\n", req);
2127
c34a8382 2128 list_move_tail(&req->list, &complete_done);
a21eb94f 2129
a21eb94f 2130 ac->pending_count--;
a21eb94f
LP		 2131 }
2132
c34a8382 2133 artpec6_crypto_process_queue(ac, &complete_in_progress);
a21eb94f
LP		 2134
2135 spin_unlock_bh(&ac->queue_lock);
c34a8382
LP		 2136
2137 /* Perform the completion callbacks without holding the queue lock
2138 * to allow new request submissions from the callbacks.
2139 */
2140 list_for_each_entry_safe(req, n, &complete_done, list) {
341a64c7
VW		 2141 artpec6_crypto_dma_unmap_all(req);
2142 artpec6_crypto_copy_bounce_buffers(req);
2143 artpec6_crypto_common_destroy(req);
2144
c34a8382
LP		 2145 req->complete(req->req);
2146 }
2147
2148 list_for_each_entry_safe(req, n, &complete_in_progress,
2149 complete_in_progress) {
2150 req->req->complete(req->req, -EINPROGRESS);
2151 }
a21eb94f
LP		 2152}
2153
2154static void artpec6_crypto_complete_crypto(struct crypto_async_request *req)
2155{
2156 req->complete(req, 0);
2157}
2158
2159static void
2160artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req)
2161{
2162 struct skcipher_request *cipher_req = container_of(req,
2163 struct skcipher_request, base);
2164
2165 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src,
2166 cipher_req->cryptlen - AES_BLOCK_SIZE,
2167 AES_BLOCK_SIZE, 0);
2168 req->complete(req, 0);
2169}
2170
2171static void
2172artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req)
2173{
2174 struct skcipher_request *cipher_req = container_of(req,
2175 struct skcipher_request, base);
2176
2177 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst,
2178 cipher_req->cryptlen - AES_BLOCK_SIZE,
2179 AES_BLOCK_SIZE, 0);
2180 req->complete(req, 0);
2181}
2182
2183static void artpec6_crypto_complete_aead(struct crypto_async_request *req)
2184{
2185 int result = 0;
2186
2187 /* Verify GCM hashtag. */
2188 struct aead_request *areq = container_of(req,
2189 struct aead_request, base);
48ef0908 2190 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
a21eb94f
LP		 2191 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
2192
2193 if (req_ctx->decrypt) {
2194 u8 input_tag[AES_BLOCK_SIZE];
48ef0908 2195 unsigned int authsize = crypto_aead_authsize(aead);
a21eb94f
LP		 2196
2197 sg_pcopy_to_buffer(areq->src,
2198 sg_nents(areq->src),
2199 input_tag,
48ef0908 2200 authsize,
a21eb94f 2201 areq->assoclen + areq->cryptlen -
48ef0908 2202 authsize);
a21eb94f 2203
5997a245
LP		 2204 if (crypto_memneq(req_ctx->decryption_tag,
2205 input_tag,
2206 authsize)) {
a21eb94f
LP		 2207 pr_debug("***EBADMSG:\n");
2208 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,
48ef0908 2209 input_tag, authsize, true);
a21eb94f
LP		 2210 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,
2211 req_ctx->decryption_tag,
48ef0908 2212 authsize, true);
a21eb94f
LP		 2213
2214 result = -EBADMSG;
2215 }
2216 }
2217
2218 req->complete(req, result);
2219}
2220
2221static void artpec6_crypto_complete_hash(struct crypto_async_request *req)
2222{
2223 req->complete(req, 0);
2224}
2225
2226
2227/*------------------- Hash functions -----------------------------------------*/
2228static int
2229artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
2230 const u8 *key, unsigned int keylen)
2231{
2232 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base);
2233 size_t blocksize;
2234 int ret;
2235
2236 if (!keylen) {
2237 pr_err("Invalid length (%d) of HMAC key\n",
2238 keylen);
2239 return -EINVAL;
2240 }
2241
2242 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2243
2244 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2245
2246 if (keylen > blocksize) {
2247 SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash);
2248
2249 hdesc->tfm = tfm_ctx->child_hash;
2250 hdesc->flags = crypto_ahash_get_flags(tfm) &
2251 CRYPTO_TFM_REQ_MAY_SLEEP;
2252
2253 tfm_ctx->hmac_key_length = blocksize;
2254 ret = crypto_shash_digest(hdesc, key, keylen,
2255 tfm_ctx->hmac_key);
2256 if (ret)
2257 return ret;
2258
2259 } else {
2260 memcpy(tfm_ctx->hmac_key, key, keylen);
2261 tfm_ctx->hmac_key_length = keylen;
2262 }
2263
2264 return 0;
2265}
2266
2267static int
2268artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac)
2269{
2270 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2271 enum artpec6_crypto_variant variant = ac->variant;
2272 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2273 u32 oper;
2274
2275 memset(req_ctx, 0, sizeof(*req_ctx));
2276
2277 req_ctx->hash_flags = HASH_FLAG_INIT_CTX;
2278 if (hmac)
2279 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY);
2280
2281 switch (type) {
2282 case ARTPEC6_CRYPTO_HASH_SHA1:
2283 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1;
2284 break;
2285 case ARTPEC6_CRYPTO_HASH_SHA256:
2286 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256;
2287 break;
a21eb94f
LP		 2288 default:
2289 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type);
2290 return -EINVAL;
2291 }
2292
2293 if (variant == ARTPEC6_CRYPTO)
2294 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper);
2295 else
2296 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper);
2297
2298 return 0;
2299}
2300
2301static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req)
2302{
2303 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2304 int ret;
2305
2306 if (!req_ctx->common.dma) {
2307 ret = artpec6_crypto_common_init(&req_ctx->common,
2308 &req->base,
2309 artpec6_crypto_complete_hash,
2310 NULL, 0);
2311
2312 if (ret)
2313 return ret;
2314 }
2315
2316 ret = artpec6_crypto_prepare_hash(req);
2317 switch (ret) {
2318 case ARTPEC6_CRYPTO_PREPARE_HASH_START:
2319 ret = artpec6_crypto_submit(&req_ctx->common);
2320 break;
2321
2322 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START:
2323 ret = 0;
2324 /* Fallthrough */
2325
2326 default:
2327 artpec6_crypto_common_destroy(&req_ctx->common);
2328 break;
2329 }
2330
2331 return ret;
2332}
2333
2334static int artpec6_crypto_hash_final(struct ahash_request *req)
2335{
2336 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2337
2338 req_ctx->hash_flags |= HASH_FLAG_FINALIZE;
2339
2340 return artpec6_crypto_prepare_submit_hash(req);
2341}
2342
2343static int artpec6_crypto_hash_update(struct ahash_request *req)
2344{
2345 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2346
2347 req_ctx->hash_flags |= HASH_FLAG_UPDATE;
2348
2349 return artpec6_crypto_prepare_submit_hash(req);
2350}
2351
2352static int artpec6_crypto_sha1_init(struct ahash_request *req)
2353{
2354 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2355}
2356
2357static int artpec6_crypto_sha1_digest(struct ahash_request *req)
2358{
2359 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2360
2361 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2362
2363 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2364
2365 return artpec6_crypto_prepare_submit_hash(req);
2366}
2367
2368static int artpec6_crypto_sha256_init(struct ahash_request *req)
2369{
2370 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2371}
2372
2373static int artpec6_crypto_sha256_digest(struct ahash_request *req)
2374{
2375 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2376
2377 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2378 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2379
2380 return artpec6_crypto_prepare_submit_hash(req);
2381}
2382
a21eb94f
LP		 2383static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req)
2384{
2385 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2386}
2387
a21eb94f
LP		 2388static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req)
2389{
2390 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2391
2392 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2393 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2394
2395 return artpec6_crypto_prepare_submit_hash(req);
2396}
2397
a21eb94f
LP		 2398static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm,
2399 const char *base_hash_name)
2400{
2401 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2402
2403 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2404 sizeof(struct artpec6_hash_request_context));
2405 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
2406
2407 if (base_hash_name) {
2408 struct crypto_shash *child;
2409
2410 child = crypto_alloc_shash(base_hash_name, 0,
2411 CRYPTO_ALG_NEED_FALLBACK);
2412
2413 if (IS_ERR(child))
2414 return PTR_ERR(child);
2415
2416 tfm_ctx->child_hash = child;
2417 }
2418
2419 return 0;
2420}
2421
2422static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm)
2423{
2424 return artpec6_crypto_ahash_init_common(tfm, NULL);
2425}
2426
2427static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm)
2428{
2429 return artpec6_crypto_ahash_init_common(tfm, "sha256");
2430}
2431
a21eb94f
LP		 2432static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm)
2433{
2434 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2435
2436 if (tfm_ctx->child_hash)
2437 crypto_free_shash(tfm_ctx->child_hash);
2438
2439 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2440 tfm_ctx->hmac_key_length = 0;
2441}
2442
2443static int artpec6_crypto_hash_export(struct ahash_request *req, void *out)
2444{
2445 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2446 struct artpec6_hash_export_state *state = out;
2447 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2448 enum artpec6_crypto_variant variant = ac->variant;
2449
2450 BUILD_BUG_ON(sizeof(state->partial_buffer) !=
2451 sizeof(ctx->partial_buffer));
2452 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate));
2453
2454 state->digcnt = ctx->digcnt;
2455 state->partial_bytes = ctx->partial_bytes;
2456 state->hash_flags = ctx->hash_flags;
2457
2458 if (variant == ARTPEC6_CRYPTO)
2459 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md);
2460 else
2461 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md);
2462
2463 memcpy(state->partial_buffer, ctx->partial_buffer,
2464 sizeof(state->partial_buffer));
2465 memcpy(state->digeststate, ctx->digeststate,
2466 sizeof(state->digeststate));
2467
2468 return 0;
2469}
2470
2471static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in)
2472{
2473 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2474 const struct artpec6_hash_export_state *state = in;
2475 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2476 enum artpec6_crypto_variant variant = ac->variant;
2477
2478 memset(ctx, 0, sizeof(*ctx));
2479
2480 ctx->digcnt = state->digcnt;
2481 ctx->partial_bytes = state->partial_bytes;
2482 ctx->hash_flags = state->hash_flags;
2483
2484 if (variant == ARTPEC6_CRYPTO)
2485 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper);
2486 else
2487 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper);
2488
2489 memcpy(ctx->partial_buffer, state->partial_buffer,
2490 sizeof(state->partial_buffer));
2491 memcpy(ctx->digeststate, state->digeststate,
2492 sizeof(state->digeststate));
2493
2494 return 0;
2495}
2496
2497static int init_crypto_hw(struct artpec6_crypto *ac)
2498{
2499 enum artpec6_crypto_variant variant = ac->variant;
2500 void __iomem *base = ac->base;
2501 u32 out_descr_buf_size;
2502 u32 out_data_buf_size;
2503 u32 in_data_buf_size;
2504 u32 in_descr_buf_size;
2505 u32 in_stat_buf_size;
2506 u32 in, out;
2507
2508 /*
2509 * The PDMA unit contains 1984 bytes of internal memory for the OUT
2510 * channels and 1024 bytes for the IN channel. This is an elastic
2511 * memory used to internally store the descriptors and data. The values
2512 * ares specified in 64 byte incremements. Trustzone buffers are not
2513 * used at this stage.
2514 */
2515 out_data_buf_size = 16; /* 1024 bytes for data */
2516 out_descr_buf_size = 15; /* 960 bytes for descriptors */
2517 in_data_buf_size = 8; /* 512 bytes for data */
2518 in_descr_buf_size = 4; /* 256 bytes for descriptors */
2519 in_stat_buf_size = 4; /* 256 bytes for stat descrs */
2520
2521 BUILD_BUG_ON_MSG((out_data_buf_size
2522 + out_descr_buf_size) * 64 > 1984,
2523 "Invalid OUT configuration");
2524
2525 BUILD_BUG_ON_MSG((in_data_buf_size
2526 + in_descr_buf_size
2527 + in_stat_buf_size) * 64 > 1024,
2528 "Invalid IN configuration");
2529
2530 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) |
2531 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) |
2532 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size);
2533
2534 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) |
2535 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size);
2536
2537 writel_relaxed(out, base + PDMA_OUT_BUF_CFG);
2538 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG);
2539
2540 if (variant == ARTPEC6_CRYPTO) {
2541 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG);
2542 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG);
2543 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA |
2544 A6_PDMA_INTR_MASK_IN_EOP_FLUSH,
2545 base + A6_PDMA_INTR_MASK);
2546 } else {
2547 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG);
2548 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG);
2549 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA |
2550 A7_PDMA_INTR_MASK_IN_EOP_FLUSH,
2551 base + A7_PDMA_INTR_MASK);
2552 }
2553
2554 return 0;
2555}
2556
2557static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)
2558{
2559 enum artpec6_crypto_variant variant = ac->variant;
2560 void __iomem *base = ac->base;
2561
2562 if (variant == ARTPEC6_CRYPTO) {
2563 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD);
2564 writel_relaxed(0, base + A6_PDMA_IN_CFG);
2565 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2566 } else {
2567 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD);
2568 writel_relaxed(0, base + A7_PDMA_IN_CFG);
2569 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2570 }
2571
2572 writel_relaxed(0, base + PDMA_OUT_CFG);
2573
2574}
2575
2576static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id)
2577{
2578 struct artpec6_crypto *ac = dev_id;
2579 enum artpec6_crypto_variant variant = ac->variant;
2580 void __iomem *base = ac->base;
2581 u32 mask_in_data, mask_in_eop_flush;
2582 u32 in_cmd_flush_stat, in_cmd_reg;
2583 u32 ack_intr_reg;
2584 u32 ack = 0;
2585 u32 intr;
2586
2587 if (variant == ARTPEC6_CRYPTO) {
2588 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR);
2589 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA;
2590 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH;
2591 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT;
2592 in_cmd_reg = A6_PDMA_IN_CMD;
2593 ack_intr_reg = A6_PDMA_ACK_INTR;
2594 } else {
2595 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR);
2596 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA;
2597 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH;
2598 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT;
2599 in_cmd_reg = A7_PDMA_IN_CMD;
2600 ack_intr_reg = A7_PDMA_ACK_INTR;
2601 }
2602
2603 /* We get two interrupt notifications from each job.
2604 * The in_data means all data was sent to memory and then
2605 * we request a status flush command to write the per-job
2606 * status to its status vector. This ensures that the
2607 * tasklet can detect exactly how many submitted jobs
2608 * that have finished.
2609 */
2610 if (intr & mask_in_data)
2611 ack |= mask_in_data;
2612
2613 if (intr & mask_in_eop_flush)
2614 ack |= mask_in_eop_flush;
2615 else
2616 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg);
2617
2618 writel_relaxed(ack, base + ack_intr_reg);
2619
2620 if (intr & mask_in_eop_flush)
2621 tasklet_schedule(&ac->task);
2622
2623 return IRQ_HANDLED;
2624}
2625
2626/*------------------- Algorithm definitions ----------------------------------*/
2627
2628/* Hashes */
2629static struct ahash_alg hash_algos[] = {
2630 /* SHA-1 */
2631 {
2632 .init = artpec6_crypto_sha1_init,
2633 .update = artpec6_crypto_hash_update,
2634 .final = artpec6_crypto_hash_final,
2635 .digest = artpec6_crypto_sha1_digest,
2636 .import = artpec6_crypto_hash_import,
2637 .export = artpec6_crypto_hash_export,
2638 .halg.digestsize = SHA1_DIGEST_SIZE,
2639 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2640 .halg.base = {
2641 .cra_name = "sha1",
2642 .cra_driver_name = "artpec-sha1",
2643 .cra_priority = 300,
6a38f622 2644 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2645 .cra_blocksize = SHA1_BLOCK_SIZE,
2646 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2647 .cra_alignmask = 3,
2648 .cra_module = THIS_MODULE,
2649 .cra_init = artpec6_crypto_ahash_init,
2650 .cra_exit = artpec6_crypto_ahash_exit,
2651 }
2652 },
2653 /* SHA-256 */
2654 {
2655 .init = artpec6_crypto_sha256_init,
2656 .update = artpec6_crypto_hash_update,
2657 .final = artpec6_crypto_hash_final,
2658 .digest = artpec6_crypto_sha256_digest,
2659 .import = artpec6_crypto_hash_import,
2660 .export = artpec6_crypto_hash_export,
2661 .halg.digestsize = SHA256_DIGEST_SIZE,
2662 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2663 .halg.base = {
2664 .cra_name = "sha256",
2665 .cra_driver_name = "artpec-sha256",
2666 .cra_priority = 300,
6a38f622 2667 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2668 .cra_blocksize = SHA256_BLOCK_SIZE,
2669 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2670 .cra_alignmask = 3,
2671 .cra_module = THIS_MODULE,
2672 .cra_init = artpec6_crypto_ahash_init,
2673 .cra_exit = artpec6_crypto_ahash_exit,
2674 }
2675 },
2676 /* HMAC SHA-256 */
2677 {
2678 .init = artpec6_crypto_hmac_sha256_init,
2679 .update = artpec6_crypto_hash_update,
2680 .final = artpec6_crypto_hash_final,
2681 .digest = artpec6_crypto_hmac_sha256_digest,
2682 .import = artpec6_crypto_hash_import,
2683 .export = artpec6_crypto_hash_export,
2684 .setkey = artpec6_crypto_hash_set_key,
2685 .halg.digestsize = SHA256_DIGEST_SIZE,
2686 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2687 .halg.base = {
2688 .cra_name = "hmac(sha256)",
2689 .cra_driver_name = "artpec-hmac-sha256",
2690 .cra_priority = 300,
6a38f622 2691 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2692 .cra_blocksize = SHA256_BLOCK_SIZE,
2693 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2694 .cra_alignmask = 3,
2695 .cra_module = THIS_MODULE,
2696 .cra_init = artpec6_crypto_ahash_init_hmac_sha256,
2697 .cra_exit = artpec6_crypto_ahash_exit,
2698 }
2699 },
2700};
2701
a21eb94f
LP		 2702/* Crypto */
2703static struct skcipher_alg crypto_algos[] = {
2704 /* AES - ECB */
2705 {
2706 .base = {
2707 .cra_name = "ecb(aes)",
2708 .cra_driver_name = "artpec6-ecb-aes",
2709 .cra_priority = 300,
2c95e6d9 2710 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2711 .cra_blocksize = AES_BLOCK_SIZE,
2712 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2713 .cra_alignmask = 3,
2714 .cra_module = THIS_MODULE,
2715 },
2716 .min_keysize = AES_MIN_KEY_SIZE,
2717 .max_keysize = AES_MAX_KEY_SIZE,
2718 .setkey = artpec6_crypto_cipher_set_key,
2719 .encrypt = artpec6_crypto_encrypt,
2720 .decrypt = artpec6_crypto_decrypt,
2721 .init = artpec6_crypto_aes_ecb_init,
2722 .exit = artpec6_crypto_aes_exit,
2723 },
2724 /* AES - CTR */
2725 {
2726 .base = {
2727 .cra_name = "ctr(aes)",
2728 .cra_driver_name = "artpec6-ctr-aes",
2729 .cra_priority = 300,
2c95e6d9 2730 .cra_flags = CRYPTO_ALG_ASYNC |
a21eb94f
LP		 2731 CRYPTO_ALG_NEED_FALLBACK,
2732 .cra_blocksize = 1,
2733 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2734 .cra_alignmask = 3,
2735 .cra_module = THIS_MODULE,
2736 },
2737 .min_keysize = AES_MIN_KEY_SIZE,
2738 .max_keysize = AES_MAX_KEY_SIZE,
2739 .ivsize = AES_BLOCK_SIZE,
2740 .setkey = artpec6_crypto_cipher_set_key,
2741 .encrypt = artpec6_crypto_ctr_encrypt,
2742 .decrypt = artpec6_crypto_ctr_decrypt,
2743 .init = artpec6_crypto_aes_ctr_init,
2744 .exit = artpec6_crypto_aes_ctr_exit,
2745 },
2746 /* AES - CBC */
2747 {
2748 .base = {
2749 .cra_name = "cbc(aes)",
2750 .cra_driver_name = "artpec6-cbc-aes",
2751 .cra_priority = 300,
2c95e6d9 2752 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2753 .cra_blocksize = AES_BLOCK_SIZE,
2754 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2755 .cra_alignmask = 3,
2756 .cra_module = THIS_MODULE,
2757 },
2758 .min_keysize = AES_MIN_KEY_SIZE,
2759 .max_keysize = AES_MAX_KEY_SIZE,
2760 .ivsize = AES_BLOCK_SIZE,
2761 .setkey = artpec6_crypto_cipher_set_key,
2762 .encrypt = artpec6_crypto_encrypt,
2763 .decrypt = artpec6_crypto_decrypt,
2764 .init = artpec6_crypto_aes_cbc_init,
2765 .exit = artpec6_crypto_aes_exit
2766 },
2767 /* AES - XTS */
2768 {
2769 .base = {
2770 .cra_name = "xts(aes)",
2771 .cra_driver_name = "artpec6-xts-aes",
2772 .cra_priority = 300,
2c95e6d9 2773 .cra_flags = CRYPTO_ALG_ASYNC,
a21eb94f
LP		 2774 .cra_blocksize = 1,
2775 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2776 .cra_alignmask = 3,
2777 .cra_module = THIS_MODULE,
2778 },
2779 .min_keysize = 2*AES_MIN_KEY_SIZE,
2780 .max_keysize = 2*AES_MAX_KEY_SIZE,
2781 .ivsize = 16,
2782 .setkey = artpec6_crypto_xts_set_key,
2783 .encrypt = artpec6_crypto_encrypt,
2784 .decrypt = artpec6_crypto_decrypt,
2785 .init = artpec6_crypto_aes_xts_init,
2786 .exit = artpec6_crypto_aes_exit,
2787 },
2788};
2789
2790static struct aead_alg aead_algos[] = {
2791 {
2792 .init = artpec6_crypto_aead_init,
2793 .setkey = artpec6_crypto_aead_set_key,
2794 .encrypt = artpec6_crypto_aead_encrypt,
2795 .decrypt = artpec6_crypto_aead_decrypt,
6d6e71fe 2796 .ivsize = GCM_AES_IV_SIZE,
a21eb94f
LP		 2797 .maxauthsize = AES_BLOCK_SIZE,
2798
2799 .base = {
2800 .cra_name = "gcm(aes)",
2801 .cra_driver_name = "artpec-gcm-aes",
2802 .cra_priority = 300,
3f4a537a 2803 .cra_flags = CRYPTO_ALG_ASYNC |
a21eb94f
LP		 2804 CRYPTO_ALG_KERN_DRIVER_ONLY,
2805 .cra_blocksize = 1,
2806 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2807 .cra_alignmask = 3,
2808 .cra_module = THIS_MODULE,
2809 },
2810 }
2811};
2812
2813#ifdef CONFIG_DEBUG_FS
2814
2815struct dbgfs_u32 {
2816 char *name;
2817 mode_t mode;
2818 u32 *flag;
2819 char *desc;
2820};
2821
40784d72
AB		 2822static struct dentry *dbgfs_root;
2823
a21eb94f
LP		 2824static void artpec6_crypto_init_debugfs(void)
2825{
2826 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL);
2827
a21eb94f
LP		 2828#ifdef CONFIG_FAULT_INJECTION
2829 fault_create_debugfs_attr("fail_status_read", dbgfs_root,
2830 &artpec6_crypto_fail_status_read);
2831
2832 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root,
2833 &artpec6_crypto_fail_dma_array_full);
2834#endif
2835}
2836
2837static void artpec6_crypto_free_debugfs(void)
2838{
a21eb94f
LP		 2839 debugfs_remove_recursive(dbgfs_root);
2840 dbgfs_root = NULL;
2841}
2842#endif
2843
2844static const struct of_device_id artpec6_crypto_of_match[] = {
2845 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
2846 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
2847 {}
2848};
2849MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match);
2850
2851static int artpec6_crypto_probe(struct platform_device *pdev)
2852{
2853 const struct of_device_id *match;
2854 enum artpec6_crypto_variant variant;
2855 struct artpec6_crypto *ac;
2856 struct device *dev = &pdev->dev;
2857 void __iomem *base;
2858 struct resource *res;
2859 int irq;
2860 int err;
2861
2862 if (artpec6_crypto_dev)
2863 return -ENODEV;
2864
2865 match = of_match_node(artpec6_crypto_of_match, dev->of_node);
2866 if (!match)
2867 return -EINVAL;
2868
2869 variant = (enum artpec6_crypto_variant)match->data;
2870
2871 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
a21eb94f
LP		 2872 base = devm_ioremap_resource(&pdev->dev, res);
2873 if (IS_ERR(base))
2874 return PTR_ERR(base);
2875
2876 irq = platform_get_irq(pdev, 0);
2877 if (irq < 0)
2878 return -ENODEV;
2879
2880 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),
2881 GFP_KERNEL);
2882 if (!ac)
2883 return -ENOMEM;
2884
2885 platform_set_drvdata(pdev, ac);
2886 ac->variant = variant;
2887
2888 spin_lock_init(&ac->queue_lock);
2889 INIT_LIST_HEAD(&ac->queue);
2890 INIT_LIST_HEAD(&ac->pending);
f34d8d50 2891 timer_setup(&ac->timer, artpec6_crypto_timeout, 0);
a21eb94f
LP		 2892
2893 ac->base = base;
2894
2895 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",
2896 sizeof(struct artpec6_crypto_dma_descriptors),
2897 64,
2898 0,
2899 NULL);
2900 if (!ac->dma_cache)
2901 return -ENOMEM;
2902
2903#ifdef CONFIG_DEBUG_FS
2904 artpec6_crypto_init_debugfs();
2905#endif
2906
2907 tasklet_init(&ac->task, artpec6_crypto_task,
2908 (unsigned long)ac);
2909
2910 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
2911 GFP_KERNEL);
2912 if (!ac->pad_buffer)
2913 return -ENOMEM;
2914 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);
2915
2916 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
2917 GFP_KERNEL);
2918 if (!ac->zero_buffer)
2919 return -ENOMEM;
2920 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);
2921
2922 err = init_crypto_hw(ac);
2923 if (err)
2924 goto free_cache;
2925
2926 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0,
2927 "artpec6-crypto", ac);
2928 if (err)
2929 goto disable_hw;
2930
2931 artpec6_crypto_dev = &pdev->dev;
2932
2933 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
2934 if (err) {
2935 dev_err(dev, "Failed to register ahashes\n");
2936 goto disable_hw;
2937 }
2938
a21eb94f
LP		 2939 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
2940 if (err) {
2941 dev_err(dev, "Failed to register ciphers\n");
f68deeba 2942 goto unregister_ahashes;
a21eb94f
LP		 2943 }
2944
2945 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos));
2946 if (err) {
2947 dev_err(dev, "Failed to register aeads\n");
2948 goto unregister_algs;
2949 }
2950
2951 return 0;
2952
2953unregister_algs:
2954 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
a21eb94f
LP		 2955unregister_ahashes:
2956 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
2957disable_hw:
2958 artpec6_crypto_disable_hw(ac);
2959free_cache:
2960 kmem_cache_destroy(ac->dma_cache);
2961 return err;
2962}
2963
2964static int artpec6_crypto_remove(struct platform_device *pdev)
2965{
2966 struct artpec6_crypto *ac = platform_get_drvdata(pdev);
2967 int irq = platform_get_irq(pdev, 0);
2968
2969 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
a21eb94f
LP		 2970 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
2971 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos));
2972
2973 tasklet_disable(&ac->task);
2974 devm_free_irq(&pdev->dev, irq, ac);
2975 tasklet_kill(&ac->task);
2976 del_timer_sync(&ac->timer);
2977
2978 artpec6_crypto_disable_hw(ac);
2979
2980 kmem_cache_destroy(ac->dma_cache);
2981#ifdef CONFIG_DEBUG_FS
2982 artpec6_crypto_free_debugfs();
2983#endif
2984 return 0;
2985}
2986
2987static struct platform_driver artpec6_crypto_driver = {
2988 .probe = artpec6_crypto_probe,
2989 .remove = artpec6_crypto_remove,
2990 .driver = {
2991 .name = "artpec6-crypto",
a21eb94f
LP		 2992 .of_match_table = artpec6_crypto_of_match,
2993 },
2994};
2995
2996module_platform_driver(artpec6_crypto_driver);
2997
2998MODULE_AUTHOR("Axis Communications AB");
2999MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");
3000MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)