projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
istallion section warnings
[linux-2.6-block.git] / drivers / crypto / talitos.c
CommitLineData
9c4a7965
KP		 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
4 * Copyright (c) 2008 Freescale Semiconductor, Inc.
5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
35#include <linux/of_platform.h>
36#include <linux/dma-mapping.h>
37#include <linux/io.h>
38#include <linux/spinlock.h>
39#include <linux/rtnetlink.h>
40
41#include <crypto/algapi.h>
42#include <crypto/aes.h>
3952f17e 43#include <crypto/des.h>
9c4a7965
KP		 44#include <crypto/sha.h>
45#include <crypto/aead.h>
46#include <crypto/authenc.h>
47
48#include "talitos.h"
49
50#define TALITOS_TIMEOUT 100000
51#define TALITOS_MAX_DATA_LEN 65535
52
53#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
54#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
55#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
56
57/* descriptor pointer entry */
58struct talitos_ptr {
59 __be16 len; /* length */
60 u8 j_extent; /* jump to sg link table and/or extent */
61 u8 eptr; /* extended address */
62 __be32 ptr; /* address */
63};
64
65/* descriptor */
66struct talitos_desc {
67 __be32 hdr; /* header high bits */
68 __be32 hdr_lo; /* header low bits */
69 struct talitos_ptr ptr[7]; /* ptr/len pair array */
70};
71
72/**
73 * talitos_request - descriptor submission request
74 * @desc: descriptor pointer (kernel virtual)
75 * @dma_desc: descriptor's physical bus address
76 * @callback: whom to call when descriptor processing is done
77 * @context: caller context (optional)
78 */
79struct talitos_request {
80 struct talitos_desc *desc;
81 dma_addr_t dma_desc;
82 void (*callback) (struct device *dev, struct talitos_desc *desc,
83 void *context, int error);
84 void *context;
85};
86
87struct talitos_private {
88 struct device *dev;
89 struct of_device *ofdev;
90 void __iomem *reg;
91 int irq;
92
93 /* SEC version geometry (from device tree node) */
94 unsigned int num_channels;
95 unsigned int chfifo_len;
96 unsigned int exec_units;
97 unsigned int desc_types;
98
f3c85bc1
LN		 99 /* SEC Compatibility info */
100 unsigned long features;
101
9c4a7965
KP		 102 /* next channel to be assigned next incoming descriptor */
103 atomic_t last_chan;
104
ec6644d6
KP		 105 /* per-channel number of requests pending in channel h/w fifo */
106 atomic_t *submit_count;
107
9c4a7965
KP		 108 /* per-channel request fifo */
109 struct talitos_request **fifo;
110
111 /*
112 * length of the request fifo
113 * fifo_len is chfifo_len rounded up to next power of 2
114 * so we can use bitwise ops to wrap
115 */
116 unsigned int fifo_len;
117
118 /* per-channel index to next free descriptor request */
119 int *head;
120
121 /* per-channel index to next in-progress/done descriptor request */
122 int *tail;
123
124 /* per-channel request submission (head) and release (tail) locks */
125 spinlock_t *head_lock;
126 spinlock_t *tail_lock;
127
128 /* request callback tasklet */
129 struct tasklet_struct done_task;
130 struct tasklet_struct error_task;
131
132 /* list of registered algorithms */
133 struct list_head alg_list;
134
135 /* hwrng device */
136 struct hwrng rng;
137};
138
f3c85bc1
LN		 139/* .features flag */
140#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
141
9c4a7965
KP		 142/*
143 * map virtual single (contiguous) pointer to h/w descriptor pointer
144 */
145static void map_single_talitos_ptr(struct device *dev,
146 struct talitos_ptr *talitos_ptr,
147 unsigned short len, void *data,
148 unsigned char extent,
149 enum dma_data_direction dir)
150{
151 talitos_ptr->len = cpu_to_be16(len);
152 talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
153 talitos_ptr->j_extent = extent;
154}
155
156/*
157 * unmap bus single (contiguous) h/w descriptor pointer
158 */
159static void unmap_single_talitos_ptr(struct device *dev,
160 struct talitos_ptr *talitos_ptr,
161 enum dma_data_direction dir)
162{
163 dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
164 be16_to_cpu(talitos_ptr->len), dir);
165}
166
167static int reset_channel(struct device *dev, int ch)
168{
169 struct talitos_private *priv = dev_get_drvdata(dev);
170 unsigned int timeout = TALITOS_TIMEOUT;
171
172 setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
173
174 while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
175 && --timeout)
176 cpu_relax();
177
178 if (timeout == 0) {
179 dev_err(dev, "failed to reset channel %d\n", ch);
180 return -EIO;
181 }
182
183 /* set done writeback and IRQ */
184 setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
185 TALITOS_CCCR_LO_CDIE);
186
187 return 0;
188}
189
190static int reset_device(struct device *dev)
191{
192 struct talitos_private *priv = dev_get_drvdata(dev);
193 unsigned int timeout = TALITOS_TIMEOUT;
194
195 setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
196
197 while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
198 && --timeout)
199 cpu_relax();
200
201 if (timeout == 0) {
202 dev_err(dev, "failed to reset device\n");
203 return -EIO;
204 }
205
206 return 0;
207}
208
209/*
210 * Reset and initialize the device
211 */
212static int init_device(struct device *dev)
213{
214 struct talitos_private *priv = dev_get_drvdata(dev);
215 int ch, err;
216
217 /*
218 * Master reset
219 * errata documentation: warning: certain SEC interrupts
220 * are not fully cleared by writing the MCR:SWR bit,
221 * set bit twice to completely reset
222 */
223 err = reset_device(dev);
224 if (err)
225 return err;
226
227 err = reset_device(dev);
228 if (err)
229 return err;
230
231 /* reset channels */
232 for (ch = 0; ch < priv->num_channels; ch++) {
233 err = reset_channel(dev, ch);
234 if (err)
235 return err;
236 }
237
238 /* enable channel done and error interrupts */
239 setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
240 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
241
242 return 0;
243}
244
245/**
246 * talitos_submit - submits a descriptor to the device for processing
247 * @dev: the SEC device to be used
248 * @desc: the descriptor to be processed by the device
249 * @callback: whom to call when processing is complete
250 * @context: a handle for use by caller (optional)
251 *
252 * desc must contain valid dma-mapped (bus physical) address pointers.
253 * callback must check err and feedback in descriptor header
254 * for device processing status.
255 */
256static int talitos_submit(struct device *dev, struct talitos_desc *desc,
257 void (*callback)(struct device *dev,
258 struct talitos_desc *desc,
259 void *context, int error),
260 void *context)
261{
262 struct talitos_private *priv = dev_get_drvdata(dev);
263 struct talitos_request *request;
264 unsigned long flags, ch;
265 int head;
266
267 /* select done notification */
268 desc->hdr |= DESC_HDR_DONE_NOTIFY;
269
270 /* emulate SEC's round-robin channel fifo polling scheme */
271 ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
272
273 spin_lock_irqsave(&priv->head_lock[ch], flags);
274
ec6644d6
KP		 275 if (!atomic_inc_not_zero(&priv->submit_count[ch])) {
276 /* h/w fifo is full */
9c4a7965
KP		 277 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
278 return -EAGAIN;
279 }
280
ec6644d6
KP		 281 head = priv->head[ch];
282 request = &priv->fifo[ch][head];
283
9c4a7965
KP		 284 /* map descriptor and save caller data */
285 request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
286 DMA_BIDIRECTIONAL);
287 request->callback = callback;
288 request->context = context;
289
290 /* increment fifo head */
291 priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
292
293 smp_wmb();
294 request->desc = desc;
295
296 /* GO! */
297 wmb();
298 out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
299
300 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
301
302 return -EINPROGRESS;
303}
304
305/*
306 * process what was done, notify callback of error if not
307 */
308static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
309{
310 struct talitos_private *priv = dev_get_drvdata(dev);
311 struct talitos_request *request, saved_req;
312 unsigned long flags;
313 int tail, status;
314
315 spin_lock_irqsave(&priv->tail_lock[ch], flags);
316
317 tail = priv->tail[ch];
318 while (priv->fifo[ch][tail].desc) {
319 request = &priv->fifo[ch][tail];
320
321 /* descriptors with their done bits set don't get the error */
322 rmb();
323 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
324 status = 0;
325 else
326 if (!error)
327 break;
328 else
329 status = error;
330
331 dma_unmap_single(dev, request->dma_desc,
332 sizeof(struct talitos_desc), DMA_BIDIRECTIONAL);
333
334 /* copy entries so we can call callback outside lock */
335 saved_req.desc = request->desc;
336 saved_req.callback = request->callback;
337 saved_req.context = request->context;
338
339 /* release request entry in fifo */
340 smp_wmb();
341 request->desc = NULL;
342
343 /* increment fifo tail */
344 priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
345
346 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
ec6644d6
KP		 347
348 atomic_dec(&priv->submit_count[ch]);
349
9c4a7965
KP		 350 saved_req.callback(dev, saved_req.desc, saved_req.context,
351 status);
352 /* channel may resume processing in single desc error case */
353 if (error && !reset_ch && status == error)
354 return;
355 spin_lock_irqsave(&priv->tail_lock[ch], flags);
356 tail = priv->tail[ch];
357 }
358
359 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
360}
361
362/*
363 * process completed requests for channels that have done status
364 */
365static void talitos_done(unsigned long data)
366{
367 struct device *dev = (struct device *)data;
368 struct talitos_private *priv = dev_get_drvdata(dev);
369 int ch;
370
371 for (ch = 0; ch < priv->num_channels; ch++)
372 flush_channel(dev, ch, 0, 0);
373}
374
375/*
376 * locate current (offending) descriptor
377 */
378static struct talitos_desc *current_desc(struct device *dev, int ch)
379{
380 struct talitos_private *priv = dev_get_drvdata(dev);
381 int tail = priv->tail[ch];
382 dma_addr_t cur_desc;
383
384 cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
385
386 while (priv->fifo[ch][tail].dma_desc != cur_desc) {
387 tail = (tail + 1) & (priv->fifo_len - 1);
388 if (tail == priv->tail[ch]) {
389 dev_err(dev, "couldn't locate current descriptor\n");
390 return NULL;
391 }
392 }
393
394 return priv->fifo[ch][tail].desc;
395}
396
397/*
398 * user diagnostics; report root cause of error based on execution unit status
399 */
400static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc)
401{
402 struct talitos_private *priv = dev_get_drvdata(dev);
403 int i;
404
405 switch (desc->hdr & DESC_HDR_SEL0_MASK) {
406 case DESC_HDR_SEL0_AFEU:
407 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
408 in_be32(priv->reg + TALITOS_AFEUISR),
409 in_be32(priv->reg + TALITOS_AFEUISR_LO));
410 break;
411 case DESC_HDR_SEL0_DEU:
412 dev_err(dev, "DEUISR 0x%08x_%08x\n",
413 in_be32(priv->reg + TALITOS_DEUISR),
414 in_be32(priv->reg + TALITOS_DEUISR_LO));
415 break;
416 case DESC_HDR_SEL0_MDEUA:
417 case DESC_HDR_SEL0_MDEUB:
418 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
419 in_be32(priv->reg + TALITOS_MDEUISR),
420 in_be32(priv->reg + TALITOS_MDEUISR_LO));
421 break;
422 case DESC_HDR_SEL0_RNG:
423 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
424 in_be32(priv->reg + TALITOS_RNGUISR),
425 in_be32(priv->reg + TALITOS_RNGUISR_LO));
426 break;
427 case DESC_HDR_SEL0_PKEU:
428 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
429 in_be32(priv->reg + TALITOS_PKEUISR),
430 in_be32(priv->reg + TALITOS_PKEUISR_LO));
431 break;
432 case DESC_HDR_SEL0_AESU:
433 dev_err(dev, "AESUISR 0x%08x_%08x\n",
434 in_be32(priv->reg + TALITOS_AESUISR),
435 in_be32(priv->reg + TALITOS_AESUISR_LO));
436 break;
437 case DESC_HDR_SEL0_CRCU:
438 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
439 in_be32(priv->reg + TALITOS_CRCUISR),
440 in_be32(priv->reg + TALITOS_CRCUISR_LO));
441 break;
442 case DESC_HDR_SEL0_KEU:
443 dev_err(dev, "KEUISR 0x%08x_%08x\n",
444 in_be32(priv->reg + TALITOS_KEUISR),
445 in_be32(priv->reg + TALITOS_KEUISR_LO));
446 break;
447 }
448
449 switch (desc->hdr & DESC_HDR_SEL1_MASK) {
450 case DESC_HDR_SEL1_MDEUA:
451 case DESC_HDR_SEL1_MDEUB:
452 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
453 in_be32(priv->reg + TALITOS_MDEUISR),
454 in_be32(priv->reg + TALITOS_MDEUISR_LO));
455 break;
456 case DESC_HDR_SEL1_CRCU:
457 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
458 in_be32(priv->reg + TALITOS_CRCUISR),
459 in_be32(priv->reg + TALITOS_CRCUISR_LO));
460 break;
461 }
462
463 for (i = 0; i < 8; i++)
464 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
465 in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
466 in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
467}
468
469/*
470 * recover from error interrupts
471 */
472static void talitos_error(unsigned long data)
473{
474 struct device *dev = (struct device *)data;
475 struct talitos_private *priv = dev_get_drvdata(dev);
476 unsigned int timeout = TALITOS_TIMEOUT;
477 int ch, error, reset_dev = 0, reset_ch = 0;
478 u32 isr, isr_lo, v, v_lo;
479
480 isr = in_be32(priv->reg + TALITOS_ISR);
481 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
482
483 for (ch = 0; ch < priv->num_channels; ch++) {
484 /* skip channels without errors */
485 if (!(isr & (1 << (ch * 2 + 1))))
486 continue;
487
488 error = -EINVAL;
489
490 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
491 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
492
493 if (v_lo & TALITOS_CCPSR_LO_DOF) {
494 dev_err(dev, "double fetch fifo overflow error\n");
495 error = -EAGAIN;
496 reset_ch = 1;
497 }
498 if (v_lo & TALITOS_CCPSR_LO_SOF) {
499 /* h/w dropped descriptor */
500 dev_err(dev, "single fetch fifo overflow error\n");
501 error = -EAGAIN;
502 }
503 if (v_lo & TALITOS_CCPSR_LO_MDTE)
504 dev_err(dev, "master data transfer error\n");
505 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
506 dev_err(dev, "s/g data length zero error\n");
507 if (v_lo & TALITOS_CCPSR_LO_FPZ)
508 dev_err(dev, "fetch pointer zero error\n");
509 if (v_lo & TALITOS_CCPSR_LO_IDH)
510 dev_err(dev, "illegal descriptor header error\n");
511 if (v_lo & TALITOS_CCPSR_LO_IEU)
512 dev_err(dev, "invalid execution unit error\n");
513 if (v_lo & TALITOS_CCPSR_LO_EU)
514 report_eu_error(dev, ch, current_desc(dev, ch));
515 if (v_lo & TALITOS_CCPSR_LO_GB)
516 dev_err(dev, "gather boundary error\n");
517 if (v_lo & TALITOS_CCPSR_LO_GRL)
518 dev_err(dev, "gather return/length error\n");
519 if (v_lo & TALITOS_CCPSR_LO_SB)
520 dev_err(dev, "scatter boundary error\n");
521 if (v_lo & TALITOS_CCPSR_LO_SRL)
522 dev_err(dev, "scatter return/length error\n");
523
524 flush_channel(dev, ch, error, reset_ch);
525
526 if (reset_ch) {
527 reset_channel(dev, ch);
528 } else {
529 setbits32(priv->reg + TALITOS_CCCR(ch),
530 TALITOS_CCCR_CONT);
531 setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
532 while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
533 TALITOS_CCCR_CONT) && --timeout)
534 cpu_relax();
535 if (timeout == 0) {
536 dev_err(dev, "failed to restart channel %d\n",
537 ch);
538 reset_dev = 1;
539 }
540 }
541 }
542 if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
543 dev_err(dev, "done overflow, internal time out, or rngu error: "
544 "ISR 0x%08x_%08x\n", isr, isr_lo);
545
546 /* purge request queues */
547 for (ch = 0; ch < priv->num_channels; ch++)
548 flush_channel(dev, ch, -EIO, 1);
549
550 /* reset and reinitialize the device */
551 init_device(dev);
552 }
553}
554
555static irqreturn_t talitos_interrupt(int irq, void *data)
556{
557 struct device *dev = data;
558 struct talitos_private *priv = dev_get_drvdata(dev);
559 u32 isr, isr_lo;
560
561 isr = in_be32(priv->reg + TALITOS_ISR);
562 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
563
564 /* ack */
565 out_be32(priv->reg + TALITOS_ICR, isr);
566 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
567
568 if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
569 talitos_error((unsigned long)data);
570 else
571 if (likely(isr & TALITOS_ISR_CHDONE))
572 tasklet_schedule(&priv->done_task);
573
574 return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
575}
576
577/*
578 * hwrng
579 */
580static int talitos_rng_data_present(struct hwrng *rng, int wait)
581{
582 struct device *dev = (struct device *)rng->priv;
583 struct talitos_private *priv = dev_get_drvdata(dev);
584 u32 ofl;
585 int i;
586
587 for (i = 0; i < 20; i++) {
588 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
589 TALITOS_RNGUSR_LO_OFL;
590 if (ofl || !wait)
591 break;
592 udelay(10);
593 }
594
595 return !!ofl;
596}
597
598static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
599{
600 struct device *dev = (struct device *)rng->priv;
601 struct talitos_private *priv = dev_get_drvdata(dev);
602
603 /* rng fifo requires 64-bit accesses */
604 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
605 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
606
607 return sizeof(u32);
608}
609
610static int talitos_rng_init(struct hwrng *rng)
611{
612 struct device *dev = (struct device *)rng->priv;
613 struct talitos_private *priv = dev_get_drvdata(dev);
614 unsigned int timeout = TALITOS_TIMEOUT;
615
616 setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
617 while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
618 && --timeout)
619 cpu_relax();
620 if (timeout == 0) {
621 dev_err(dev, "failed to reset rng hw\n");
622 return -ENODEV;
623 }
624
625 /* start generating */
626 setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
627
628 return 0;
629}
630
631static int talitos_register_rng(struct device *dev)
632{
633 struct talitos_private *priv = dev_get_drvdata(dev);
634
635 priv->rng.name = dev_driver_string(dev),
636 priv->rng.init = talitos_rng_init,
637 priv->rng.data_present = talitos_rng_data_present,
638 priv->rng.data_read = talitos_rng_data_read,
639 priv->rng.priv = (unsigned long)dev;
640
641 return hwrng_register(&priv->rng);
642}
643
644static void talitos_unregister_rng(struct device *dev)
645{
646 struct talitos_private *priv = dev_get_drvdata(dev);
647
648 hwrng_unregister(&priv->rng);
649}
650
651/*
652 * crypto alg
653 */
654#define TALITOS_CRA_PRIORITY 3000
655#define TALITOS_MAX_KEY_SIZE 64
3952f17e 656#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
70bcaca7 657
3952f17e 658#define MD5_DIGEST_SIZE 16
9c4a7965
KP		 659
660struct talitos_ctx {
661 struct device *dev;
662 __be32 desc_hdr_template;
663 u8 key[TALITOS_MAX_KEY_SIZE];
70bcaca7 664 u8 iv[TALITOS_MAX_IV_LENGTH];
9c4a7965
KP		 665 unsigned int keylen;
666 unsigned int enckeylen;
667 unsigned int authkeylen;
668 unsigned int authsize;
669};
670
70bcaca7 671static int aead_authenc_setauthsize(struct crypto_aead *authenc,
9c4a7965
KP		 672 unsigned int authsize)
673{
674 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
675
676 ctx->authsize = authsize;
677
678 return 0;
679}
680
70bcaca7 681static int aead_authenc_setkey(struct crypto_aead *authenc,
9c4a7965
KP		 682 const u8 *key, unsigned int keylen)
683{
684 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
685 struct rtattr *rta = (void *)key;
686 struct crypto_authenc_key_param *param;
687 unsigned int authkeylen;
688 unsigned int enckeylen;
689
690 if (!RTA_OK(rta, keylen))
691 goto badkey;
692
693 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
694 goto badkey;
695
696 if (RTA_PAYLOAD(rta) < sizeof(*param))
697 goto badkey;
698
699 param = RTA_DATA(rta);
700 enckeylen = be32_to_cpu(param->enckeylen);
701
702 key += RTA_ALIGN(rta->rta_len);
703 keylen -= RTA_ALIGN(rta->rta_len);
704
705 if (keylen < enckeylen)
706 goto badkey;
707
708 authkeylen = keylen - enckeylen;
709
710 if (keylen > TALITOS_MAX_KEY_SIZE)
711 goto badkey;
712
713 memcpy(&ctx->key, key, keylen);
714
715 ctx->keylen = keylen;
716 ctx->enckeylen = enckeylen;
717 ctx->authkeylen = authkeylen;
718
719 return 0;
720
721badkey:
722 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
723 return -EINVAL;
724}
725
726/*
727 * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
728 * @src_nents: number of segments in input scatterlist
729 * @dst_nents: number of segments in output scatterlist
730 * @dma_len: length of dma mapped link_tbl space
731 * @dma_link_tbl: bus physical address of link_tbl
732 * @desc: h/w descriptor
733 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
734 *
735 * if decrypting (with authcheck), or either one of src_nents or dst_nents
736 * is greater than 1, an integrity check value is concatenated to the end
737 * of link_tbl data
738 */
739struct ipsec_esp_edesc {
740 int src_nents;
741 int dst_nents;
742 int dma_len;
743 dma_addr_t dma_link_tbl;
744 struct talitos_desc desc;
745 struct talitos_ptr link_tbl[0];
746};
747
748static void ipsec_esp_unmap(struct device *dev,
749 struct ipsec_esp_edesc *edesc,
750 struct aead_request *areq)
751{
752 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
753 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
754 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
755 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
756
757 dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
758
759 if (areq->src != areq->dst) {
760 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
761 DMA_TO_DEVICE);
762 dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1,
763 DMA_FROM_DEVICE);
764 } else {
765 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
766 DMA_BIDIRECTIONAL);
767 }
768
769 if (edesc->dma_len)
770 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
771 DMA_BIDIRECTIONAL);
772}
773
774/*
775 * ipsec_esp descriptor callbacks
776 */
777static void ipsec_esp_encrypt_done(struct device *dev,
778 struct talitos_desc *desc, void *context,
779 int err)
780{
781 struct aead_request *areq = context;
782 struct ipsec_esp_edesc *edesc =
783 container_of(desc, struct ipsec_esp_edesc, desc);
784 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
785 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
786 struct scatterlist *sg;
787 void *icvdata;
788
789 ipsec_esp_unmap(dev, edesc, areq);
790
791 /* copy the generated ICV to dst */
792 if (edesc->dma_len) {
793 icvdata = &edesc->link_tbl[edesc->src_nents +
f3c85bc1 794 edesc->dst_nents + 2];
9c4a7965
KP		 795 sg = sg_last(areq->dst, edesc->dst_nents);
796 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
797 icvdata, ctx->authsize);
798 }
799
800 kfree(edesc);
801
802 aead_request_complete(areq, err);
803}
804
805static void ipsec_esp_decrypt_done(struct device *dev,
806 struct talitos_desc *desc, void *context,
807 int err)
808{
809 struct aead_request *req = context;
810 struct ipsec_esp_edesc *edesc =
811 container_of(desc, struct ipsec_esp_edesc, desc);
812 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
813 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
814 struct scatterlist *sg;
815 void *icvdata;
816
817 ipsec_esp_unmap(dev, edesc, req);
818
819 if (!err) {
820 /* auth check */
821 if (edesc->dma_len)
822 icvdata = &edesc->link_tbl[edesc->src_nents +
f3c85bc1 823 edesc->dst_nents + 2];
9c4a7965
KP		 824 else
825 icvdata = &edesc->link_tbl[0];
826
827 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
828 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
829 ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
830 }
831
832 kfree(edesc);
833
834 aead_request_complete(req, err);
835}
836
837/*
838 * convert scatterlist to SEC h/w link table format
839 * stop at cryptlen bytes
840 */
70bcaca7 841static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
9c4a7965
KP		 842 int cryptlen, struct talitos_ptr *link_tbl_ptr)
843{
70bcaca7
LN		 844 int n_sg = sg_count;
845
846 while (n_sg--) {
9c4a7965
KP		 847 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
848 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
849 link_tbl_ptr->j_extent = 0;
850 link_tbl_ptr++;
851 cryptlen -= sg_dma_len(sg);
852 sg = sg_next(sg);
853 }
854
70bcaca7 855 /* adjust (decrease) last one (or two) entry's len to cryptlen */
9c4a7965 856 link_tbl_ptr--;
c0e741d4 857 while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
70bcaca7
LN		 858 /* Empty this entry, and move to previous one */
859 cryptlen += be16_to_cpu(link_tbl_ptr->len);
860 link_tbl_ptr->len = 0;
861 sg_count--;
862 link_tbl_ptr--;
863 }
9c4a7965
KP		 864 link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
865 + cryptlen);
866
867 /* tag end of link table */
868 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
70bcaca7
LN		 869
870 return sg_count;
9c4a7965
KP		 871}
872
873/*
874 * fill in and submit ipsec_esp descriptor
875 */
876static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
877 u8 *giv, u64 seq,
878 void (*callback) (struct device *dev,
879 struct talitos_desc *desc,
880 void *context, int error))
881{
882 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
883 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
884 struct device *dev = ctx->dev;
885 struct talitos_desc *desc = &edesc->desc;
886 unsigned int cryptlen = areq->cryptlen;
887 unsigned int authsize = ctx->authsize;
888 unsigned int ivsize;
fa86a267 889 int sg_count, ret;
9c4a7965
KP		 890
891 /* hmac key */
892 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
893 0, DMA_TO_DEVICE);
894 /* hmac data */
895 map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) -
896 sg_virt(areq->assoc), sg_virt(areq->assoc), 0,
897 DMA_TO_DEVICE);
898 /* cipher iv */
899 ivsize = crypto_aead_ivsize(aead);
900 map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
901 DMA_TO_DEVICE);
902
903 /* cipher key */
904 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
905 (char *)&ctx->key + ctx->authkeylen, 0,
906 DMA_TO_DEVICE);
907
908 /*
909 * cipher in
910 * map and adjust cipher len to aead request cryptlen.
911 * extent is bytes of HMAC postpended to ciphertext,
912 * typically 12 for ipsec
913 */
914 desc->ptr[4].len = cpu_to_be16(cryptlen);
915 desc->ptr[4].j_extent = authsize;
916
917 if (areq->src == areq->dst)
918 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
919 DMA_BIDIRECTIONAL);
920 else
921 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
922 DMA_TO_DEVICE);
923
924 if (sg_count == 1) {
925 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
926 } else {
70bcaca7
LN		 927 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
928 &edesc->link_tbl[0]);
929 if (sg_count > 1) {
f3c85bc1
LN		 930 struct talitos_ptr *link_tbl_ptr =
931 &edesc->link_tbl[sg_count-1];
932 struct scatterlist *sg;
933 struct talitos_private *priv = dev_get_drvdata(dev);
934
70bcaca7
LN		 935 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
936 desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
937 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
938 edesc->dma_len, DMA_BIDIRECTIONAL);
f3c85bc1
LN		 939 /* If necessary for this SEC revision,
940 * add a link table entry for ICV.
941 */
942 if ((priv->features &
943 TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) &&
944 (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) {
945 link_tbl_ptr->j_extent = 0;
946 link_tbl_ptr++;
947 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
948 link_tbl_ptr->len = cpu_to_be16(authsize);
949 sg = sg_last(areq->src, edesc->src_nents ? : 1);
950 link_tbl_ptr->ptr = cpu_to_be32(
951 (char *)sg_dma_address(sg)
952 + sg->length - authsize);
953 }
70bcaca7
LN		 954 } else {
955 /* Only one segment now, so no link tbl needed */
956 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
957 }
9c4a7965
KP		 958 }
959
960 /* cipher out */
961 desc->ptr[5].len = cpu_to_be16(cryptlen);
962 desc->ptr[5].j_extent = authsize;
963
964 if (areq->src != areq->dst) {
965 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
966 DMA_FROM_DEVICE);
967 }
968
969 if (sg_count == 1) {
970 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
971 } else {
972 struct talitos_ptr *link_tbl_ptr =
f3c85bc1 973 &edesc->link_tbl[edesc->src_nents + 1];
9c4a7965
KP		 974
975 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
976 edesc->dma_link_tbl +
f3c85bc1 977 edesc->src_nents + 1);
9c4a7965
KP		 978 if (areq->src == areq->dst) {
979 memcpy(link_tbl_ptr, &edesc->link_tbl[0],
980 edesc->src_nents * sizeof(struct talitos_ptr));
981 } else {
70bcaca7
LN		 982 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
983 link_tbl_ptr);
9c4a7965 984 }
f3c85bc1 985 /* Add an entry to the link table for ICV data */
9c4a7965 986 link_tbl_ptr += sg_count - 1;
9c4a7965 987 link_tbl_ptr->j_extent = 0;
f3c85bc1 988 sg_count++;
9c4a7965
KP		 989 link_tbl_ptr++;
990 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
991 link_tbl_ptr->len = cpu_to_be16(authsize);
992
993 /* icv data follows link tables */
994 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
995 edesc->dma_link_tbl +
996 edesc->src_nents +
f3c85bc1 997 edesc->dst_nents + 2);
9c4a7965
KP		 998
999 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1000 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1001 edesc->dma_len, DMA_BIDIRECTIONAL);
1002 }
1003
1004 /* iv out */
1005 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1006 DMA_FROM_DEVICE);
1007
fa86a267
KP		 1008 ret = talitos_submit(dev, desc, callback, areq);
1009 if (ret != -EINPROGRESS) {
1010 ipsec_esp_unmap(dev, edesc, areq);
1011 kfree(edesc);
1012 }
1013 return ret;
9c4a7965
KP		 1014}
1015
1016
1017/*
1018 * derive number of elements in scatterlist
1019 */
1020static int sg_count(struct scatterlist *sg_list, int nbytes)
1021{
1022 struct scatterlist *sg = sg_list;
1023 int sg_nents = 0;
1024
1025 while (nbytes) {
1026 sg_nents++;
1027 nbytes -= sg->length;
1028 sg = sg_next(sg);
1029 }
1030
1031 return sg_nents;
1032}
1033
1034/*
1035 * allocate and map the ipsec_esp extended descriptor
1036 */
1037static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
1038 int icv_stashing)
1039{
1040 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1041 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1042 struct ipsec_esp_edesc *edesc;
1043 int src_nents, dst_nents, alloc_len, dma_len;
586725f8
KP		 1044 gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1045 GFP_ATOMIC;
9c4a7965
KP		 1046
1047 if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) {
1048 dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n");
1049 return ERR_PTR(-EINVAL);
1050 }
1051
1052 src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize);
1053 src_nents = (src_nents == 1) ? 0 : src_nents;
1054
1055 if (areq->dst == areq->src) {
1056 dst_nents = src_nents;
1057 } else {
1058 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize);
695ad589 1059 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
9c4a7965
KP		 1060 }
1061
1062 /*
1063 * allocate space for base edesc plus the link tables,
f3c85bc1 1064 * allowing for two separate entries for ICV and generated ICV (+ 2),
9c4a7965
KP		 1065 * and the ICV data itself
1066 */
1067 alloc_len = sizeof(struct ipsec_esp_edesc);
1068 if (src_nents || dst_nents) {
f3c85bc1 1069 dma_len = (src_nents + dst_nents + 2) *
9c4a7965
KP		 1070 sizeof(struct talitos_ptr) + ctx->authsize;
1071 alloc_len += dma_len;
1072 } else {
1073 dma_len = 0;
1074 alloc_len += icv_stashing ? ctx->authsize : 0;
1075 }
1076
586725f8 1077 edesc = kmalloc(alloc_len, GFP_DMA | flags);
9c4a7965
KP		 1078 if (!edesc) {
1079 dev_err(ctx->dev, "could not allocate edescriptor\n");
1080 return ERR_PTR(-ENOMEM);
1081 }
1082
1083 edesc->src_nents = src_nents;
1084 edesc->dst_nents = dst_nents;
1085 edesc->dma_len = dma_len;
1086 edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0],
1087 edesc->dma_len, DMA_BIDIRECTIONAL);
1088
1089 return edesc;
1090}
1091
70bcaca7 1092static int aead_authenc_encrypt(struct aead_request *req)
9c4a7965
KP		 1093{
1094 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1095 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1096 struct ipsec_esp_edesc *edesc;
1097
1098 /* allocate extended descriptor */
1099 edesc = ipsec_esp_edesc_alloc(req, 0);
1100 if (IS_ERR(edesc))
1101 return PTR_ERR(edesc);
1102
1103 /* set encrypt */
70bcaca7 1104 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965
KP		 1105
1106 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1107}
1108
70bcaca7 1109static int aead_authenc_decrypt(struct aead_request *req)
9c4a7965
KP		 1110{
1111 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1112 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1113 unsigned int authsize = ctx->authsize;
1114 struct ipsec_esp_edesc *edesc;
1115 struct scatterlist *sg;
1116 void *icvdata;
1117
1118 req->cryptlen -= authsize;
1119
1120 /* allocate extended descriptor */
1121 edesc = ipsec_esp_edesc_alloc(req, 1);
1122 if (IS_ERR(edesc))
1123 return PTR_ERR(edesc);
1124
1125 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1126 if (edesc->dma_len)
1127 icvdata = &edesc->link_tbl[edesc->src_nents +
f3c85bc1 1128 edesc->dst_nents + 2];
9c4a7965
KP		 1129 else
1130 icvdata = &edesc->link_tbl[0];
1131
1132 sg = sg_last(req->src, edesc->src_nents ? : 1);
1133
1134 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1135 ctx->authsize);
1136
1137 /* decrypt */
1138 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1139
1140 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done);
1141}
1142
70bcaca7 1143static int aead_authenc_givencrypt(
9c4a7965
KP		 1144 struct aead_givcrypt_request *req)
1145{
1146 struct aead_request *areq = &req->areq;
1147 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1148 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1149 struct ipsec_esp_edesc *edesc;
1150
1151 /* allocate extended descriptor */
1152 edesc = ipsec_esp_edesc_alloc(areq, 0);
1153 if (IS_ERR(edesc))
1154 return PTR_ERR(edesc);
1155
1156 /* set encrypt */
70bcaca7 1157 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965
KP		 1158
1159 memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
ba95487d
KP		 1160 /* avoid consecutive packets going out with same IV */
1161 *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
9c4a7965
KP		 1162
1163 return ipsec_esp(edesc, areq, req->giv, req->seq,
1164 ipsec_esp_encrypt_done);
1165}
1166
1167struct talitos_alg_template {
1168 char name[CRYPTO_MAX_ALG_NAME];
1169 char driver_name[CRYPTO_MAX_ALG_NAME];
1170 unsigned int blocksize;
1171 struct aead_alg aead;
1172 struct device *dev;
1173 __be32 desc_hdr_template;
1174};
1175
1176static struct talitos_alg_template driver_algs[] = {
1177 /* single-pass ipsec_esp descriptor */
1178 {
1179 .name = "authenc(hmac(sha1),cbc(aes))",
ebbcf336 1180 .driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
3952f17e 1181 .blocksize = AES_BLOCK_SIZE,
9c4a7965 1182 .aead = {
70bcaca7
LN		 1183 .setkey = aead_authenc_setkey,
1184 .setauthsize = aead_authenc_setauthsize,
1185 .encrypt = aead_authenc_encrypt,
1186 .decrypt = aead_authenc_decrypt,
1187 .givencrypt = aead_authenc_givencrypt,
9c4a7965 1188 .geniv = "<built-in>",
3952f17e
LN		 1189 .ivsize = AES_BLOCK_SIZE,
1190 .maxauthsize = SHA1_DIGEST_SIZE,
9c4a7965
KP		 1191 },
1192 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1193 DESC_HDR_SEL0_AESU |
1194 DESC_HDR_MODE0_AESU_CBC |
1195 DESC_HDR_SEL1_MDEUA |
1196 DESC_HDR_MODE1_MDEU_INIT |
1197 DESC_HDR_MODE1_MDEU_PAD |
1198 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
70bcaca7
LN		 1199 },
1200 {
1201 .name = "authenc(hmac(sha1),cbc(des3_ede))",
ebbcf336 1202 .driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
3952f17e 1203 .blocksize = DES3_EDE_BLOCK_SIZE,
70bcaca7
LN		 1204 .aead = {
1205 .setkey = aead_authenc_setkey,
1206 .setauthsize = aead_authenc_setauthsize,
1207 .encrypt = aead_authenc_encrypt,
1208 .decrypt = aead_authenc_decrypt,
1209 .givencrypt = aead_authenc_givencrypt,
1210 .geniv = "<built-in>",
3952f17e
LN		 1211 .ivsize = DES3_EDE_BLOCK_SIZE,
1212 .maxauthsize = SHA1_DIGEST_SIZE,
70bcaca7
LN		 1213 },
1214 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1215 DESC_HDR_SEL0_DEU |
1216 DESC_HDR_MODE0_DEU_CBC |
1217 DESC_HDR_MODE0_DEU_3DES |
1218 DESC_HDR_SEL1_MDEUA |
1219 DESC_HDR_MODE1_MDEU_INIT |
1220 DESC_HDR_MODE1_MDEU_PAD |
1221 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
3952f17e
LN		 1222 },
1223 {
1224 .name = "authenc(hmac(sha256),cbc(aes))",
1225 .driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1226 .blocksize = AES_BLOCK_SIZE,
1227 .aead = {
1228 .setkey = aead_authenc_setkey,
1229 .setauthsize = aead_authenc_setauthsize,
1230 .encrypt = aead_authenc_encrypt,
1231 .decrypt = aead_authenc_decrypt,
1232 .givencrypt = aead_authenc_givencrypt,
1233 .geniv = "<built-in>",
1234 .ivsize = AES_BLOCK_SIZE,
1235 .maxauthsize = SHA256_DIGEST_SIZE,
1236 },
1237 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1238 DESC_HDR_SEL0_AESU |
1239 DESC_HDR_MODE0_AESU_CBC |
1240 DESC_HDR_SEL1_MDEUA |
1241 DESC_HDR_MODE1_MDEU_INIT |
1242 DESC_HDR_MODE1_MDEU_PAD |
1243 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1244 },
1245 {
1246 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1247 .driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1248 .blocksize = DES3_EDE_BLOCK_SIZE,
1249 .aead = {
1250 .setkey = aead_authenc_setkey,
1251 .setauthsize = aead_authenc_setauthsize,
1252 .encrypt = aead_authenc_encrypt,
1253 .decrypt = aead_authenc_decrypt,
1254 .givencrypt = aead_authenc_givencrypt,
1255 .geniv = "<built-in>",
1256 .ivsize = DES3_EDE_BLOCK_SIZE,
1257 .maxauthsize = SHA256_DIGEST_SIZE,
1258 },
1259 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1260 DESC_HDR_SEL0_DEU |
1261 DESC_HDR_MODE0_DEU_CBC |
1262 DESC_HDR_MODE0_DEU_3DES |
1263 DESC_HDR_SEL1_MDEUA |
1264 DESC_HDR_MODE1_MDEU_INIT |
1265 DESC_HDR_MODE1_MDEU_PAD |
1266 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1267 },
1268 {
1269 .name = "authenc(hmac(md5),cbc(aes))",
1270 .driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1271 .blocksize = AES_BLOCK_SIZE,
1272 .aead = {
1273 .setkey = aead_authenc_setkey,
1274 .setauthsize = aead_authenc_setauthsize,
1275 .encrypt = aead_authenc_encrypt,
1276 .decrypt = aead_authenc_decrypt,
1277 .givencrypt = aead_authenc_givencrypt,
1278 .geniv = "<built-in>",
1279 .ivsize = AES_BLOCK_SIZE,
1280 .maxauthsize = MD5_DIGEST_SIZE,
1281 },
1282 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1283 DESC_HDR_SEL0_AESU |
1284 DESC_HDR_MODE0_AESU_CBC |
1285 DESC_HDR_SEL1_MDEUA |
1286 DESC_HDR_MODE1_MDEU_INIT |
1287 DESC_HDR_MODE1_MDEU_PAD |
1288 DESC_HDR_MODE1_MDEU_MD5_HMAC,
1289 },
1290 {
1291 .name = "authenc(hmac(md5),cbc(des3_ede))",
1292 .driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1293 .blocksize = DES3_EDE_BLOCK_SIZE,
1294 .aead = {
1295 .setkey = aead_authenc_setkey,
1296 .setauthsize = aead_authenc_setauthsize,
1297 .encrypt = aead_authenc_encrypt,
1298 .decrypt = aead_authenc_decrypt,
1299 .givencrypt = aead_authenc_givencrypt,
1300 .geniv = "<built-in>",
1301 .ivsize = DES3_EDE_BLOCK_SIZE,
1302 .maxauthsize = MD5_DIGEST_SIZE,
1303 },
1304 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1305 DESC_HDR_SEL0_DEU |
1306 DESC_HDR_MODE0_DEU_CBC |
1307 DESC_HDR_MODE0_DEU_3DES |
1308 DESC_HDR_SEL1_MDEUA |
1309 DESC_HDR_MODE1_MDEU_INIT |
1310 DESC_HDR_MODE1_MDEU_PAD |
1311 DESC_HDR_MODE1_MDEU_MD5_HMAC,
9c4a7965
KP		 1312 }
1313};
1314
1315struct talitos_crypto_alg {
1316 struct list_head entry;
1317 struct device *dev;
1318 __be32 desc_hdr_template;
1319 struct crypto_alg crypto_alg;
1320};
1321
1322static int talitos_cra_init(struct crypto_tfm *tfm)
1323{
1324 struct crypto_alg *alg = tfm->__crt_alg;
1325 struct talitos_crypto_alg *talitos_alg =
1326 container_of(alg, struct talitos_crypto_alg, crypto_alg);
1327 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1328
1329 /* update context with ptr to dev */
1330 ctx->dev = talitos_alg->dev;
1331 /* copy descriptor header template value */
1332 ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1333
1334 /* random first IV */
70bcaca7 1335 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
9c4a7965
KP		 1336
1337 return 0;
1338}
1339
1340/*
1341 * given the alg's descriptor header template, determine whether descriptor
1342 * type and primary/secondary execution units required match the hw
1343 * capabilities description provided in the device tree node.
1344 */
1345static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1346{
1347 struct talitos_private *priv = dev_get_drvdata(dev);
1348 int ret;
1349
1350 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1351 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1352
1353 if (SECONDARY_EU(desc_hdr_template))
1354 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1355 & priv->exec_units);
1356
1357 return ret;
1358}
1359
1360static int __devexit talitos_remove(struct of_device *ofdev)
1361{
1362 struct device *dev = &ofdev->dev;
1363 struct talitos_private *priv = dev_get_drvdata(dev);
1364 struct talitos_crypto_alg *t_alg, *n;
1365 int i;
1366
1367 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1368 crypto_unregister_alg(&t_alg->crypto_alg);
1369 list_del(&t_alg->entry);
1370 kfree(t_alg);
1371 }
1372
1373 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1374 talitos_unregister_rng(dev);
1375
ec6644d6 1376 kfree(priv->submit_count);
9c4a7965
KP		 1377 kfree(priv->tail);
1378 kfree(priv->head);
1379
1380 if (priv->fifo)
1381 for (i = 0; i < priv->num_channels; i++)
1382 kfree(priv->fifo[i]);
1383
1384 kfree(priv->fifo);
1385 kfree(priv->head_lock);
1386 kfree(priv->tail_lock);
1387
1388 if (priv->irq != NO_IRQ) {
1389 free_irq(priv->irq, dev);
1390 irq_dispose_mapping(priv->irq);
1391 }
1392
1393 tasklet_kill(&priv->done_task);
1394 tasklet_kill(&priv->error_task);
1395
1396 iounmap(priv->reg);
1397
1398 dev_set_drvdata(dev, NULL);
1399
1400 kfree(priv);
1401
1402 return 0;
1403}
1404
1405static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1406 struct talitos_alg_template
1407 *template)
1408{
1409 struct talitos_crypto_alg *t_alg;
1410 struct crypto_alg *alg;
1411
1412 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1413 if (!t_alg)
1414 return ERR_PTR(-ENOMEM);
1415
1416 alg = &t_alg->crypto_alg;
1417
1418 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1419 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1420 template->driver_name);
1421 alg->cra_module = THIS_MODULE;
1422 alg->cra_init = talitos_cra_init;
1423 alg->cra_priority = TALITOS_CRA_PRIORITY;
1424 alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1425 alg->cra_blocksize = template->blocksize;
1426 alg->cra_alignmask = 0;
1427 alg->cra_type = &crypto_aead_type;
1428 alg->cra_ctxsize = sizeof(struct talitos_ctx);
1429 alg->cra_u.aead = template->aead;
1430
1431 t_alg->desc_hdr_template = template->desc_hdr_template;
1432 t_alg->dev = dev;
1433
1434 return t_alg;
1435}
1436
1437static int talitos_probe(struct of_device *ofdev,
1438 const struct of_device_id *match)
1439{
1440 struct device *dev = &ofdev->dev;
1441 struct device_node *np = ofdev->node;
1442 struct talitos_private *priv;
1443 const unsigned int *prop;
1444 int i, err;
1445
1446 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1447 if (!priv)
1448 return -ENOMEM;
1449
1450 dev_set_drvdata(dev, priv);
1451
1452 priv->ofdev = ofdev;
1453
ba95487d
KP		 1454 INIT_LIST_HEAD(&priv->alg_list);
1455
9c4a7965
KP		 1456 tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1457 tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev);
1458
1459 priv->irq = irq_of_parse_and_map(np, 0);
1460
1461 if (priv->irq == NO_IRQ) {
1462 dev_err(dev, "failed to map irq\n");
1463 err = -EINVAL;
1464 goto err_out;
1465 }
1466
1467 /* get the irq line */
1468 err = request_irq(priv->irq, talitos_interrupt, 0,
1469 dev_driver_string(dev), dev);
1470 if (err) {
1471 dev_err(dev, "failed to request irq %d\n", priv->irq);
1472 irq_dispose_mapping(priv->irq);
1473 priv->irq = NO_IRQ;
1474 goto err_out;
1475 }
1476
1477 priv->reg = of_iomap(np, 0);
1478 if (!priv->reg) {
1479 dev_err(dev, "failed to of_iomap\n");
1480 err = -ENOMEM;
1481 goto err_out;
1482 }
1483
1484 /* get SEC version capabilities from device tree */
1485 prop = of_get_property(np, "fsl,num-channels", NULL);
1486 if (prop)
1487 priv->num_channels = *prop;
1488
1489 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1490 if (prop)
1491 priv->chfifo_len = *prop;
1492
1493 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1494 if (prop)
1495 priv->exec_units = *prop;
1496
1497 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1498 if (prop)
1499 priv->desc_types = *prop;
1500
1501 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1502 !priv->exec_units || !priv->desc_types) {
1503 dev_err(dev, "invalid property data in device tree node\n");
1504 err = -EINVAL;
1505 goto err_out;
1506 }
1507
f3c85bc1
LN		 1508 if (of_device_is_compatible(np, "fsl,sec3.0"))
1509 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1510
9c4a7965
KP		 1511 priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1512 GFP_KERNEL);
1513 priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1514 GFP_KERNEL);
1515 if (!priv->head_lock || !priv->tail_lock) {
1516 dev_err(dev, "failed to allocate fifo locks\n");
1517 err = -ENOMEM;
1518 goto err_out;
1519 }
1520
1521 for (i = 0; i < priv->num_channels; i++) {
1522 spin_lock_init(&priv->head_lock[i]);
1523 spin_lock_init(&priv->tail_lock[i]);
1524 }
1525
1526 priv->fifo = kmalloc(sizeof(struct talitos_request *) *
1527 priv->num_channels, GFP_KERNEL);
1528 if (!priv->fifo) {
1529 dev_err(dev, "failed to allocate request fifo\n");
1530 err = -ENOMEM;
1531 goto err_out;
1532 }
1533
1534 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1535
1536 for (i = 0; i < priv->num_channels; i++) {
1537 priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
1538 priv->fifo_len, GFP_KERNEL);
1539 if (!priv->fifo[i]) {
1540 dev_err(dev, "failed to allocate request fifo %d\n", i);
1541 err = -ENOMEM;
1542 goto err_out;
1543 }
1544 }
1545
586725f8 1546 priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels,
ec6644d6
KP		 1547 GFP_KERNEL);
1548 if (!priv->submit_count) {
1549 dev_err(dev, "failed to allocate fifo submit count space\n");
1550 err = -ENOMEM;
1551 goto err_out;
1552 }
1553 for (i = 0; i < priv->num_channels; i++)
1554 atomic_set(&priv->submit_count[i], -priv->chfifo_len);
1555
9c4a7965
KP		 1556 priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1557 priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1558 if (!priv->head || !priv->tail) {
1559 dev_err(dev, "failed to allocate request index space\n");
1560 err = -ENOMEM;
1561 goto err_out;
1562 }
1563
1564 /* reset and initialize the h/w */
1565 err = init_device(dev);
1566 if (err) {
1567 dev_err(dev, "failed to initialize device\n");
1568 goto err_out;
1569 }
1570
1571 /* register the RNG, if available */
1572 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1573 err = talitos_register_rng(dev);
1574 if (err) {
1575 dev_err(dev, "failed to register hwrng: %d\n", err);
1576 goto err_out;
1577 } else
1578 dev_info(dev, "hwrng\n");
1579 }
1580
1581 /* register crypto algorithms the device supports */
9c4a7965
KP		 1582 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1583 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1584 struct talitos_crypto_alg *t_alg;
1585
1586 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1587 if (IS_ERR(t_alg)) {
1588 err = PTR_ERR(t_alg);
1589 goto err_out;
1590 }
1591
1592 err = crypto_register_alg(&t_alg->crypto_alg);
1593 if (err) {
1594 dev_err(dev, "%s alg registration failed\n",
1595 t_alg->crypto_alg.cra_driver_name);
1596 kfree(t_alg);
1597 } else {
1598 list_add_tail(&t_alg->entry, &priv->alg_list);
1599 dev_info(dev, "%s\n",
1600 t_alg->crypto_alg.cra_driver_name);
1601 }
1602 }
1603 }
1604
1605 return 0;
1606
1607err_out:
1608 talitos_remove(ofdev);
9c4a7965
KP		 1609
1610 return err;
1611}
1612
1613static struct of_device_id talitos_match[] = {
1614 {
1615 .compatible = "fsl,sec2.0",
1616 },
1617 {},
1618};
1619MODULE_DEVICE_TABLE(of, talitos_match);
1620
1621static struct of_platform_driver talitos_driver = {
1622 .name = "talitos",
1623 .match_table = talitos_match,
1624 .probe = talitos_probe,
1625 .remove = __devexit_p(talitos_remove),
1626};
1627
1628static int __init talitos_init(void)
1629{
1630 return of_register_platform_driver(&talitos_driver);
1631}
1632module_init(talitos_init);
1633
1634static void __exit talitos_exit(void)
1635{
1636 of_unregister_platform_driver(&talitos_driver);
1637}
1638module_exit(talitos_exit);
1639
1640MODULE_LICENSE("GPL");
1641MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1642MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Linux 6.x block layer and io_uring tree(s)