projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: talitos - fix ECB algs ivsize
[linux-2.6-block.git] / drivers / crypto / talitos.c
CommitLineData
9c4a7965
KP		 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
5228f0f7 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
9c4a7965
KP		 5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
5af50730
RH		 35#include <linux/of_address.h>
36#include <linux/of_irq.h>
9c4a7965
KP		 37#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
5a0e3ad6 42#include <linux/slab.h>
9c4a7965
KP		 43
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
3952f17e 46#include <crypto/des.h>
9c4a7965 47#include <crypto/sha.h>
497f2e6b 48#include <crypto/md5.h>
e98014ab 49#include <crypto/internal/aead.h>
9c4a7965 50#include <crypto/authenc.h>
4de9d0b5 51#include <crypto/skcipher.h>
acbf7c62
LN		 52#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
4de9d0b5 54#include <crypto/scatterwalk.h>
9c4a7965
KP		 55
56#include "talitos.h"
57
922f9dc8 58static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
da9de146 59 unsigned int len, bool is_sec1)
81eb024c 60{
edc6bd69 61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
da9de146
LC		 62 if (is_sec1) {
63 ptr->len1 = cpu_to_be16(len);
64 } else {
65 ptr->len = cpu_to_be16(len);
922f9dc8 66 ptr->eptr = upper_32_bits(dma_addr);
da9de146 67 }
81eb024c
KP		 68}
69
340ff60a
HG		 70static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
71 struct talitos_ptr *src_ptr, bool is_sec1)
72{
73 dst_ptr->ptr = src_ptr->ptr;
922f9dc8 74 if (is_sec1) {
da9de146 75 dst_ptr->len1 = src_ptr->len1;
922f9dc8 76 } else {
da9de146
LC		 77 dst_ptr->len = src_ptr->len;
78 dst_ptr->eptr = src_ptr->eptr;
922f9dc8 79 }
538caf83
LC		 80}
81
922f9dc8
LC		 82static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
83 bool is_sec1)
538caf83 84{
922f9dc8
LC		 85 if (is_sec1)
86 return be16_to_cpu(ptr->len1);
87 else
88 return be16_to_cpu(ptr->len);
538caf83
LC		 89}
90
b096b544
LC		 91static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
92 bool is_sec1)
185eb79f 93{
922f9dc8 94 if (!is_sec1)
b096b544
LC		 95 ptr->j_extent = val;
96}
97
98static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
99{
100 if (!is_sec1)
101 ptr->j_extent |= val;
185eb79f
LC		 102}
103
9c4a7965
KP		 104/*
105 * map virtual single (contiguous) pointer to h/w descriptor pointer
106 */
6a4967c3
LC		 107static void __map_single_talitos_ptr(struct device *dev,
108 struct talitos_ptr *ptr,
109 unsigned int len, void *data,
110 enum dma_data_direction dir,
111 unsigned long attrs)
112{
113 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
114 struct talitos_private *priv = dev_get_drvdata(dev);
115 bool is_sec1 = has_ftr_sec1(priv);
116
117 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
118}
119
9c4a7965 120static void map_single_talitos_ptr(struct device *dev,
edc6bd69 121 struct talitos_ptr *ptr,
42e8b0d7 122 unsigned int len, void *data,
9c4a7965
KP		 123 enum dma_data_direction dir)
124{
6a4967c3
LC		 125 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
126}
81eb024c 127
6a4967c3
LC		 128static void map_single_talitos_ptr_nosync(struct device *dev,
129 struct talitos_ptr *ptr,
130 unsigned int len, void *data,
131 enum dma_data_direction dir)
132{
133 __map_single_talitos_ptr(dev, ptr, len, data, dir,
134 DMA_ATTR_SKIP_CPU_SYNC);
9c4a7965
KP		 135}
136
137/*
138 * unmap bus single (contiguous) h/w descriptor pointer
139 */
140static void unmap_single_talitos_ptr(struct device *dev,
edc6bd69 141 struct talitos_ptr *ptr,
9c4a7965
KP		 142 enum dma_data_direction dir)
143{
922f9dc8
LC		 144 struct talitos_private *priv = dev_get_drvdata(dev);
145 bool is_sec1 = has_ftr_sec1(priv);
146
edc6bd69 147 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
922f9dc8 148 from_talitos_ptr_len(ptr, is_sec1), dir);
9c4a7965
KP		 149}
150
151static int reset_channel(struct device *dev, int ch)
152{
153 struct talitos_private *priv = dev_get_drvdata(dev);
154 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 155 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 156
dd3c0987
LC		 157 if (is_sec1) {
158 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
159 TALITOS1_CCCR_LO_RESET);
9c4a7965 160
dd3c0987
LC		 161 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
162 TALITOS1_CCCR_LO_RESET) && --timeout)
163 cpu_relax();
164 } else {
165 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
166 TALITOS2_CCCR_RESET);
167
168 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
169 TALITOS2_CCCR_RESET) && --timeout)
170 cpu_relax();
171 }
9c4a7965
KP		 172
173 if (timeout == 0) {
174 dev_err(dev, "failed to reset channel %d\n", ch);
175 return -EIO;
176 }
177
81eb024c 178 /* set 36-bit addressing, done writeback enable and done IRQ enable */
ad42d5fc 179 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
81eb024c 180 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
37b5e889
LC		 181 /* enable chaining descriptors */
182 if (is_sec1)
183 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
184 TALITOS_CCCR_LO_NE);
9c4a7965 185
fe5720e2
KP		 186 /* and ICCR writeback, if available */
187 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
ad42d5fc 188 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
fe5720e2
KP		 189 TALITOS_CCCR_LO_IWSE);
190
9c4a7965
KP		 191 return 0;
192}
193
194static int reset_device(struct device *dev)
195{
196 struct talitos_private *priv = dev_get_drvdata(dev);
197 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987
LC		 198 bool is_sec1 = has_ftr_sec1(priv);
199 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
9c4a7965 200
c3e337f8 201 setbits32(priv->reg + TALITOS_MCR, mcr);
9c4a7965 202
dd3c0987 203 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
9c4a7965
KP		 204 && --timeout)
205 cpu_relax();
206
2cdba3cf 207 if (priv->irq[1]) {
c3e337f8
KP		 208 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
209 setbits32(priv->reg + TALITOS_MCR, mcr);
210 }
211
9c4a7965
KP		 212 if (timeout == 0) {
213 dev_err(dev, "failed to reset device\n");
214 return -EIO;
215 }
216
217 return 0;
218}
219
220/*
221 * Reset and initialize the device
222 */
223static int init_device(struct device *dev)
224{
225 struct talitos_private *priv = dev_get_drvdata(dev);
226 int ch, err;
dd3c0987 227 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965
KP		 228
229 /*
230 * Master reset
231 * errata documentation: warning: certain SEC interrupts
232 * are not fully cleared by writing the MCR:SWR bit,
233 * set bit twice to completely reset
234 */
235 err = reset_device(dev);
236 if (err)
237 return err;
238
239 err = reset_device(dev);
240 if (err)
241 return err;
242
243 /* reset channels */
244 for (ch = 0; ch < priv->num_channels; ch++) {
245 err = reset_channel(dev, ch);
246 if (err)
247 return err;
248 }
249
250 /* enable channel done and error interrupts */
dd3c0987
LC		 251 if (is_sec1) {
252 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
253 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
254 /* disable parity error check in DEU (erroneous? test vect.) */
255 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
256 } else {
257 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
258 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
259 }
9c4a7965 260
fe5720e2
KP		 261 /* disable integrity check error interrupts (use writeback instead) */
262 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
5fa7fa14 263 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
fe5720e2
KP		 264 TALITOS_MDEUICR_LO_ICE);
265
9c4a7965
KP		 266 return 0;
267}
268
269/**
270 * talitos_submit - submits a descriptor to the device for processing
271 * @dev: the SEC device to be used
5228f0f7 272 * @ch: the SEC device channel to be used
9c4a7965
KP		 273 * @desc: the descriptor to be processed by the device
274 * @callback: whom to call when processing is complete
275 * @context: a handle for use by caller (optional)
276 *
277 * desc must contain valid dma-mapped (bus physical) address pointers.
278 * callback must check err and feedback in descriptor header
279 * for device processing status.
280 */
865d5061
HG		 281int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
282 void (*callback)(struct device *dev,
283 struct talitos_desc *desc,
284 void *context, int error),
285 void *context)
9c4a7965
KP		 286{
287 struct talitos_private *priv = dev_get_drvdata(dev);
288 struct talitos_request *request;
5228f0f7 289 unsigned long flags;
9c4a7965 290 int head;
7d607c6a 291 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 292
4b992628 293 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
9c4a7965 294
4b992628 295 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
ec6644d6 296 /* h/w fifo is full */
4b992628 297 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 298 return -EAGAIN;
299 }
300
4b992628
KP		 301 head = priv->chan[ch].head;
302 request = &priv->chan[ch].fifo[head];
ec6644d6 303
9c4a7965 304 /* map descriptor and save caller data */
7d607c6a
LC		 305 if (is_sec1) {
306 desc->hdr1 = desc->hdr;
7d607c6a
LC		 307 request->dma_desc = dma_map_single(dev, &desc->hdr1,
308 TALITOS_DESC_SIZE,
309 DMA_BIDIRECTIONAL);
310 } else {
311 request->dma_desc = dma_map_single(dev, desc,
312 TALITOS_DESC_SIZE,
313 DMA_BIDIRECTIONAL);
314 }
9c4a7965
KP		 315 request->callback = callback;
316 request->context = context;
317
318 /* increment fifo head */
4b992628 319 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
9c4a7965
KP		 320
321 smp_wmb();
322 request->desc = desc;
323
324 /* GO! */
325 wmb();
ad42d5fc
KP		 326 out_be32(priv->chan[ch].reg + TALITOS_FF,
327 upper_32_bits(request->dma_desc));
328 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
a752447a 329 lower_32_bits(request->dma_desc));
9c4a7965 330
4b992628 331 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 332
333 return -EINPROGRESS;
334}
865d5061 335EXPORT_SYMBOL(talitos_submit);
9c4a7965
KP		 336
337/*
338 * process what was done, notify callback of error if not
339 */
340static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
341{
342 struct talitos_private *priv = dev_get_drvdata(dev);
343 struct talitos_request *request, saved_req;
344 unsigned long flags;
345 int tail, status;
7d607c6a 346 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 347
4b992628 348 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
9c4a7965 349
4b992628
KP		 350 tail = priv->chan[ch].tail;
351 while (priv->chan[ch].fifo[tail].desc) {
7d607c6a
LC		 352 __be32 hdr;
353
4b992628 354 request = &priv->chan[ch].fifo[tail];
9c4a7965
KP		 355
356 /* descriptors with their done bits set don't get the error */
357 rmb();
37b5e889
LC		 358 if (!is_sec1)
359 hdr = request->desc->hdr;
360 else if (request->desc->next_desc)
361 hdr = (request->desc + 1)->hdr1;
362 else
363 hdr = request->desc->hdr1;
7d607c6a
LC		 364
365 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
9c4a7965 366 status = 0;
ca38a814 367 else
9c4a7965
KP		 368 if (!error)
369 break;
370 else
371 status = error;
372
373 dma_unmap_single(dev, request->dma_desc,
7d607c6a 374 TALITOS_DESC_SIZE,
e938e465 375 DMA_BIDIRECTIONAL);
9c4a7965
KP		 376
377 /* copy entries so we can call callback outside lock */
378 saved_req.desc = request->desc;
379 saved_req.callback = request->callback;
380 saved_req.context = request->context;
381
382 /* release request entry in fifo */
383 smp_wmb();
384 request->desc = NULL;
385
386 /* increment fifo tail */
4b992628 387 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
9c4a7965 388
4b992628 389 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
ec6644d6 390
4b992628 391 atomic_dec(&priv->chan[ch].submit_count);
ec6644d6 392
9c4a7965
KP		 393 saved_req.callback(dev, saved_req.desc, saved_req.context,
394 status);
395 /* channel may resume processing in single desc error case */
396 if (error && !reset_ch && status == error)
397 return;
4b992628
KP		 398 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
399 tail = priv->chan[ch].tail;
9c4a7965
KP		 400 }
401
4b992628 402 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
9c4a7965
KP		 403}
404
405/*
406 * process completed requests for channels that have done status
407 */
dd3c0987
LC		 408#define DEF_TALITOS1_DONE(name, ch_done_mask) \
409static void talitos1_done_##name(unsigned long data) \
410{ \
411 struct device *dev = (struct device *)data; \
412 struct talitos_private *priv = dev_get_drvdata(dev); \
413 unsigned long flags; \
414 \
415 if (ch_done_mask & 0x10000000) \
416 flush_channel(dev, 0, 0, 0); \
dd3c0987
LC		 417 if (ch_done_mask & 0x40000000) \
418 flush_channel(dev, 1, 0, 0); \
419 if (ch_done_mask & 0x00010000) \
420 flush_channel(dev, 2, 0, 0); \
421 if (ch_done_mask & 0x00040000) \
422 flush_channel(dev, 3, 0, 0); \
423 \
dd3c0987
LC		 424 /* At this point, all completed channels have been processed */ \
425 /* Unmask done interrupts for channels completed later on. */ \
426 spin_lock_irqsave(&priv->reg_lock, flags); \
427 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
428 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
429 spin_unlock_irqrestore(&priv->reg_lock, flags); \
430}
431
432DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
9c02e285 433DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
dd3c0987
LC		 434
435#define DEF_TALITOS2_DONE(name, ch_done_mask) \
436static void talitos2_done_##name(unsigned long data) \
c3e337f8
KP		 437{ \
438 struct device *dev = (struct device *)data; \
439 struct talitos_private *priv = dev_get_drvdata(dev); \
511d63cb 440 unsigned long flags; \
c3e337f8
KP		 441 \
442 if (ch_done_mask & 1) \
443 flush_channel(dev, 0, 0, 0); \
c3e337f8
KP		 444 if (ch_done_mask & (1 << 2)) \
445 flush_channel(dev, 1, 0, 0); \
446 if (ch_done_mask & (1 << 4)) \
447 flush_channel(dev, 2, 0, 0); \
448 if (ch_done_mask & (1 << 6)) \
449 flush_channel(dev, 3, 0, 0); \
450 \
c3e337f8
KP		 451 /* At this point, all completed channels have been processed */ \
452 /* Unmask done interrupts for channels completed later on. */ \
511d63cb 453 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8 454 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
dd3c0987 455 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
511d63cb 456 spin_unlock_irqrestore(&priv->reg_lock, flags); \
9c4a7965 457}
dd3c0987
LC		 458
459DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
9c02e285 460DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
dd3c0987
LC		 461DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
462DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
9c4a7965
KP		 463
464/*
465 * locate current (offending) descriptor
466 */
3e721aeb 467static u32 current_desc_hdr(struct device *dev, int ch)
9c4a7965
KP		 468{
469 struct talitos_private *priv = dev_get_drvdata(dev);
b62ffd8c 470 int tail, iter;
9c4a7965
KP		 471 dma_addr_t cur_desc;
472
b62ffd8c
HG		 473 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
474 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
9c4a7965 475
b62ffd8c
HG		 476 if (!cur_desc) {
477 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
478 return 0;
479 }
480
481 tail = priv->chan[ch].tail;
482
483 iter = tail;
37b5e889
LC		 484 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
485 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) {
b62ffd8c
HG		 486 iter = (iter + 1) & (priv->fifo_len - 1);
487 if (iter == tail) {
9c4a7965 488 dev_err(dev, "couldn't locate current descriptor\n");
3e721aeb 489 return 0;
9c4a7965
KP		 490 }
491 }
492
37b5e889
LC		 493 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
494 return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
495
b62ffd8c 496 return priv->chan[ch].fifo[iter].desc->hdr;
9c4a7965
KP		 497}
498
499/*
500 * user diagnostics; report root cause of error based on execution unit status
501 */
3e721aeb 502static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
9c4a7965
KP		 503{
504 struct talitos_private *priv = dev_get_drvdata(dev);
505 int i;
506
3e721aeb 507 if (!desc_hdr)
ad42d5fc 508 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
3e721aeb
KP		 509
510 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
9c4a7965
KP		 511 case DESC_HDR_SEL0_AFEU:
512 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 513 in_be32(priv->reg_afeu + TALITOS_EUISR),
514 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
9c4a7965
KP		 515 break;
516 case DESC_HDR_SEL0_DEU:
517 dev_err(dev, "DEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 518 in_be32(priv->reg_deu + TALITOS_EUISR),
519 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
9c4a7965
KP		 520 break;
521 case DESC_HDR_SEL0_MDEUA:
522 case DESC_HDR_SEL0_MDEUB:
523 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 524 in_be32(priv->reg_mdeu + TALITOS_EUISR),
525 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 526 break;
527 case DESC_HDR_SEL0_RNG:
528 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
5fa7fa14
LC		 529 in_be32(priv->reg_rngu + TALITOS_ISR),
530 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
9c4a7965
KP		 531 break;
532 case DESC_HDR_SEL0_PKEU:
533 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 534 in_be32(priv->reg_pkeu + TALITOS_EUISR),
535 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 536 break;
537 case DESC_HDR_SEL0_AESU:
538 dev_err(dev, "AESUISR 0x%08x_%08x\n",
5fa7fa14
LC		 539 in_be32(priv->reg_aesu + TALITOS_EUISR),
540 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
9c4a7965
KP		 541 break;
542 case DESC_HDR_SEL0_CRCU:
543 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 544 in_be32(priv->reg_crcu + TALITOS_EUISR),
545 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 546 break;
547 case DESC_HDR_SEL0_KEU:
548 dev_err(dev, "KEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 549 in_be32(priv->reg_pkeu + TALITOS_EUISR),
550 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 551 break;
552 }
553
3e721aeb 554 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
9c4a7965
KP		 555 case DESC_HDR_SEL1_MDEUA:
556 case DESC_HDR_SEL1_MDEUB:
557 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 558 in_be32(priv->reg_mdeu + TALITOS_EUISR),
559 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 560 break;
561 case DESC_HDR_SEL1_CRCU:
562 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 563 in_be32(priv->reg_crcu + TALITOS_EUISR),
564 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 565 break;
566 }
567
568 for (i = 0; i < 8; i++)
569 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
ad42d5fc
KP		 570 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
571 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
9c4a7965
KP		 572}
573
574/*
575 * recover from error interrupts
576 */
5e718a09 577static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
9c4a7965 578{
9c4a7965
KP		 579 struct talitos_private *priv = dev_get_drvdata(dev);
580 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 581 int ch, error, reset_dev = 0;
42e8b0d7 582 u32 v_lo;
dd3c0987
LC		 583 bool is_sec1 = has_ftr_sec1(priv);
584 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
9c4a7965
KP		 585
586 for (ch = 0; ch < priv->num_channels; ch++) {
587 /* skip channels without errors */
dd3c0987
LC		 588 if (is_sec1) {
589 /* bits 29, 31, 17, 19 */
590 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
591 continue;
592 } else {
593 if (!(isr & (1 << (ch * 2 + 1))))
594 continue;
595 }
9c4a7965
KP		 596
597 error = -EINVAL;
598
ad42d5fc 599 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
9c4a7965
KP		 600
601 if (v_lo & TALITOS_CCPSR_LO_DOF) {
602 dev_err(dev, "double fetch fifo overflow error\n");
603 error = -EAGAIN;
604 reset_ch = 1;
605 }
606 if (v_lo & TALITOS_CCPSR_LO_SOF) {
607 /* h/w dropped descriptor */
608 dev_err(dev, "single fetch fifo overflow error\n");
609 error = -EAGAIN;
610 }
611 if (v_lo & TALITOS_CCPSR_LO_MDTE)
612 dev_err(dev, "master data transfer error\n");
613 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
4d9b3a5b 614 dev_err(dev, is_sec1 ? "pointer not complete error\n"
dd3c0987 615 : "s/g data length zero error\n");
9c4a7965 616 if (v_lo & TALITOS_CCPSR_LO_FPZ)
dd3c0987
LC		 617 dev_err(dev, is_sec1 ? "parity error\n"
618 : "fetch pointer zero error\n");
9c4a7965
KP		 619 if (v_lo & TALITOS_CCPSR_LO_IDH)
620 dev_err(dev, "illegal descriptor header error\n");
621 if (v_lo & TALITOS_CCPSR_LO_IEU)
dd3c0987
LC		 622 dev_err(dev, is_sec1 ? "static assignment error\n"
623 : "invalid exec unit error\n");
9c4a7965 624 if (v_lo & TALITOS_CCPSR_LO_EU)
3e721aeb 625 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
dd3c0987
LC		 626 if (!is_sec1) {
627 if (v_lo & TALITOS_CCPSR_LO_GB)
628 dev_err(dev, "gather boundary error\n");
629 if (v_lo & TALITOS_CCPSR_LO_GRL)
630 dev_err(dev, "gather return/length error\n");
631 if (v_lo & TALITOS_CCPSR_LO_SB)
632 dev_err(dev, "scatter boundary error\n");
633 if (v_lo & TALITOS_CCPSR_LO_SRL)
634 dev_err(dev, "scatter return/length error\n");
635 }
9c4a7965
KP		 636
637 flush_channel(dev, ch, error, reset_ch);
638
639 if (reset_ch) {
640 reset_channel(dev, ch);
641 } else {
ad42d5fc 642 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
dd3c0987 643 TALITOS2_CCCR_CONT);
ad42d5fc
KP		 644 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
645 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
dd3c0987 646 TALITOS2_CCCR_CONT) && --timeout)
9c4a7965
KP		 647 cpu_relax();
648 if (timeout == 0) {
649 dev_err(dev, "failed to restart channel %d\n",
650 ch);
651 reset_dev = 1;
652 }
653 }
654 }
dd3c0987
LC		 655 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
656 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
657 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
658 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
659 isr, isr_lo);
660 else
661 dev_err(dev, "done overflow, internal time out, or "
662 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
9c4a7965
KP		 663
664 /* purge request queues */
665 for (ch = 0; ch < priv->num_channels; ch++)
666 flush_channel(dev, ch, -EIO, 1);
667
668 /* reset and reinitialize the device */
669 init_device(dev);
670 }
671}
672
dd3c0987
LC		 673#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
674static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
675{ \
676 struct device *dev = data; \
677 struct talitos_private *priv = dev_get_drvdata(dev); \
678 u32 isr, isr_lo; \
679 unsigned long flags; \
680 \
681 spin_lock_irqsave(&priv->reg_lock, flags); \
682 isr = in_be32(priv->reg + TALITOS_ISR); \
683 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
684 /* Acknowledge interrupt */ \
685 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
686 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
687 \
688 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
689 spin_unlock_irqrestore(&priv->reg_lock, flags); \
690 talitos_error(dev, isr & ch_err_mask, isr_lo); \
691 } \
692 else { \
693 if (likely(isr & ch_done_mask)) { \
694 /* mask further done interrupts. */ \
695 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
696 /* done_task will unmask done interrupts at exit */ \
697 tasklet_schedule(&priv->done_task[tlet]); \
698 } \
699 spin_unlock_irqrestore(&priv->reg_lock, flags); \
700 } \
701 \
702 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
703 IRQ_NONE; \
704}
705
706DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
707
708#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
709static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
c3e337f8
KP		 710{ \
711 struct device *dev = data; \
712 struct talitos_private *priv = dev_get_drvdata(dev); \
713 u32 isr, isr_lo; \
511d63cb 714 unsigned long flags; \
c3e337f8 715 \
511d63cb 716 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8
KP		 717 isr = in_be32(priv->reg + TALITOS_ISR); \
718 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
719 /* Acknowledge interrupt */ \
720 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
721 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
722 \
511d63cb
HG		 723 if (unlikely(isr & ch_err_mask || isr_lo)) { \
724 spin_unlock_irqrestore(&priv->reg_lock, flags); \
725 talitos_error(dev, isr & ch_err_mask, isr_lo); \
726 } \
727 else { \
c3e337f8
KP		 728 if (likely(isr & ch_done_mask)) { \
729 /* mask further done interrupts. */ \
730 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
731 /* done_task will unmask done interrupts at exit */ \
732 tasklet_schedule(&priv->done_task[tlet]); \
733 } \
511d63cb
HG		 734 spin_unlock_irqrestore(&priv->reg_lock, flags); \
735 } \
c3e337f8
KP		 736 \
737 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
738 IRQ_NONE; \
9c4a7965 739}
dd3c0987
LC		 740
741DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
742DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
743 0)
744DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
745 1)
9c4a7965
KP		 746
747/*
748 * hwrng
749 */
750static int talitos_rng_data_present(struct hwrng *rng, int wait)
751{
752 struct device *dev = (struct device *)rng->priv;
753 struct talitos_private *priv = dev_get_drvdata(dev);
754 u32 ofl;
755 int i;
756
757 for (i = 0; i < 20; i++) {
5fa7fa14 758 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
9c4a7965
KP		 759 TALITOS_RNGUSR_LO_OFL;
760 if (ofl || !wait)
761 break;
762 udelay(10);
763 }
764
765 return !!ofl;
766}
767
768static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
769{
770 struct device *dev = (struct device *)rng->priv;
771 struct talitos_private *priv = dev_get_drvdata(dev);
772
773 /* rng fifo requires 64-bit accesses */
5fa7fa14
LC		 774 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
775 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
9c4a7965
KP		 776
777 return sizeof(u32);
778}
779
780static int talitos_rng_init(struct hwrng *rng)
781{
782 struct device *dev = (struct device *)rng->priv;
783 struct talitos_private *priv = dev_get_drvdata(dev);
784 unsigned int timeout = TALITOS_TIMEOUT;
785
5fa7fa14
LC		 786 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
787 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
788 & TALITOS_RNGUSR_LO_RD)
9c4a7965
KP		 789 && --timeout)
790 cpu_relax();
791 if (timeout == 0) {
792 dev_err(dev, "failed to reset rng hw\n");
793 return -ENODEV;
794 }
795
796 /* start generating */
5fa7fa14 797 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
9c4a7965
KP		 798
799 return 0;
800}
801
802static int talitos_register_rng(struct device *dev)
803{
804 struct talitos_private *priv = dev_get_drvdata(dev);
35a3bb3d 805 int err;
9c4a7965
KP		 806
807 priv->rng.name = dev_driver_string(dev),
808 priv->rng.init = talitos_rng_init,
809 priv->rng.data_present = talitos_rng_data_present,
810 priv->rng.data_read = talitos_rng_data_read,
811 priv->rng.priv = (unsigned long)dev;
812
35a3bb3d
AS		 813 err = hwrng_register(&priv->rng);
814 if (!err)
815 priv->rng_registered = true;
816
817 return err;
9c4a7965
KP		 818}
819
820static void talitos_unregister_rng(struct device *dev)
821{
822 struct talitos_private *priv = dev_get_drvdata(dev);
823
35a3bb3d
AS		 824 if (!priv->rng_registered)
825 return;
826
9c4a7965 827 hwrng_unregister(&priv->rng);
35a3bb3d 828 priv->rng_registered = false;
9c4a7965
KP		 829}
830
831/*
832 * crypto alg
833 */
834#define TALITOS_CRA_PRIORITY 3000
7405c8d7
LC		 835/*
836 * Defines a priority for doing AEAD with descriptors type
837 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
838 */
839#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
b8fbdc2b 840#ifdef CONFIG_CRYPTO_DEV_TALITOS_SEC2
03d2c511 841#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
b8fbdc2b
CL		 842#else
843#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
844#endif
3952f17e 845#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
70bcaca7 846
9c4a7965
KP		 847struct talitos_ctx {
848 struct device *dev;
5228f0f7 849 int ch;
9c4a7965
KP		 850 __be32 desc_hdr_template;
851 u8 key[TALITOS_MAX_KEY_SIZE];
70bcaca7 852 u8 iv[TALITOS_MAX_IV_LENGTH];
2e13ce08 853 dma_addr_t dma_key;
9c4a7965
KP		 854 unsigned int keylen;
855 unsigned int enckeylen;
856 unsigned int authkeylen;
9c4a7965
KP		 857};
858
497f2e6b
LN		 859#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
860#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
861
862struct talitos_ahash_req_ctx {
60f208d7 863 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
497f2e6b 864 unsigned int hw_context_size;
3c0dd190
LC		 865 u8 buf[2][HASH_MAX_BLOCK_SIZE];
866 int buf_idx;
60f208d7 867 unsigned int swinit;
497f2e6b
LN		 868 unsigned int first;
869 unsigned int last;
870 unsigned int to_hash_later;
42e8b0d7 871 unsigned int nbuf;
497f2e6b
LN		 872 struct scatterlist bufsl[2];
873 struct scatterlist *psrc;
874};
875
3639ca84
HG		 876struct talitos_export_state {
877 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
878 u8 buf[HASH_MAX_BLOCK_SIZE];
879 unsigned int swinit;
880 unsigned int first;
881 unsigned int last;
882 unsigned int to_hash_later;
883 unsigned int nbuf;
884};
885
56af8cd4
LN		 886static int aead_setkey(struct crypto_aead *authenc,
887 const u8 *key, unsigned int keylen)
9c4a7965
KP		 888{
889 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
2e13ce08 890 struct device *dev = ctx->dev;
c306a98d 891 struct crypto_authenc_keys keys;
9c4a7965 892
c306a98d 893 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
9c4a7965
KP		 894 goto badkey;
895
c306a98d 896 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
9c4a7965
KP		 897 goto badkey;
898
2e13ce08
LC		 899 if (ctx->keylen)
900 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
901
c306a98d
MK		 902 memcpy(ctx->key, keys.authkey, keys.authkeylen);
903 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
9c4a7965 904
c306a98d
MK		 905 ctx->keylen = keys.authkeylen + keys.enckeylen;
906 ctx->enckeylen = keys.enckeylen;
907 ctx->authkeylen = keys.authkeylen;
2e13ce08
LC		 908 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
909 DMA_TO_DEVICE);
9c4a7965 910
8f0691fc 911 memzero_explicit(&keys, sizeof(keys));
9c4a7965
KP		 912 return 0;
913
914badkey:
915 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
8f0691fc 916 memzero_explicit(&keys, sizeof(keys));
9c4a7965
KP		 917 return -EINVAL;
918}
919
ef7c5c85
HX		 920static int aead_des3_setkey(struct crypto_aead *authenc,
921 const u8 *key, unsigned int keylen)
922{
923 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
924 struct device *dev = ctx->dev;
925 struct crypto_authenc_keys keys;
926 u32 flags;
927 int err;
928
929 err = crypto_authenc_extractkeys(&keys, key, keylen);
930 if (unlikely(err))
931 goto badkey;
932
933 err = -EINVAL;
934 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
935 goto badkey;
936
937 if (keys.enckeylen != DES3_EDE_KEY_SIZE)
938 goto badkey;
939
940 flags = crypto_aead_get_flags(authenc);
941 err = __des3_verify_key(&flags, keys.enckey);
942 if (unlikely(err)) {
943 crypto_aead_set_flags(authenc, flags);
944 goto out;
945 }
946
947 if (ctx->keylen)
948 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
949
950 memcpy(ctx->key, keys.authkey, keys.authkeylen);
951 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
952
953 ctx->keylen = keys.authkeylen + keys.enckeylen;
954 ctx->enckeylen = keys.enckeylen;
955 ctx->authkeylen = keys.authkeylen;
956 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
957 DMA_TO_DEVICE);
958
959out:
960 memzero_explicit(&keys, sizeof(keys));
961 return err;
962
963badkey:
964 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
965 goto out;
966}
967
9c4a7965 968/*
56af8cd4 969 * talitos_edesc - s/w-extended descriptor
9c4a7965
KP		 970 * @src_nents: number of segments in input scatterlist
971 * @dst_nents: number of segments in output scatterlist
aeb4c132 972 * @icv_ool: whether ICV is out-of-line
79fd31d3 973 * @iv_dma: dma address of iv for checking continuity and link table
9c4a7965 974 * @dma_len: length of dma mapped link_tbl space
6f65f6ac 975 * @dma_link_tbl: bus physical address of link_tbl/buf
9c4a7965 976 * @desc: h/w descriptor
6f65f6ac
LC		 977 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
978 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
9c4a7965
KP		 979 *
980 * if decrypting (with authcheck), or either one of src_nents or dst_nents
981 * is greater than 1, an integrity check value is concatenated to the end
982 * of link_tbl data
983 */
56af8cd4 984struct talitos_edesc {
9c4a7965
KP		 985 int src_nents;
986 int dst_nents;
aeb4c132 987 bool icv_ool;
79fd31d3 988 dma_addr_t iv_dma;
9c4a7965
KP		 989 int dma_len;
990 dma_addr_t dma_link_tbl;
991 struct talitos_desc desc;
6f65f6ac
LC		 992 union {
993 struct talitos_ptr link_tbl[0];
994 u8 buf[0];
995 };
9c4a7965
KP		 996};
997
4de9d0b5
LN		 998static void talitos_sg_unmap(struct device *dev,
999 struct talitos_edesc *edesc,
1000 struct scatterlist *src,
6a1e8d14
LC		 1001 struct scatterlist *dst,
1002 unsigned int len, unsigned int offset)
4de9d0b5 1003{
6a1e8d14
LC		 1004 struct talitos_private *priv = dev_get_drvdata(dev);
1005 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 1006 unsigned int src_nents = edesc->src_nents ? : 1;
1007 unsigned int dst_nents = edesc->dst_nents ? : 1;
1008
6a1e8d14
LC		 1009 if (is_sec1 && dst && dst_nents > 1) {
1010 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
1011 len, DMA_FROM_DEVICE);
1012 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
1013 offset);
1014 }
4de9d0b5 1015 if (src != dst) {
6a1e8d14
LC		 1016 if (src_nents == 1 || !is_sec1)
1017 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
4de9d0b5 1018
6a1e8d14 1019 if (dst && (dst_nents == 1 || !is_sec1))
b8a011d4 1020 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
6a1e8d14 1021 } else if (src_nents == 1 || !is_sec1) {
b8a011d4 1022 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
246a87cd
LC		 1023 }
1024}
1025
9c4a7965 1026static void ipsec_esp_unmap(struct device *dev,
56af8cd4 1027 struct talitos_edesc *edesc,
9c4a7965
KP		 1028 struct aead_request *areq)
1029{
549bd8bc
LC		 1030 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1031 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1032 unsigned int ivsize = crypto_aead_ivsize(aead);
9a655608
LC		 1033 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
1034 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
549bd8bc 1035
9a655608 1036 if (is_ipsec_esp)
549bd8bc
LC		 1037 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
1038 DMA_FROM_DEVICE);
9a655608 1039 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
9c4a7965 1040
6a1e8d14
LC		 1041 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
1042 areq->assoclen);
9c4a7965
KP		 1043
1044 if (edesc->dma_len)
1045 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1046 DMA_BIDIRECTIONAL);
549bd8bc 1047
9a655608 1048 if (!is_ipsec_esp) {
549bd8bc
LC		 1049 unsigned int dst_nents = edesc->dst_nents ? : 1;
1050
1051 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1052 areq->assoclen + areq->cryptlen - ivsize);
1053 }
9c4a7965
KP		 1054}
1055
1056/*
1057 * ipsec_esp descriptor callbacks
1058 */
1059static void ipsec_esp_encrypt_done(struct device *dev,
1060 struct talitos_desc *desc, void *context,
1061 int err)
1062{
549bd8bc
LC		 1063 struct talitos_private *priv = dev_get_drvdata(dev);
1064 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 1065 struct aead_request *areq = context;
9c4a7965 1066 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 1067 unsigned int authsize = crypto_aead_authsize(authenc);
2e13ce08 1068 unsigned int ivsize = crypto_aead_ivsize(authenc);
19bbbc63 1069 struct talitos_edesc *edesc;
9c4a7965
KP		 1070 struct scatterlist *sg;
1071 void *icvdata;
1072
19bbbc63
KP		 1073 edesc = container_of(desc, struct talitos_edesc, desc);
1074
9c4a7965
KP		 1075 ipsec_esp_unmap(dev, edesc, areq);
1076
1077 /* copy the generated ICV to dst */
aeb4c132 1078 if (edesc->icv_ool) {
549bd8bc
LC		 1079 if (is_sec1)
1080 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
1081 else
1082 icvdata = &edesc->link_tbl[edesc->src_nents +
1083 edesc->dst_nents + 2];
9c4a7965 1084 sg = sg_last(areq->dst, edesc->dst_nents);
aeb4c132
HX		 1085 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1086 icvdata, authsize);
9c4a7965
KP		 1087 }
1088
2e13ce08
LC		 1089 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1090
9c4a7965
KP		 1091 kfree(edesc);
1092
1093 aead_request_complete(areq, err);
1094}
1095
fe5720e2 1096static void ipsec_esp_decrypt_swauth_done(struct device *dev,
e938e465
KP		 1097 struct talitos_desc *desc,
1098 void *context, int err)
9c4a7965
KP		 1099{
1100 struct aead_request *req = context;
9c4a7965 1101 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1102 unsigned int authsize = crypto_aead_authsize(authenc);
19bbbc63 1103 struct talitos_edesc *edesc;
9c4a7965 1104 struct scatterlist *sg;
aeb4c132 1105 char *oicv, *icv;
549bd8bc
LC		 1106 struct talitos_private *priv = dev_get_drvdata(dev);
1107 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 1108
19bbbc63
KP		 1109 edesc = container_of(desc, struct talitos_edesc, desc);
1110
9c4a7965
KP		 1111 ipsec_esp_unmap(dev, edesc, req);
1112
1113 if (!err) {
1114 /* auth check */
9c4a7965 1115 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
aeb4c132
HX		 1116 icv = (char *)sg_virt(sg) + sg->length - authsize;
1117
1118 if (edesc->dma_len) {
549bd8bc
LC		 1119 if (is_sec1)
1120 oicv = (char *)&edesc->dma_link_tbl +
1121 req->assoclen + req->cryptlen;
1122 else
1123 oicv = (char *)
1124 &edesc->link_tbl[edesc->src_nents +
aeb4c132
HX		 1125 edesc->dst_nents + 2];
1126 if (edesc->icv_ool)
1127 icv = oicv + authsize;
1128 } else
1129 oicv = (char *)&edesc->link_tbl[0];
1130
79960943 1131 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
9c4a7965
KP		 1132 }
1133
1134 kfree(edesc);
1135
1136 aead_request_complete(req, err);
1137}
1138
fe5720e2 1139static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
e938e465
KP		 1140 struct talitos_desc *desc,
1141 void *context, int err)
fe5720e2
KP		 1142{
1143 struct aead_request *req = context;
19bbbc63
KP		 1144 struct talitos_edesc *edesc;
1145
1146 edesc = container_of(desc, struct talitos_edesc, desc);
fe5720e2
KP		 1147
1148 ipsec_esp_unmap(dev, edesc, req);
1149
1150 /* check ICV auth status */
e938e465
KP		 1151 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1152 DESC_HDR_LO_ICCR1_PASS))
1153 err = -EBADMSG;
fe5720e2
KP		 1154
1155 kfree(edesc);
1156
1157 aead_request_complete(req, err);
1158}
1159
9c4a7965
KP		 1160/*
1161 * convert scatterlist to SEC h/w link table format
1162 * stop at cryptlen bytes
1163 */
aeb4c132
HX		 1164static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1165 unsigned int offset, int cryptlen,
1166 struct talitos_ptr *link_tbl_ptr)
9c4a7965 1167{
70bcaca7 1168 int n_sg = sg_count;
aeb4c132 1169 int count = 0;
70bcaca7 1170
aeb4c132
HX		 1171 while (cryptlen && sg && n_sg--) {
1172 unsigned int len = sg_dma_len(sg);
9c4a7965 1173
aeb4c132
HX		 1174 if (offset >= len) {
1175 offset -= len;
1176 goto next;
1177 }
1178
1179 len -= offset;
1180
1181 if (len > cryptlen)
1182 len = cryptlen;
1183
1184 to_talitos_ptr(link_tbl_ptr + count,
da9de146 1185 sg_dma_address(sg) + offset, len, 0);
b096b544 1186 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
aeb4c132
HX		 1187 count++;
1188 cryptlen -= len;
1189 offset = 0;
1190
1191next:
1192 sg = sg_next(sg);
70bcaca7 1193 }
9c4a7965
KP		 1194
1195 /* tag end of link table */
aeb4c132 1196 if (count > 0)
b096b544
LC		 1197 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1198 DESC_PTR_LNKTBL_RETURN, 0);
70bcaca7 1199
aeb4c132
HX		 1200 return count;
1201}
1202
2b122730
LC		 1203static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1204 unsigned int len, struct talitos_edesc *edesc,
1205 struct talitos_ptr *ptr, int sg_count,
1206 unsigned int offset, int tbl_off, int elen)
246a87cd 1207{
246a87cd
LC		 1208 struct talitos_private *priv = dev_get_drvdata(dev);
1209 bool is_sec1 = has_ftr_sec1(priv);
1210
87a81dce
LC		 1211 if (!src) {
1212 to_talitos_ptr(ptr, 0, 0, is_sec1);
1213 return 1;
1214 }
2b122730 1215 to_talitos_ptr_ext_set(ptr, elen, is_sec1);
6a1e8d14 1216 if (sg_count == 1) {
da9de146 1217 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
6a1e8d14 1218 return sg_count;
246a87cd 1219 }
246a87cd 1220 if (is_sec1) {
da9de146 1221 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
6a1e8d14 1222 return sg_count;
246a87cd 1223 }
2b122730 1224 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen,
6a1e8d14
LC		 1225 &edesc->link_tbl[tbl_off]);
1226 if (sg_count == 1) {
1227 /* Only one segment now, so no link tbl needed*/
1228 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1229 return sg_count;
1230 }
1231 to_talitos_ptr(ptr, edesc->dma_link_tbl +
da9de146 1232 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
6a1e8d14
LC		 1233 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1234
1235 return sg_count;
246a87cd
LC		 1236}
1237
2b122730
LC		 1238static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1239 unsigned int len, struct talitos_edesc *edesc,
1240 struct talitos_ptr *ptr, int sg_count,
1241 unsigned int offset, int tbl_off)
1242{
1243 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1244 tbl_off, 0);
1245}
1246
9c4a7965
KP		 1247/*
1248 * fill in and submit ipsec_esp descriptor
1249 */
56af8cd4 1250static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
aeb4c132
HX		 1251 void (*callback)(struct device *dev,
1252 struct talitos_desc *desc,
1253 void *context, int error))
9c4a7965
KP		 1254{
1255 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
aeb4c132 1256 unsigned int authsize = crypto_aead_authsize(aead);
9c4a7965
KP		 1257 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1258 struct device *dev = ctx->dev;
1259 struct talitos_desc *desc = &edesc->desc;
1260 unsigned int cryptlen = areq->cryptlen;
e41256f1 1261 unsigned int ivsize = crypto_aead_ivsize(aead);
aeb4c132 1262 int tbl_off = 0;
fa86a267 1263 int sg_count, ret;
2b122730 1264 int elen = 0;
549bd8bc
LC		 1265 bool sync_needed = false;
1266 struct talitos_private *priv = dev_get_drvdata(dev);
1267 bool is_sec1 = has_ftr_sec1(priv);
9a655608
LC		 1268 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1269 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1270 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
9c4a7965
KP		 1271
1272 /* hmac key */
2e13ce08 1273 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
79fd31d3 1274
549bd8bc
LC		 1275 sg_count = edesc->src_nents ?: 1;
1276 if (is_sec1 && sg_count > 1)
1277 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1278 areq->assoclen + cryptlen);
1279 else
1280 sg_count = dma_map_sg(dev, areq->src, sg_count,
1281 (areq->src == areq->dst) ?
1282 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
79fd31d3 1283
549bd8bc
LC		 1284 /* hmac data */
1285 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1286 &desc->ptr[1], sg_count, 0, tbl_off);
340ff60a 1287
549bd8bc 1288 if (ret > 1) {
340ff60a 1289 tbl_off += ret;
549bd8bc 1290 sync_needed = true;
79fd31d3
HG		 1291 }
1292
9c4a7965 1293 /* cipher iv */
9a655608 1294 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
9c4a7965
KP		 1295
1296 /* cipher key */
2e13ce08
LC		 1297 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1298 ctx->enckeylen, is_sec1);
9c4a7965
KP		 1299
1300 /*
1301 * cipher in
1302 * map and adjust cipher len to aead request cryptlen.
1303 * extent is bytes of HMAC postpended to ciphertext,
1304 * typically 12 for ipsec
1305 */
2b122730
LC		 1306 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1307 elen = authsize;
9c4a7965 1308
2b122730
LC		 1309 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1310 sg_count, areq->assoclen, tbl_off, elen);
549bd8bc 1311
ec8c7d14
LC		 1312 if (ret > 1) {
1313 tbl_off += ret;
549bd8bc
LC		 1314 sync_needed = true;
1315 }
9c4a7965 1316
549bd8bc
LC		 1317 /* cipher out */
1318 if (areq->src != areq->dst) {
1319 sg_count = edesc->dst_nents ? : 1;
1320 if (!is_sec1 || sg_count == 1)
1321 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1322 }
9c4a7965 1323
e04a61be
LC		 1324 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1325 sg_count, areq->assoclen, tbl_off);
aeb4c132 1326
9a655608 1327 if (is_ipsec_esp)
549bd8bc 1328 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
aeb4c132 1329
e04a61be
LC		 1330 /* ICV data */
1331 if (ret > 1) {
1332 tbl_off += ret;
aeb4c132 1333 edesc->icv_ool = true;
549bd8bc
LC		 1334 sync_needed = true;
1335
9a655608 1336 if (is_ipsec_esp) {
549bd8bc
LC		 1337 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1338 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1339 sizeof(struct talitos_ptr) + authsize;
1340
1341 /* Add an entry to the link table for ICV data */
e04a61be 1342 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
549bd8bc
LC		 1343 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1344 is_sec1);
549bd8bc
LC		 1345
1346 /* icv data follows link tables */
1347 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
da9de146 1348 authsize, is_sec1);
e04a61be
LC		 1349 } else {
1350 dma_addr_t addr = edesc->dma_link_tbl;
1351
1352 if (is_sec1)
1353 addr += areq->assoclen + cryptlen;
1354 else
1355 addr += sizeof(struct talitos_ptr) * tbl_off;
1356
da9de146 1357 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
e04a61be 1358 }
9a655608 1359 } else if (!is_ipsec_esp) {
e04a61be
LC		 1360 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1361 &desc->ptr[6], sg_count, areq->assoclen +
1362 cryptlen,
1363 tbl_off);
1364 if (ret > 1) {
1365 tbl_off += ret;
1366 edesc->icv_ool = true;
1367 sync_needed = true;
1368 } else {
1369 edesc->icv_ool = false;
549bd8bc 1370 }
340ff60a 1371 } else {
549bd8bc
LC		 1372 edesc->icv_ool = false;
1373 }
1374
9c4a7965 1375 /* iv out */
9a655608 1376 if (is_ipsec_esp)
549bd8bc
LC		 1377 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1378 DMA_FROM_DEVICE);
1379
1380 if (sync_needed)
1381 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1382 edesc->dma_len,
1383 DMA_BIDIRECTIONAL);
9c4a7965 1384
5228f0f7 1385 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
fa86a267
KP		 1386 if (ret != -EINPROGRESS) {
1387 ipsec_esp_unmap(dev, edesc, areq);
1388 kfree(edesc);
1389 }
1390 return ret;
9c4a7965
KP		 1391}
1392
9c4a7965 1393/*
56af8cd4 1394 * allocate and map the extended descriptor
9c4a7965 1395 */
4de9d0b5
LN		 1396static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1397 struct scatterlist *src,
1398 struct scatterlist *dst,
79fd31d3
HG		 1399 u8 *iv,
1400 unsigned int assoclen,
4de9d0b5
LN		 1401 unsigned int cryptlen,
1402 unsigned int authsize,
79fd31d3 1403 unsigned int ivsize,
4de9d0b5 1404 int icv_stashing,
62293a37
HG		 1405 u32 cryptoflags,
1406 bool encrypt)
9c4a7965 1407{
56af8cd4 1408 struct talitos_edesc *edesc;
6a1e8d14 1409 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
79fd31d3 1410 dma_addr_t iv_dma = 0;
4de9d0b5 1411 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
586725f8 1412 GFP_ATOMIC;
6f65f6ac
LC		 1413 struct talitos_private *priv = dev_get_drvdata(dev);
1414 bool is_sec1 = has_ftr_sec1(priv);
1415 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
9c4a7965 1416
6f65f6ac 1417 if (cryptlen + authsize > max_len) {
4de9d0b5 1418 dev_err(dev, "length exceeds h/w max limit\n");
9c4a7965
KP		 1419 return ERR_PTR(-EINVAL);
1420 }
1421
62293a37 1422 if (!dst || dst == src) {
6a1e8d14
LC		 1423 src_len = assoclen + cryptlen + authsize;
1424 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1425 if (src_nents < 0) {
1426 dev_err(dev, "Invalid number of src SG.\n");
c56c2e17 1427 return ERR_PTR(-EINVAL);
8e409fe1 1428 }
62293a37
HG		 1429 src_nents = (src_nents == 1) ? 0 : src_nents;
1430 dst_nents = dst ? src_nents : 0;
6a1e8d14 1431 dst_len = 0;
62293a37 1432 } else { /* dst && dst != src*/
6a1e8d14
LC		 1433 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1434 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1435 if (src_nents < 0) {
1436 dev_err(dev, "Invalid number of src SG.\n");
c56c2e17 1437 return ERR_PTR(-EINVAL);
8e409fe1 1438 }
62293a37 1439 src_nents = (src_nents == 1) ? 0 : src_nents;
6a1e8d14
LC		 1440 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1441 dst_nents = sg_nents_for_len(dst, dst_len);
8e409fe1
LC		 1442 if (dst_nents < 0) {
1443 dev_err(dev, "Invalid number of dst SG.\n");
c56c2e17 1444 return ERR_PTR(-EINVAL);
8e409fe1 1445 }
62293a37 1446 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
9c4a7965
KP		 1447 }
1448
1449 /*
1450 * allocate space for base edesc plus the link tables,
aeb4c132
HX		 1451 * allowing for two separate entries for AD and generated ICV (+ 2),
1452 * and space for two sets of ICVs (stashed and generated)
9c4a7965 1453 */
56af8cd4 1454 alloc_len = sizeof(struct talitos_edesc);
aeb4c132 1455 if (src_nents || dst_nents) {
6f65f6ac 1456 if (is_sec1)
6a1e8d14
LC		 1457 dma_len = (src_nents ? src_len : 0) +
1458 (dst_nents ? dst_len : 0);
6f65f6ac 1459 else
aeb4c132
HX		 1460 dma_len = (src_nents + dst_nents + 2) *
1461 sizeof(struct talitos_ptr) + authsize * 2;
9c4a7965
KP		 1462 alloc_len += dma_len;
1463 } else {
1464 dma_len = 0;
4de9d0b5 1465 alloc_len += icv_stashing ? authsize : 0;
9c4a7965
KP		 1466 }
1467
37b5e889
LC		 1468 /* if its a ahash, add space for a second desc next to the first one */
1469 if (is_sec1 && !dst)
1470 alloc_len += sizeof(struct talitos_desc);
1bea445b 1471 alloc_len += ivsize;
37b5e889 1472
586725f8 1473 edesc = kmalloc(alloc_len, GFP_DMA | flags);
c56c2e17
CL		 1474 if (!edesc)
1475 return ERR_PTR(-ENOMEM);
1bea445b
CL		 1476 if (ivsize) {
1477 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
c56c2e17 1478 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1bea445b 1479 }
e4a647c4 1480 memset(&edesc->desc, 0, sizeof(edesc->desc));
9c4a7965
KP		 1481
1482 edesc->src_nents = src_nents;
1483 edesc->dst_nents = dst_nents;
79fd31d3 1484 edesc->iv_dma = iv_dma;
9c4a7965 1485 edesc->dma_len = dma_len;
37b5e889
LC		 1486 if (dma_len) {
1487 void *addr = &edesc->link_tbl[0];
1488
1489 if (is_sec1 && !dst)
1490 addr += sizeof(struct talitos_desc);
1491 edesc->dma_link_tbl = dma_map_single(dev, addr,
497f2e6b
LN		 1492 edesc->dma_len,
1493 DMA_BIDIRECTIONAL);
37b5e889 1494 }
9c4a7965
KP		 1495 return edesc;
1496}
1497
79fd31d3 1498static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
62293a37 1499 int icv_stashing, bool encrypt)
4de9d0b5
LN		 1500{
1501 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 1502 unsigned int authsize = crypto_aead_authsize(authenc);
4de9d0b5 1503 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
79fd31d3 1504 unsigned int ivsize = crypto_aead_ivsize(authenc);
4de9d0b5 1505
aeb4c132 1506 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1507 iv, areq->assoclen, areq->cryptlen,
aeb4c132 1508 authsize, ivsize, icv_stashing,
62293a37 1509 areq->base.flags, encrypt);
4de9d0b5
LN		 1510}
1511
56af8cd4 1512static int aead_encrypt(struct aead_request *req)
9c4a7965
KP		 1513{
1514 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1515 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
56af8cd4 1516 struct talitos_edesc *edesc;
9c4a7965
KP		 1517
1518 /* allocate extended descriptor */
62293a37 1519 edesc = aead_edesc_alloc(req, req->iv, 0, true);
9c4a7965
KP		 1520 if (IS_ERR(edesc))
1521 return PTR_ERR(edesc);
1522
1523 /* set encrypt */
70bcaca7 1524 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965 1525
aeb4c132 1526 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
9c4a7965
KP		 1527}
1528
56af8cd4 1529static int aead_decrypt(struct aead_request *req)
9c4a7965
KP		 1530{
1531 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1532 unsigned int authsize = crypto_aead_authsize(authenc);
9c4a7965 1533 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
fe5720e2 1534 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
56af8cd4 1535 struct talitos_edesc *edesc;
9c4a7965
KP		 1536 struct scatterlist *sg;
1537 void *icvdata;
1538
1539 req->cryptlen -= authsize;
1540
1541 /* allocate extended descriptor */
62293a37 1542 edesc = aead_edesc_alloc(req, req->iv, 1, false);
9c4a7965
KP		 1543 if (IS_ERR(edesc))
1544 return PTR_ERR(edesc);
1545
fe5720e2 1546 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
e938e465
KP		 1547 ((!edesc->src_nents && !edesc->dst_nents) ||
1548 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
9c4a7965 1549
fe5720e2 1550 /* decrypt and check the ICV */
e938e465
KP		 1551 edesc->desc.hdr = ctx->desc_hdr_template |
1552 DESC_HDR_DIR_INBOUND |
fe5720e2 1553 DESC_HDR_MODE1_MDEU_CICV;
9c4a7965 1554
fe5720e2 1555 /* reset integrity check result bits */
9c4a7965 1556
aeb4c132 1557 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
e938e465 1558 }
fe5720e2 1559
e938e465
KP		 1560 /* Have to check the ICV with software */
1561 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
fe5720e2 1562
e938e465
KP		 1563 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1564 if (edesc->dma_len)
aeb4c132
HX		 1565 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1566 edesc->dst_nents + 2];
e938e465
KP		 1567 else
1568 icvdata = &edesc->link_tbl[0];
fe5720e2 1569
e938e465 1570 sg = sg_last(req->src, edesc->src_nents ? : 1);
fe5720e2 1571
aeb4c132 1572 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
9c4a7965 1573
aeb4c132 1574 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
9c4a7965
KP		 1575}
1576
4de9d0b5
LN		 1577static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1578 const u8 *key, unsigned int keylen)
1579{
1580 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
2e13ce08 1581 struct device *dev = ctx->dev;
4de9d0b5 1582
ef7c5c85
HX		 1583 if (ctx->keylen)
1584 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1585
1586 memcpy(&ctx->key, key, keylen);
1587 ctx->keylen = keylen;
1588
1589 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1590
1591 return 0;
1592}
1593
1594static int ablkcipher_des_setkey(struct crypto_ablkcipher *cipher,
1595 const u8 *key, unsigned int keylen)
1596{
1597 u32 tmp[DES_EXPKEY_WORDS];
03d2c511 1598
f384cdc4 1599 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
231baecd 1600 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) &&
f384cdc4
LC		 1601 !des_ekey(tmp, key)) {
1602 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1603 return -EINVAL;
1604 }
1605
ef7c5c85
HX		 1606 return ablkcipher_setkey(cipher, key, keylen);
1607}
2e13ce08 1608
ef7c5c85
HX		 1609static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher,
1610 const u8 *key, unsigned int keylen)
1611{
1612 u32 flags;
1613 int err;
4de9d0b5 1614
ef7c5c85
HX		 1615 flags = crypto_ablkcipher_get_flags(cipher);
1616 err = __des3_verify_key(&flags, key);
1617 if (unlikely(err)) {
1618 crypto_ablkcipher_set_flags(cipher, flags);
1619 return err;
1620 }
2e13ce08 1621
ef7c5c85 1622 return ablkcipher_setkey(cipher, key, keylen);
4de9d0b5
LN		 1623}
1624
1ba34e71
CL		 1625static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher,
1626 const u8 *key, unsigned int keylen)
1627{
1628 if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
1629 keylen == AES_KEYSIZE_256)
1630 return ablkcipher_setkey(cipher, key, keylen);
1631
1632 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1633
1634 return -EINVAL;
1635}
1636
4de9d0b5
LN		 1637static void common_nonsnoop_unmap(struct device *dev,
1638 struct talitos_edesc *edesc,
1639 struct ablkcipher_request *areq)
1640{
1641 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
032d197e 1642
6a1e8d14 1643 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
4de9d0b5
LN		 1644 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1645
4de9d0b5
LN		 1646 if (edesc->dma_len)
1647 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1648 DMA_BIDIRECTIONAL);
1649}
1650
1651static void ablkcipher_done(struct device *dev,
1652 struct talitos_desc *desc, void *context,
1653 int err)
1654{
1655 struct ablkcipher_request *areq = context;
3e03e792
CL		 1656 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1657 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1658 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
19bbbc63
KP		 1659 struct talitos_edesc *edesc;
1660
1661 edesc = container_of(desc, struct talitos_edesc, desc);
4de9d0b5
LN		 1662
1663 common_nonsnoop_unmap(dev, edesc, areq);
3e03e792 1664 memcpy(areq->info, ctx->iv, ivsize);
4de9d0b5
LN		 1665
1666 kfree(edesc);
1667
1668 areq->base.complete(&areq->base, err);
1669}
1670
1671static int common_nonsnoop(struct talitos_edesc *edesc,
1672 struct ablkcipher_request *areq,
4de9d0b5
LN		 1673 void (*callback) (struct device *dev,
1674 struct talitos_desc *desc,
1675 void *context, int error))
1676{
1677 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1678 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1679 struct device *dev = ctx->dev;
1680 struct talitos_desc *desc = &edesc->desc;
1681 unsigned int cryptlen = areq->nbytes;
79fd31d3 1682 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1683 int sg_count, ret;
6a1e8d14 1684 bool sync_needed = false;
922f9dc8
LC		 1685 struct talitos_private *priv = dev_get_drvdata(dev);
1686 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 1687
1688 /* first DWORD empty */
4de9d0b5
LN		 1689
1690 /* cipher iv */
da9de146 1691 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
4de9d0b5
LN		 1692
1693 /* cipher key */
2e13ce08 1694 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
4de9d0b5 1695
6a1e8d14
LC		 1696 sg_count = edesc->src_nents ?: 1;
1697 if (is_sec1 && sg_count > 1)
1698 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1699 cryptlen);
1700 else
1701 sg_count = dma_map_sg(dev, areq->src, sg_count,
1702 (areq->src == areq->dst) ?
1703 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
4de9d0b5
LN		 1704 /*
1705 * cipher in
1706 */
6a1e8d14
LC		 1707 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1708 &desc->ptr[3], sg_count, 0, 0);
1709 if (sg_count > 1)
1710 sync_needed = true;
4de9d0b5
LN		 1711
1712 /* cipher out */
6a1e8d14
LC		 1713 if (areq->src != areq->dst) {
1714 sg_count = edesc->dst_nents ? : 1;
1715 if (!is_sec1 || sg_count == 1)
1716 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1717 }
1718
1719 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1720 sg_count, 0, (edesc->src_nents + 1));
1721 if (ret > 1)
1722 sync_needed = true;
4de9d0b5
LN		 1723
1724 /* iv out */
a2b35aa8 1725 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
4de9d0b5
LN		 1726 DMA_FROM_DEVICE);
1727
1728 /* last DWORD empty */
4de9d0b5 1729
6a1e8d14
LC		 1730 if (sync_needed)
1731 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1732 edesc->dma_len, DMA_BIDIRECTIONAL);
1733
5228f0f7 1734 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
4de9d0b5
LN		 1735 if (ret != -EINPROGRESS) {
1736 common_nonsnoop_unmap(dev, edesc, areq);
1737 kfree(edesc);
1738 }
1739 return ret;
1740}
1741
e938e465 1742static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
62293a37 1743 areq, bool encrypt)
4de9d0b5
LN		 1744{
1745 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1746 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
79fd31d3 1747 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1748
aeb4c132 1749 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1750 areq->info, 0, areq->nbytes, 0, ivsize, 0,
62293a37 1751 areq->base.flags, encrypt);
4de9d0b5
LN		 1752}
1753
1754static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1755{
1756 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1757 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1758 struct talitos_edesc *edesc;
ee483d32
CL		 1759 unsigned int blocksize =
1760 crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
1761
1762 if (!areq->nbytes)
1763 return 0;
1764
1765 if (areq->nbytes % blocksize)
1766 return -EINVAL;
4de9d0b5
LN		 1767
1768 /* allocate extended descriptor */
62293a37 1769 edesc = ablkcipher_edesc_alloc(areq, true);
4de9d0b5
LN		 1770 if (IS_ERR(edesc))
1771 return PTR_ERR(edesc);
1772
1773 /* set encrypt */
1774 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1775
febec542 1776 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1777}
1778
1779static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1780{
1781 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1782 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1783 struct talitos_edesc *edesc;
ee483d32
CL		 1784 unsigned int blocksize =
1785 crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
1786
1787 if (!areq->nbytes)
1788 return 0;
1789
1790 if (areq->nbytes % blocksize)
1791 return -EINVAL;
4de9d0b5
LN		 1792
1793 /* allocate extended descriptor */
62293a37 1794 edesc = ablkcipher_edesc_alloc(areq, false);
4de9d0b5
LN		 1795 if (IS_ERR(edesc))
1796 return PTR_ERR(edesc);
1797
1798 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1799
febec542 1800 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1801}
1802
497f2e6b
LN		 1803static void common_nonsnoop_hash_unmap(struct device *dev,
1804 struct talitos_edesc *edesc,
1805 struct ahash_request *areq)
1806{
1807 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
ad4cd51f
LC		 1808 struct talitos_private *priv = dev_get_drvdata(dev);
1809 bool is_sec1 = has_ftr_sec1(priv);
1810 struct talitos_desc *desc = &edesc->desc;
1811 struct talitos_desc *desc2 = desc + 1;
1812
1813 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1814 if (desc->next_desc &&
1815 desc->ptr[5].ptr != desc2->ptr[5].ptr)
1816 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
497f2e6b 1817
6a1e8d14 1818 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
032d197e 1819
ad4cd51f
LC		 1820 /* When using hashctx-in, must unmap it. */
1821 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1822 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1823 DMA_TO_DEVICE);
1824 else if (desc->next_desc)
1825 unmap_single_talitos_ptr(dev, &desc2->ptr[1],
1826 DMA_TO_DEVICE);
1827
1828 if (is_sec1 && req_ctx->nbuf)
1829 unmap_single_talitos_ptr(dev, &desc->ptr[3],
1830 DMA_TO_DEVICE);
1831
497f2e6b
LN		 1832 if (edesc->dma_len)
1833 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1834 DMA_BIDIRECTIONAL);
1835
37b5e889
LC		 1836 if (edesc->desc.next_desc)
1837 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1838 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
497f2e6b
LN		 1839}
1840
1841static void ahash_done(struct device *dev,
1842 struct talitos_desc *desc, void *context,
1843 int err)
1844{
1845 struct ahash_request *areq = context;
1846 struct talitos_edesc *edesc =
1847 container_of(desc, struct talitos_edesc, desc);
1848 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1849
1850 if (!req_ctx->last && req_ctx->to_hash_later) {
1851 /* Position any partial block for next update/final/finup */
3c0dd190 1852 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
5e833bc4 1853 req_ctx->nbuf = req_ctx->to_hash_later;
497f2e6b
LN		 1854 }
1855 common_nonsnoop_hash_unmap(dev, edesc, areq);
1856
1857 kfree(edesc);
1858
1859 areq->base.complete(&areq->base, err);
1860}
1861
2d02905e
LC		 1862/*
1863 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1864 * ourself and submit a padded block
1865 */
5b2cf268 1866static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
2d02905e
LC		 1867 struct talitos_edesc *edesc,
1868 struct talitos_ptr *ptr)
1869{
1870 static u8 padded_hash[64] = {
1871 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1872 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1873 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1874 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1875 };
1876
1877 pr_err_once("Bug in SEC1, padding ourself\n");
1878 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1879 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1880 (char *)padded_hash, DMA_TO_DEVICE);
1881}
1882
497f2e6b
LN		 1883static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1884 struct ahash_request *areq, unsigned int length,
37b5e889 1885 unsigned int offset,
497f2e6b
LN		 1886 void (*callback) (struct device *dev,
1887 struct talitos_desc *desc,
1888 void *context, int error))
1889{
1890 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1891 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1892 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1893 struct device *dev = ctx->dev;
1894 struct talitos_desc *desc = &edesc->desc;
032d197e 1895 int ret;
6a1e8d14 1896 bool sync_needed = false;
922f9dc8
LC		 1897 struct talitos_private *priv = dev_get_drvdata(dev);
1898 bool is_sec1 = has_ftr_sec1(priv);
6a1e8d14 1899 int sg_count;
497f2e6b
LN		 1900
1901 /* first DWORD empty */
497f2e6b 1902
60f208d7
KP		 1903 /* hash context in */
1904 if (!req_ctx->first || req_ctx->swinit) {
6a4967c3
LC		 1905 map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1906 req_ctx->hw_context_size,
1907 req_ctx->hw_context,
1908 DMA_TO_DEVICE);
60f208d7 1909 req_ctx->swinit = 0;
497f2e6b 1910 }
afd62fa2
LC		 1911 /* Indicate next op is not the first. */
1912 req_ctx->first = 0;
497f2e6b
LN		 1913
1914 /* HMAC key */
1915 if (ctx->keylen)
2e13ce08
LC		 1916 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1917 is_sec1);
497f2e6b 1918
37b5e889
LC		 1919 if (is_sec1 && req_ctx->nbuf)
1920 length -= req_ctx->nbuf;
1921
6a1e8d14
LC		 1922 sg_count = edesc->src_nents ?: 1;
1923 if (is_sec1 && sg_count > 1)
37b5e889
LC		 1924 sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
1925 edesc->buf + sizeof(struct talitos_desc),
1926 length, req_ctx->nbuf);
1927 else if (length)
6a1e8d14
LC		 1928 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1929 DMA_TO_DEVICE);
497f2e6b
LN		 1930 /*
1931 * data in
1932 */
37b5e889 1933 if (is_sec1 && req_ctx->nbuf) {
ad4cd51f
LC		 1934 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
1935 req_ctx->buf[req_ctx->buf_idx],
1936 DMA_TO_DEVICE);
37b5e889
LC		 1937 } else {
1938 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1939 &desc->ptr[3], sg_count, offset, 0);
1940 if (sg_count > 1)
1941 sync_needed = true;
1942 }
497f2e6b
LN		 1943
1944 /* fifth DWORD empty */
497f2e6b
LN		 1945
1946 /* hash/HMAC out -or- hash context out */
1947 if (req_ctx->last)
1948 map_single_talitos_ptr(dev, &desc->ptr[5],
1949 crypto_ahash_digestsize(tfm),
a2b35aa8 1950 areq->result, DMA_FROM_DEVICE);
497f2e6b 1951 else
6a4967c3
LC		 1952 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1953 req_ctx->hw_context_size,
1954 req_ctx->hw_context,
1955 DMA_FROM_DEVICE);
497f2e6b
LN		 1956
1957 /* last DWORD empty */
497f2e6b 1958
2d02905e
LC		 1959 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1960 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1961
37b5e889
LC		 1962 if (is_sec1 && req_ctx->nbuf && length) {
1963 struct talitos_desc *desc2 = desc + 1;
1964 dma_addr_t next_desc;
1965
1966 memset(desc2, 0, sizeof(*desc2));
1967 desc2->hdr = desc->hdr;
1968 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1969 desc2->hdr1 = desc2->hdr;
1970 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1971 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1972 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1973
ad4cd51f
LC		 1974 if (desc->ptr[1].ptr)
1975 copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
1976 is_sec1);
1977 else
6a4967c3
LC		 1978 map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
1979 req_ctx->hw_context_size,
1980 req_ctx->hw_context,
1981 DMA_TO_DEVICE);
37b5e889
LC		 1982 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1983 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1984 &desc2->ptr[3], sg_count, offset, 0);
1985 if (sg_count > 1)
1986 sync_needed = true;
1987 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1988 if (req_ctx->last)
6a4967c3
LC		 1989 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1990 req_ctx->hw_context_size,
1991 req_ctx->hw_context,
1992 DMA_FROM_DEVICE);
37b5e889
LC		 1993
1994 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1995 DMA_BIDIRECTIONAL);
1996 desc->next_desc = cpu_to_be32(next_desc);
1997 }
1998
6a1e8d14
LC		 1999 if (sync_needed)
2000 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
2001 edesc->dma_len, DMA_BIDIRECTIONAL);
2002
5228f0f7 2003 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
497f2e6b
LN		 2004 if (ret != -EINPROGRESS) {
2005 common_nonsnoop_hash_unmap(dev, edesc, areq);
2006 kfree(edesc);
2007 }
2008 return ret;
2009}
2010
2011static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
2012 unsigned int nbytes)
2013{
2014 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2015 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2016 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
37b5e889
LC		 2017 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
2018 bool is_sec1 = has_ftr_sec1(priv);
2019
2020 if (is_sec1)
2021 nbytes -= req_ctx->nbuf;
497f2e6b 2022
aeb4c132 2023 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
62293a37 2024 nbytes, 0, 0, 0, areq->base.flags, false);
497f2e6b
LN		 2025}
2026
2027static int ahash_init(struct ahash_request *areq)
2028{
2029 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
6a4967c3
LC		 2030 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2031 struct device *dev = ctx->dev;
497f2e6b 2032 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
49f9783b 2033 unsigned int size;
6a4967c3 2034 dma_addr_t dma;
497f2e6b
LN		 2035
2036 /* Initialize the context */
3c0dd190 2037 req_ctx->buf_idx = 0;
5e833bc4 2038 req_ctx->nbuf = 0;
60f208d7
KP		 2039 req_ctx->first = 1; /* first indicates h/w must init its context */
2040 req_ctx->swinit = 0; /* assume h/w init of context */
49f9783b 2041 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
497f2e6b
LN		 2042 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2043 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
49f9783b 2044 req_ctx->hw_context_size = size;
497f2e6b 2045
6a4967c3
LC		 2046 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2047 DMA_TO_DEVICE);
2048 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2049
497f2e6b
LN		 2050 return 0;
2051}
2052
60f208d7
KP		 2053/*
2054 * on h/w without explicit sha224 support, we initialize h/w context
2055 * manually with sha224 constants, and tell it to run sha256.
2056 */
2057static int ahash_init_sha224_swinit(struct ahash_request *areq)
2058{
2059 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2060
a752447a
KP		 2061 req_ctx->hw_context[0] = SHA224_H0;
2062 req_ctx->hw_context[1] = SHA224_H1;
2063 req_ctx->hw_context[2] = SHA224_H2;
2064 req_ctx->hw_context[3] = SHA224_H3;
2065 req_ctx->hw_context[4] = SHA224_H4;
2066 req_ctx->hw_context[5] = SHA224_H5;
2067 req_ctx->hw_context[6] = SHA224_H6;
2068 req_ctx->hw_context[7] = SHA224_H7;
60f208d7
KP		 2069
2070 /* init 64-bit count */
2071 req_ctx->hw_context[8] = 0;
2072 req_ctx->hw_context[9] = 0;
2073
6a4967c3
LC		 2074 ahash_init(areq);
2075 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
2076
60f208d7
KP		 2077 return 0;
2078}
2079
497f2e6b
LN		 2080static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
2081{
2082 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2083 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2084 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2085 struct talitos_edesc *edesc;
2086 unsigned int blocksize =
2087 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2088 unsigned int nbytes_to_hash;
2089 unsigned int to_hash_later;
5e833bc4 2090 unsigned int nsg;
8e409fe1 2091 int nents;
37b5e889
LC		 2092 struct device *dev = ctx->dev;
2093 struct talitos_private *priv = dev_get_drvdata(dev);
2094 bool is_sec1 = has_ftr_sec1(priv);
2095 int offset = 0;
3c0dd190 2096 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
497f2e6b 2097
5e833bc4
LN		 2098 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
2099 /* Buffer up to one whole block */
8e409fe1
LC		 2100 nents = sg_nents_for_len(areq->src, nbytes);
2101 if (nents < 0) {
2102 dev_err(ctx->dev, "Invalid number of src SG.\n");
2103 return nents;
2104 }
2105 sg_copy_to_buffer(areq->src, nents,
3c0dd190 2106 ctx_buf + req_ctx->nbuf, nbytes);
5e833bc4 2107 req_ctx->nbuf += nbytes;
497f2e6b
LN		 2108 return 0;
2109 }
2110
5e833bc4
LN		 2111 /* At least (blocksize + 1) bytes are available to hash */
2112 nbytes_to_hash = nbytes + req_ctx->nbuf;
2113 to_hash_later = nbytes_to_hash & (blocksize - 1);
2114
2115 if (req_ctx->last)
2116 to_hash_later = 0;
2117 else if (to_hash_later)
2118 /* There is a partial block. Hash the full block(s) now */
2119 nbytes_to_hash -= to_hash_later;
2120 else {
2121 /* Keep one block buffered */
2122 nbytes_to_hash -= blocksize;
2123 to_hash_later = blocksize;
2124 }
2125
2126 /* Chain in any previously buffered data */
37b5e889 2127 if (!is_sec1 && req_ctx->nbuf) {
5e833bc4
LN		 2128 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2129 sg_init_table(req_ctx->bufsl, nsg);
3c0dd190 2130 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
5e833bc4 2131 if (nsg > 1)
c56f6d12 2132 sg_chain(req_ctx->bufsl, 2, areq->src);
497f2e6b 2133 req_ctx->psrc = req_ctx->bufsl;
37b5e889
LC		 2134 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2135 if (nbytes_to_hash > blocksize)
2136 offset = blocksize - req_ctx->nbuf;
2137 else
2138 offset = nbytes_to_hash - req_ctx->nbuf;
2139 nents = sg_nents_for_len(areq->src, offset);
2140 if (nents < 0) {
2141 dev_err(ctx->dev, "Invalid number of src SG.\n");
2142 return nents;
2143 }
2144 sg_copy_to_buffer(areq->src, nents,
3c0dd190 2145 ctx_buf + req_ctx->nbuf, offset);
37b5e889
LC		 2146 req_ctx->nbuf += offset;
2147 req_ctx->psrc = areq->src;
5e833bc4 2148 } else
497f2e6b 2149 req_ctx->psrc = areq->src;
5e833bc4
LN		 2150
2151 if (to_hash_later) {
8e409fe1
LC		 2152 nents = sg_nents_for_len(areq->src, nbytes);
2153 if (nents < 0) {
2154 dev_err(ctx->dev, "Invalid number of src SG.\n");
2155 return nents;
2156 }
d0525723 2157 sg_pcopy_to_buffer(areq->src, nents,
3c0dd190 2158 req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
5e833bc4
LN		 2159 to_hash_later,
2160 nbytes - to_hash_later);
497f2e6b 2161 }
5e833bc4 2162 req_ctx->to_hash_later = to_hash_later;
497f2e6b 2163
5e833bc4 2164 /* Allocate extended descriptor */
497f2e6b
LN		 2165 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2166 if (IS_ERR(edesc))
2167 return PTR_ERR(edesc);
2168
2169 edesc->desc.hdr = ctx->desc_hdr_template;
2170
2171 /* On last one, request SEC to pad; otherwise continue */
2172 if (req_ctx->last)
2173 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2174 else
2175 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2176
60f208d7
KP		 2177 /* request SEC to INIT hash. */
2178 if (req_ctx->first && !req_ctx->swinit)
497f2e6b
LN		 2179 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2180
2181 /* When the tfm context has a keylen, it's an HMAC.
2182 * A first or last (ie. not middle) descriptor must request HMAC.
2183 */
2184 if (ctx->keylen && (req_ctx->first || req_ctx->last))
2185 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2186
37b5e889 2187 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
497f2e6b
LN		 2188 ahash_done);
2189}
2190
2191static int ahash_update(struct ahash_request *areq)
2192{
2193 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2194
2195 req_ctx->last = 0;
2196
2197 return ahash_process_req(areq, areq->nbytes);
2198}
2199
2200static int ahash_final(struct ahash_request *areq)
2201{
2202 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2203
2204 req_ctx->last = 1;
2205
2206 return ahash_process_req(areq, 0);
2207}
2208
2209static int ahash_finup(struct ahash_request *areq)
2210{
2211 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2212
2213 req_ctx->last = 1;
2214
2215 return ahash_process_req(areq, areq->nbytes);
2216}
2217
2218static int ahash_digest(struct ahash_request *areq)
2219{
2220 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
60f208d7 2221 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
497f2e6b 2222
60f208d7 2223 ahash->init(areq);
497f2e6b
LN		 2224 req_ctx->last = 1;
2225
2226 return ahash_process_req(areq, areq->nbytes);
2227}
2228
3639ca84
HG		 2229static int ahash_export(struct ahash_request *areq, void *out)
2230{
2231 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2232 struct talitos_export_state *export = out;
6a4967c3
LC		 2233 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2234 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2235 struct device *dev = ctx->dev;
2236 dma_addr_t dma;
2237
2238 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2239 DMA_FROM_DEVICE);
2240 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
3639ca84
HG		 2241
2242 memcpy(export->hw_context, req_ctx->hw_context,
2243 req_ctx->hw_context_size);
3c0dd190 2244 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
3639ca84
HG		 2245 export->swinit = req_ctx->swinit;
2246 export->first = req_ctx->first;
2247 export->last = req_ctx->last;
2248 export->to_hash_later = req_ctx->to_hash_later;
2249 export->nbuf = req_ctx->nbuf;
2250
2251 return 0;
2252}
2253
2254static int ahash_import(struct ahash_request *areq, const void *in)
2255{
2256 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2257 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
6a4967c3
LC		 2258 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2259 struct device *dev = ctx->dev;
3639ca84 2260 const struct talitos_export_state *export = in;
49f9783b 2261 unsigned int size;
6a4967c3 2262 dma_addr_t dma;
3639ca84
HG		 2263
2264 memset(req_ctx, 0, sizeof(*req_ctx));
49f9783b 2265 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
3639ca84
HG		 2266 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2267 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
49f9783b 2268 req_ctx->hw_context_size = size;
49f9783b 2269 memcpy(req_ctx->hw_context, export->hw_context, size);
3c0dd190 2270 memcpy(req_ctx->buf[0], export->buf, export->nbuf);
3639ca84
HG		 2271 req_ctx->swinit = export->swinit;
2272 req_ctx->first = export->first;
2273 req_ctx->last = export->last;
2274 req_ctx->to_hash_later = export->to_hash_later;
2275 req_ctx->nbuf = export->nbuf;
2276
6a4967c3
LC		 2277 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2278 DMA_TO_DEVICE);
2279 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2280
3639ca84
HG		 2281 return 0;
2282}
2283
79b3a418
LN		 2284static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2285 u8 *hash)
2286{
2287 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2288
2289 struct scatterlist sg[1];
2290 struct ahash_request *req;
f1c90ac3 2291 struct crypto_wait wait;
79b3a418
LN		 2292 int ret;
2293
f1c90ac3 2294 crypto_init_wait(&wait);
79b3a418
LN		 2295
2296 req = ahash_request_alloc(tfm, GFP_KERNEL);
2297 if (!req)
2298 return -ENOMEM;
2299
2300 /* Keep tfm keylen == 0 during hash of the long key */
2301 ctx->keylen = 0;
2302 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
f1c90ac3 2303 crypto_req_done, &wait);
79b3a418
LN		 2304
2305 sg_init_one(&sg[0], key, keylen);
2306
2307 ahash_request_set_crypt(req, sg, hash, keylen);
f1c90ac3
GBY		 2308 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2309
79b3a418
LN		 2310 ahash_request_free(req);
2311
2312 return ret;
2313}
2314
2315static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2316 unsigned int keylen)
2317{
2318 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2e13ce08 2319 struct device *dev = ctx->dev;
79b3a418
LN		 2320 unsigned int blocksize =
2321 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2322 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2323 unsigned int keysize = keylen;
2324 u8 hash[SHA512_DIGEST_SIZE];
2325 int ret;
2326
2327 if (keylen <= blocksize)
2328 memcpy(ctx->key, key, keysize);
2329 else {
2330 /* Must get the hash of the long key */
2331 ret = keyhash(tfm, key, keylen, hash);
2332
2333 if (ret) {
2334 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2335 return -EINVAL;
2336 }
2337
2338 keysize = digestsize;
2339 memcpy(ctx->key, hash, digestsize);
2340 }
2341
2e13ce08
LC		 2342 if (ctx->keylen)
2343 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2344
79b3a418 2345 ctx->keylen = keysize;
2e13ce08 2346 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
79b3a418
LN		 2347
2348 return 0;
2349}
2350
2351
9c4a7965 2352struct talitos_alg_template {
d5e4aaef 2353 u32 type;
b0057763 2354 u32 priority;
d5e4aaef
LN		 2355 union {
2356 struct crypto_alg crypto;
acbf7c62 2357 struct ahash_alg hash;
aeb4c132 2358 struct aead_alg aead;
d5e4aaef 2359 } alg;
9c4a7965
KP		 2360 __be32 desc_hdr_template;
2361};
2362
2363static struct talitos_alg_template driver_algs[] = {
991155ba 2364 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
d5e4aaef 2365 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2366 .alg.aead = {
2367 .base = {
2368 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2369 .cra_driver_name = "authenc-hmac-sha1-"
2370 "cbc-aes-talitos",
2371 .cra_blocksize = AES_BLOCK_SIZE,
2372 .cra_flags = CRYPTO_ALG_ASYNC,
2373 },
2374 .ivsize = AES_BLOCK_SIZE,
2375 .maxauthsize = SHA1_DIGEST_SIZE,
56af8cd4 2376 },
9c4a7965
KP		 2377 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2378 DESC_HDR_SEL0_AESU |
2379 DESC_HDR_MODE0_AESU_CBC |
2380 DESC_HDR_SEL1_MDEUA |
2381 DESC_HDR_MODE1_MDEU_INIT |
2382 DESC_HDR_MODE1_MDEU_PAD |
2383 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
70bcaca7 2384 },
7405c8d7
LC		 2385 { .type = CRYPTO_ALG_TYPE_AEAD,
2386 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2387 .alg.aead = {
2388 .base = {
2389 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2390 .cra_driver_name = "authenc-hmac-sha1-"
a1a42f84 2391 "cbc-aes-talitos-hsna",
7405c8d7
LC		 2392 .cra_blocksize = AES_BLOCK_SIZE,
2393 .cra_flags = CRYPTO_ALG_ASYNC,
2394 },
2395 .ivsize = AES_BLOCK_SIZE,
2396 .maxauthsize = SHA1_DIGEST_SIZE,
2397 },
2398 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2399 DESC_HDR_SEL0_AESU |
2400 DESC_HDR_MODE0_AESU_CBC |
2401 DESC_HDR_SEL1_MDEUA |
2402 DESC_HDR_MODE1_MDEU_INIT |
2403 DESC_HDR_MODE1_MDEU_PAD |
2404 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2405 },
d5e4aaef 2406 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2407 .alg.aead = {
2408 .base = {
2409 .cra_name = "authenc(hmac(sha1),"
2410 "cbc(des3_ede))",
2411 .cra_driver_name = "authenc-hmac-sha1-"
2412 "cbc-3des-talitos",
2413 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2414 .cra_flags = CRYPTO_ALG_ASYNC,
2415 },
2416 .ivsize = DES3_EDE_BLOCK_SIZE,
2417 .maxauthsize = SHA1_DIGEST_SIZE,
ef7c5c85 2418 .setkey = aead_des3_setkey,
56af8cd4 2419 },
70bcaca7
LN		 2420 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2421 DESC_HDR_SEL0_DEU |
2422 DESC_HDR_MODE0_DEU_CBC |
2423 DESC_HDR_MODE0_DEU_3DES |
2424 DESC_HDR_SEL1_MDEUA |
2425 DESC_HDR_MODE1_MDEU_INIT |
2426 DESC_HDR_MODE1_MDEU_PAD |
2427 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
3952f17e 2428 },
7405c8d7
LC		 2429 { .type = CRYPTO_ALG_TYPE_AEAD,
2430 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2431 .alg.aead = {
2432 .base = {
2433 .cra_name = "authenc(hmac(sha1),"
2434 "cbc(des3_ede))",
2435 .cra_driver_name = "authenc-hmac-sha1-"
a1a42f84 2436 "cbc-3des-talitos-hsna",
7405c8d7
LC		 2437 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2438 .cra_flags = CRYPTO_ALG_ASYNC,
2439 },
2440 .ivsize = DES3_EDE_BLOCK_SIZE,
2441 .maxauthsize = SHA1_DIGEST_SIZE,
ef7c5c85 2442 .setkey = aead_des3_setkey,
7405c8d7
LC		 2443 },
2444 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2445 DESC_HDR_SEL0_DEU |
2446 DESC_HDR_MODE0_DEU_CBC |
2447 DESC_HDR_MODE0_DEU_3DES |
2448 DESC_HDR_SEL1_MDEUA |
2449 DESC_HDR_MODE1_MDEU_INIT |
2450 DESC_HDR_MODE1_MDEU_PAD |
2451 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2452 },
357fb605 2453 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2454 .alg.aead = {
2455 .base = {
2456 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2457 .cra_driver_name = "authenc-hmac-sha224-"
2458 "cbc-aes-talitos",
2459 .cra_blocksize = AES_BLOCK_SIZE,
2460 .cra_flags = CRYPTO_ALG_ASYNC,
2461 },
2462 .ivsize = AES_BLOCK_SIZE,
2463 .maxauthsize = SHA224_DIGEST_SIZE,
357fb605
HG		 2464 },
2465 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2466 DESC_HDR_SEL0_AESU |
2467 DESC_HDR_MODE0_AESU_CBC |
2468 DESC_HDR_SEL1_MDEUA |
2469 DESC_HDR_MODE1_MDEU_INIT |
2470 DESC_HDR_MODE1_MDEU_PAD |
2471 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2472 },
7405c8d7
LC		 2473 { .type = CRYPTO_ALG_TYPE_AEAD,
2474 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2475 .alg.aead = {
2476 .base = {
2477 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2478 .cra_driver_name = "authenc-hmac-sha224-"
a1a42f84 2479 "cbc-aes-talitos-hsna",
7405c8d7
LC		 2480 .cra_blocksize = AES_BLOCK_SIZE,
2481 .cra_flags = CRYPTO_ALG_ASYNC,
2482 },
2483 .ivsize = AES_BLOCK_SIZE,
2484 .maxauthsize = SHA224_DIGEST_SIZE,
2485 },
2486 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2487 DESC_HDR_SEL0_AESU |
2488 DESC_HDR_MODE0_AESU_CBC |
2489 DESC_HDR_SEL1_MDEUA |
2490 DESC_HDR_MODE1_MDEU_INIT |
2491 DESC_HDR_MODE1_MDEU_PAD |
2492 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2493 },
357fb605 2494 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2495 .alg.aead = {
2496 .base = {
2497 .cra_name = "authenc(hmac(sha224),"
2498 "cbc(des3_ede))",
2499 .cra_driver_name = "authenc-hmac-sha224-"
2500 "cbc-3des-talitos",
2501 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2502 .cra_flags = CRYPTO_ALG_ASYNC,
2503 },
2504 .ivsize = DES3_EDE_BLOCK_SIZE,
2505 .maxauthsize = SHA224_DIGEST_SIZE,
ef7c5c85 2506 .setkey = aead_des3_setkey,
357fb605
HG		 2507 },
2508 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2509 DESC_HDR_SEL0_DEU |
2510 DESC_HDR_MODE0_DEU_CBC |
2511 DESC_HDR_MODE0_DEU_3DES |
2512 DESC_HDR_SEL1_MDEUA |
2513 DESC_HDR_MODE1_MDEU_INIT |
2514 DESC_HDR_MODE1_MDEU_PAD |
2515 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2516 },
7405c8d7
LC		 2517 { .type = CRYPTO_ALG_TYPE_AEAD,
2518 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2519 .alg.aead = {
2520 .base = {
2521 .cra_name = "authenc(hmac(sha224),"
2522 "cbc(des3_ede))",
2523 .cra_driver_name = "authenc-hmac-sha224-"
a1a42f84 2524 "cbc-3des-talitos-hsna",
7405c8d7
LC		 2525 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2526 .cra_flags = CRYPTO_ALG_ASYNC,
2527 },
2528 .ivsize = DES3_EDE_BLOCK_SIZE,
2529 .maxauthsize = SHA224_DIGEST_SIZE,
ef7c5c85 2530 .setkey = aead_des3_setkey,
7405c8d7
LC		 2531 },
2532 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2533 DESC_HDR_SEL0_DEU |
2534 DESC_HDR_MODE0_DEU_CBC |
2535 DESC_HDR_MODE0_DEU_3DES |
2536 DESC_HDR_SEL1_MDEUA |
2537 DESC_HDR_MODE1_MDEU_INIT |
2538 DESC_HDR_MODE1_MDEU_PAD |
2539 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2540 },
d5e4aaef 2541 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2542 .alg.aead = {
2543 .base = {
2544 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2545 .cra_driver_name = "authenc-hmac-sha256-"
2546 "cbc-aes-talitos",
2547 .cra_blocksize = AES_BLOCK_SIZE,
2548 .cra_flags = CRYPTO_ALG_ASYNC,
2549 },
2550 .ivsize = AES_BLOCK_SIZE,
2551 .maxauthsize = SHA256_DIGEST_SIZE,
56af8cd4 2552 },
3952f17e
LN		 2553 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2554 DESC_HDR_SEL0_AESU |
2555 DESC_HDR_MODE0_AESU_CBC |
2556 DESC_HDR_SEL1_MDEUA |
2557 DESC_HDR_MODE1_MDEU_INIT |
2558 DESC_HDR_MODE1_MDEU_PAD |
2559 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2560 },
7405c8d7
LC		 2561 { .type = CRYPTO_ALG_TYPE_AEAD,
2562 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2563 .alg.aead = {
2564 .base = {
2565 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2566 .cra_driver_name = "authenc-hmac-sha256-"
a1a42f84 2567 "cbc-aes-talitos-hsna",
7405c8d7
LC		 2568 .cra_blocksize = AES_BLOCK_SIZE,
2569 .cra_flags = CRYPTO_ALG_ASYNC,
2570 },
2571 .ivsize = AES_BLOCK_SIZE,
2572 .maxauthsize = SHA256_DIGEST_SIZE,
2573 },
2574 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2575 DESC_HDR_SEL0_AESU |
2576 DESC_HDR_MODE0_AESU_CBC |
2577 DESC_HDR_SEL1_MDEUA |
2578 DESC_HDR_MODE1_MDEU_INIT |
2579 DESC_HDR_MODE1_MDEU_PAD |
2580 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2581 },
d5e4aaef 2582 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2583 .alg.aead = {
2584 .base = {
2585 .cra_name = "authenc(hmac(sha256),"
2586 "cbc(des3_ede))",
2587 .cra_driver_name = "authenc-hmac-sha256-"
2588 "cbc-3des-talitos",
2589 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2590 .cra_flags = CRYPTO_ALG_ASYNC,
2591 },
2592 .ivsize = DES3_EDE_BLOCK_SIZE,
2593 .maxauthsize = SHA256_DIGEST_SIZE,
ef7c5c85 2594 .setkey = aead_des3_setkey,
56af8cd4 2595 },
3952f17e
LN		 2596 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2597 DESC_HDR_SEL0_DEU |
2598 DESC_HDR_MODE0_DEU_CBC |
2599 DESC_HDR_MODE0_DEU_3DES |
2600 DESC_HDR_SEL1_MDEUA |
2601 DESC_HDR_MODE1_MDEU_INIT |
2602 DESC_HDR_MODE1_MDEU_PAD |
2603 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2604 },
7405c8d7
LC		 2605 { .type = CRYPTO_ALG_TYPE_AEAD,
2606 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2607 .alg.aead = {
2608 .base = {
2609 .cra_name = "authenc(hmac(sha256),"
2610 "cbc(des3_ede))",
2611 .cra_driver_name = "authenc-hmac-sha256-"
a1a42f84 2612 "cbc-3des-talitos-hsna",
7405c8d7
LC		 2613 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2614 .cra_flags = CRYPTO_ALG_ASYNC,
2615 },
2616 .ivsize = DES3_EDE_BLOCK_SIZE,
2617 .maxauthsize = SHA256_DIGEST_SIZE,
ef7c5c85 2618 .setkey = aead_des3_setkey,
7405c8d7
LC		 2619 },
2620 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2621 DESC_HDR_SEL0_DEU |
2622 DESC_HDR_MODE0_DEU_CBC |
2623 DESC_HDR_MODE0_DEU_3DES |
2624 DESC_HDR_SEL1_MDEUA |
2625 DESC_HDR_MODE1_MDEU_INIT |
2626 DESC_HDR_MODE1_MDEU_PAD |
2627 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2628 },
d5e4aaef 2629 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2630 .alg.aead = {
2631 .base = {
2632 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2633 .cra_driver_name = "authenc-hmac-sha384-"
2634 "cbc-aes-talitos",
2635 .cra_blocksize = AES_BLOCK_SIZE,
2636 .cra_flags = CRYPTO_ALG_ASYNC,
2637 },
2638 .ivsize = AES_BLOCK_SIZE,
2639 .maxauthsize = SHA384_DIGEST_SIZE,
357fb605
HG		 2640 },
2641 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2642 DESC_HDR_SEL0_AESU |
2643 DESC_HDR_MODE0_AESU_CBC |
2644 DESC_HDR_SEL1_MDEUB |
2645 DESC_HDR_MODE1_MDEU_INIT |
2646 DESC_HDR_MODE1_MDEU_PAD |
2647 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2648 },
2649 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2650 .alg.aead = {
2651 .base = {
2652 .cra_name = "authenc(hmac(sha384),"
2653 "cbc(des3_ede))",
2654 .cra_driver_name = "authenc-hmac-sha384-"
2655 "cbc-3des-talitos",
2656 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2657 .cra_flags = CRYPTO_ALG_ASYNC,
2658 },
2659 .ivsize = DES3_EDE_BLOCK_SIZE,
2660 .maxauthsize = SHA384_DIGEST_SIZE,
ef7c5c85 2661 .setkey = aead_des3_setkey,
357fb605
HG		 2662 },
2663 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2664 DESC_HDR_SEL0_DEU |
2665 DESC_HDR_MODE0_DEU_CBC |
2666 DESC_HDR_MODE0_DEU_3DES |
2667 DESC_HDR_SEL1_MDEUB |
2668 DESC_HDR_MODE1_MDEU_INIT |
2669 DESC_HDR_MODE1_MDEU_PAD |
2670 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2671 },
2672 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2673 .alg.aead = {
2674 .base = {
2675 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2676 .cra_driver_name = "authenc-hmac-sha512-"
2677 "cbc-aes-talitos",
2678 .cra_blocksize = AES_BLOCK_SIZE,
2679 .cra_flags = CRYPTO_ALG_ASYNC,
2680 },
2681 .ivsize = AES_BLOCK_SIZE,
2682 .maxauthsize = SHA512_DIGEST_SIZE,
357fb605
HG		 2683 },
2684 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2685 DESC_HDR_SEL0_AESU |
2686 DESC_HDR_MODE0_AESU_CBC |
2687 DESC_HDR_SEL1_MDEUB |
2688 DESC_HDR_MODE1_MDEU_INIT |
2689 DESC_HDR_MODE1_MDEU_PAD |
2690 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2691 },
2692 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2693 .alg.aead = {
2694 .base = {
2695 .cra_name = "authenc(hmac(sha512),"
2696 "cbc(des3_ede))",
2697 .cra_driver_name = "authenc-hmac-sha512-"
2698 "cbc-3des-talitos",
2699 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2700 .cra_flags = CRYPTO_ALG_ASYNC,
2701 },
2702 .ivsize = DES3_EDE_BLOCK_SIZE,
2703 .maxauthsize = SHA512_DIGEST_SIZE,
ef7c5c85 2704 .setkey = aead_des3_setkey,
357fb605
HG		 2705 },
2706 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2707 DESC_HDR_SEL0_DEU |
2708 DESC_HDR_MODE0_DEU_CBC |
2709 DESC_HDR_MODE0_DEU_3DES |
2710 DESC_HDR_SEL1_MDEUB |
2711 DESC_HDR_MODE1_MDEU_INIT |
2712 DESC_HDR_MODE1_MDEU_PAD |
2713 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2714 },
2715 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2716 .alg.aead = {
2717 .base = {
2718 .cra_name = "authenc(hmac(md5),cbc(aes))",
2719 .cra_driver_name = "authenc-hmac-md5-"
2720 "cbc-aes-talitos",
2721 .cra_blocksize = AES_BLOCK_SIZE,
2722 .cra_flags = CRYPTO_ALG_ASYNC,
2723 },
2724 .ivsize = AES_BLOCK_SIZE,
2725 .maxauthsize = MD5_DIGEST_SIZE,
56af8cd4 2726 },
3952f17e
LN		 2727 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2728 DESC_HDR_SEL0_AESU |
2729 DESC_HDR_MODE0_AESU_CBC |
2730 DESC_HDR_SEL1_MDEUA |
2731 DESC_HDR_MODE1_MDEU_INIT |
2732 DESC_HDR_MODE1_MDEU_PAD |
2733 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2734 },
7405c8d7
LC		 2735 { .type = CRYPTO_ALG_TYPE_AEAD,
2736 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2737 .alg.aead = {
2738 .base = {
2739 .cra_name = "authenc(hmac(md5),cbc(aes))",
2740 .cra_driver_name = "authenc-hmac-md5-"
a1a42f84 2741 "cbc-aes-talitos-hsna",
7405c8d7
LC		 2742 .cra_blocksize = AES_BLOCK_SIZE,
2743 .cra_flags = CRYPTO_ALG_ASYNC,
2744 },
2745 .ivsize = AES_BLOCK_SIZE,
2746 .maxauthsize = MD5_DIGEST_SIZE,
2747 },
2748 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2749 DESC_HDR_SEL0_AESU |
2750 DESC_HDR_MODE0_AESU_CBC |
2751 DESC_HDR_SEL1_MDEUA |
2752 DESC_HDR_MODE1_MDEU_INIT |
2753 DESC_HDR_MODE1_MDEU_PAD |
2754 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2755 },
d5e4aaef 2756 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2757 .alg.aead = {
2758 .base = {
2759 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2760 .cra_driver_name = "authenc-hmac-md5-"
2761 "cbc-3des-talitos",
2762 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2763 .cra_flags = CRYPTO_ALG_ASYNC,
2764 },
2765 .ivsize = DES3_EDE_BLOCK_SIZE,
2766 .maxauthsize = MD5_DIGEST_SIZE,
ef7c5c85 2767 .setkey = aead_des3_setkey,
56af8cd4 2768 },
3952f17e
LN		 2769 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2770 DESC_HDR_SEL0_DEU |
2771 DESC_HDR_MODE0_DEU_CBC |
2772 DESC_HDR_MODE0_DEU_3DES |
2773 DESC_HDR_SEL1_MDEUA |
2774 DESC_HDR_MODE1_MDEU_INIT |
2775 DESC_HDR_MODE1_MDEU_PAD |
2776 DESC_HDR_MODE1_MDEU_MD5_HMAC,
4de9d0b5 2777 },
7405c8d7
LC		 2778 { .type = CRYPTO_ALG_TYPE_AEAD,
2779 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2780 .alg.aead = {
2781 .base = {
2782 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2783 .cra_driver_name = "authenc-hmac-md5-"
a1a42f84 2784 "cbc-3des-talitos-hsna",
7405c8d7
LC		 2785 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2786 .cra_flags = CRYPTO_ALG_ASYNC,
2787 },
2788 .ivsize = DES3_EDE_BLOCK_SIZE,
2789 .maxauthsize = MD5_DIGEST_SIZE,
ef7c5c85 2790 .setkey = aead_des3_setkey,
7405c8d7
LC		 2791 },
2792 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2793 DESC_HDR_SEL0_DEU |
2794 DESC_HDR_MODE0_DEU_CBC |
2795 DESC_HDR_MODE0_DEU_3DES |
2796 DESC_HDR_SEL1_MDEUA |
2797 DESC_HDR_MODE1_MDEU_INIT |
2798 DESC_HDR_MODE1_MDEU_PAD |
2799 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2800 },
4de9d0b5 2801 /* ABLKCIPHER algorithms. */
5e75ae1b
LC		 2802 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2803 .alg.crypto = {
2804 .cra_name = "ecb(aes)",
2805 .cra_driver_name = "ecb-aes-talitos",
2806 .cra_blocksize = AES_BLOCK_SIZE,
2807 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2808 CRYPTO_ALG_ASYNC,
2809 .cra_ablkcipher = {
2810 .min_keysize = AES_MIN_KEY_SIZE,
2811 .max_keysize = AES_MAX_KEY_SIZE,
1ba34e71 2812 .setkey = ablkcipher_aes_setkey,
5e75ae1b
LC		 2813 }
2814 },
2815 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2816 DESC_HDR_SEL0_AESU,
2817 },
d5e4aaef
LN		 2818 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2819 .alg.crypto = {
4de9d0b5
LN		 2820 .cra_name = "cbc(aes)",
2821 .cra_driver_name = "cbc-aes-talitos",
2822 .cra_blocksize = AES_BLOCK_SIZE,
2823 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2824 CRYPTO_ALG_ASYNC,
4de9d0b5 2825 .cra_ablkcipher = {
4de9d0b5
LN		 2826 .min_keysize = AES_MIN_KEY_SIZE,
2827 .max_keysize = AES_MAX_KEY_SIZE,
2828 .ivsize = AES_BLOCK_SIZE,
1ba34e71 2829 .setkey = ablkcipher_aes_setkey,
4de9d0b5
LN		 2830 }
2831 },
2832 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2833 DESC_HDR_SEL0_AESU |
2834 DESC_HDR_MODE0_AESU_CBC,
2835 },
5e75ae1b
LC		 2836 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2837 .alg.crypto = {
2838 .cra_name = "ctr(aes)",
2839 .cra_driver_name = "ctr-aes-talitos",
b9a05b60 2840 .cra_blocksize = 1,
5e75ae1b
LC		 2841 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2842 CRYPTO_ALG_ASYNC,
2843 .cra_ablkcipher = {
2844 .min_keysize = AES_MIN_KEY_SIZE,
2845 .max_keysize = AES_MAX_KEY_SIZE,
2846 .ivsize = AES_BLOCK_SIZE,
1ba34e71 2847 .setkey = ablkcipher_aes_setkey,
5e75ae1b
LC		 2848 }
2849 },
70d355cc 2850 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
5e75ae1b
LC		 2851 DESC_HDR_SEL0_AESU |
2852 DESC_HDR_MODE0_AESU_CTR,
2853 },
2854 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2855 .alg.crypto = {
2856 .cra_name = "ecb(des)",
2857 .cra_driver_name = "ecb-des-talitos",
2858 .cra_blocksize = DES_BLOCK_SIZE,
2859 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2860 CRYPTO_ALG_ASYNC,
2861 .cra_ablkcipher = {
2862 .min_keysize = DES_KEY_SIZE,
2863 .max_keysize = DES_KEY_SIZE,
ef7c5c85 2864 .setkey = ablkcipher_des_setkey,
5e75ae1b
LC		 2865 }
2866 },
2867 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2868 DESC_HDR_SEL0_DEU,
2869 },
2870 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2871 .alg.crypto = {
2872 .cra_name = "cbc(des)",
2873 .cra_driver_name = "cbc-des-talitos",
2874 .cra_blocksize = DES_BLOCK_SIZE,
2875 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2876 CRYPTO_ALG_ASYNC,
2877 .cra_ablkcipher = {
2878 .min_keysize = DES_KEY_SIZE,
2879 .max_keysize = DES_KEY_SIZE,
2880 .ivsize = DES_BLOCK_SIZE,
ef7c5c85 2881 .setkey = ablkcipher_des_setkey,
5e75ae1b
LC		 2882 }
2883 },
2884 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2885 DESC_HDR_SEL0_DEU |
2886 DESC_HDR_MODE0_DEU_CBC,
2887 },
2888 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2889 .alg.crypto = {
2890 .cra_name = "ecb(des3_ede)",
2891 .cra_driver_name = "ecb-3des-talitos",
2892 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2893 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2894 CRYPTO_ALG_ASYNC,
2895 .cra_ablkcipher = {
2896 .min_keysize = DES3_EDE_KEY_SIZE,
2897 .max_keysize = DES3_EDE_KEY_SIZE,
ef7c5c85 2898 .setkey = ablkcipher_des3_setkey,
5e75ae1b
LC		 2899 }
2900 },
2901 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2902 DESC_HDR_SEL0_DEU |
2903 DESC_HDR_MODE0_DEU_3DES,
2904 },
d5e4aaef
LN		 2905 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2906 .alg.crypto = {
4de9d0b5
LN		 2907 .cra_name = "cbc(des3_ede)",
2908 .cra_driver_name = "cbc-3des-talitos",
2909 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2910 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2911 CRYPTO_ALG_ASYNC,
4de9d0b5 2912 .cra_ablkcipher = {
4de9d0b5
LN		 2913 .min_keysize = DES3_EDE_KEY_SIZE,
2914 .max_keysize = DES3_EDE_KEY_SIZE,
2915 .ivsize = DES3_EDE_BLOCK_SIZE,
ef7c5c85 2916 .setkey = ablkcipher_des3_setkey,
4de9d0b5
LN		 2917 }
2918 },
2919 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2920 DESC_HDR_SEL0_DEU |
2921 DESC_HDR_MODE0_DEU_CBC |
2922 DESC_HDR_MODE0_DEU_3DES,
497f2e6b
LN		 2923 },
2924 /* AHASH algorithms. */
2925 { .type = CRYPTO_ALG_TYPE_AHASH,
2926 .alg.hash = {
497f2e6b 2927 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 2928 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2929 .halg.base = {
2930 .cra_name = "md5",
2931 .cra_driver_name = "md5-talitos",
b3988618 2932 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
6a38f622 2933 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2934 }
2935 },
2936 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2937 DESC_HDR_SEL0_MDEUA |
2938 DESC_HDR_MODE0_MDEU_MD5,
2939 },
2940 { .type = CRYPTO_ALG_TYPE_AHASH,
2941 .alg.hash = {
497f2e6b 2942 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 2943 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2944 .halg.base = {
2945 .cra_name = "sha1",
2946 .cra_driver_name = "sha1-talitos",
2947 .cra_blocksize = SHA1_BLOCK_SIZE,
6a38f622 2948 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2949 }
2950 },
2951 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2952 DESC_HDR_SEL0_MDEUA |
2953 DESC_HDR_MODE0_MDEU_SHA1,
2954 },
60f208d7
KP		 2955 { .type = CRYPTO_ALG_TYPE_AHASH,
2956 .alg.hash = {
60f208d7 2957 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 2958 .halg.statesize = sizeof(struct talitos_export_state),
60f208d7
KP		 2959 .halg.base = {
2960 .cra_name = "sha224",
2961 .cra_driver_name = "sha224-talitos",
2962 .cra_blocksize = SHA224_BLOCK_SIZE,
6a38f622 2963 .cra_flags = CRYPTO_ALG_ASYNC,
60f208d7
KP		 2964 }
2965 },
2966 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2967 DESC_HDR_SEL0_MDEUA |
2968 DESC_HDR_MODE0_MDEU_SHA224,
2969 },
497f2e6b
LN		 2970 { .type = CRYPTO_ALG_TYPE_AHASH,
2971 .alg.hash = {
497f2e6b 2972 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 2973 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2974 .halg.base = {
2975 .cra_name = "sha256",
2976 .cra_driver_name = "sha256-talitos",
2977 .cra_blocksize = SHA256_BLOCK_SIZE,
6a38f622 2978 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2979 }
2980 },
2981 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2982 DESC_HDR_SEL0_MDEUA |
2983 DESC_HDR_MODE0_MDEU_SHA256,
2984 },
2985 { .type = CRYPTO_ALG_TYPE_AHASH,
2986 .alg.hash = {
497f2e6b 2987 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 2988 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2989 .halg.base = {
2990 .cra_name = "sha384",
2991 .cra_driver_name = "sha384-talitos",
2992 .cra_blocksize = SHA384_BLOCK_SIZE,
6a38f622 2993 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2994 }
2995 },
2996 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2997 DESC_HDR_SEL0_MDEUB |
2998 DESC_HDR_MODE0_MDEUB_SHA384,
2999 },
3000 { .type = CRYPTO_ALG_TYPE_AHASH,
3001 .alg.hash = {
497f2e6b 3002 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 3003 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 3004 .halg.base = {
3005 .cra_name = "sha512",
3006 .cra_driver_name = "sha512-talitos",
3007 .cra_blocksize = SHA512_BLOCK_SIZE,
6a38f622 3008 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 3009 }
3010 },
3011 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3012 DESC_HDR_SEL0_MDEUB |
3013 DESC_HDR_MODE0_MDEUB_SHA512,
3014 },
79b3a418
LN		 3015 { .type = CRYPTO_ALG_TYPE_AHASH,
3016 .alg.hash = {
79b3a418 3017 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 3018 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3019 .halg.base = {
3020 .cra_name = "hmac(md5)",
3021 .cra_driver_name = "hmac-md5-talitos",
b3988618 3022 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
6a38f622 3023 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3024 }
3025 },
3026 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3027 DESC_HDR_SEL0_MDEUA |
3028 DESC_HDR_MODE0_MDEU_MD5,
3029 },
3030 { .type = CRYPTO_ALG_TYPE_AHASH,
3031 .alg.hash = {
79b3a418 3032 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 3033 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3034 .halg.base = {
3035 .cra_name = "hmac(sha1)",
3036 .cra_driver_name = "hmac-sha1-talitos",
3037 .cra_blocksize = SHA1_BLOCK_SIZE,
6a38f622 3038 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3039 }
3040 },
3041 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3042 DESC_HDR_SEL0_MDEUA |
3043 DESC_HDR_MODE0_MDEU_SHA1,
3044 },
3045 { .type = CRYPTO_ALG_TYPE_AHASH,
3046 .alg.hash = {
79b3a418 3047 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 3048 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3049 .halg.base = {
3050 .cra_name = "hmac(sha224)",
3051 .cra_driver_name = "hmac-sha224-talitos",
3052 .cra_blocksize = SHA224_BLOCK_SIZE,
6a38f622 3053 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3054 }
3055 },
3056 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3057 DESC_HDR_SEL0_MDEUA |
3058 DESC_HDR_MODE0_MDEU_SHA224,
3059 },
3060 { .type = CRYPTO_ALG_TYPE_AHASH,
3061 .alg.hash = {
79b3a418 3062 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 3063 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3064 .halg.base = {
3065 .cra_name = "hmac(sha256)",
3066 .cra_driver_name = "hmac-sha256-talitos",
3067 .cra_blocksize = SHA256_BLOCK_SIZE,
6a38f622 3068 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3069 }
3070 },
3071 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3072 DESC_HDR_SEL0_MDEUA |
3073 DESC_HDR_MODE0_MDEU_SHA256,
3074 },
3075 { .type = CRYPTO_ALG_TYPE_AHASH,
3076 .alg.hash = {
79b3a418 3077 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 3078 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3079 .halg.base = {
3080 .cra_name = "hmac(sha384)",
3081 .cra_driver_name = "hmac-sha384-talitos",
3082 .cra_blocksize = SHA384_BLOCK_SIZE,
6a38f622 3083 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3084 }
3085 },
3086 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3087 DESC_HDR_SEL0_MDEUB |
3088 DESC_HDR_MODE0_MDEUB_SHA384,
3089 },
3090 { .type = CRYPTO_ALG_TYPE_AHASH,
3091 .alg.hash = {
79b3a418 3092 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 3093 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 3094 .halg.base = {
3095 .cra_name = "hmac(sha512)",
3096 .cra_driver_name = "hmac-sha512-talitos",
3097 .cra_blocksize = SHA512_BLOCK_SIZE,
6a38f622 3098 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 3099 }
3100 },
3101 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3102 DESC_HDR_SEL0_MDEUB |
3103 DESC_HDR_MODE0_MDEUB_SHA512,
3104 }
9c4a7965
KP		 3105};
3106
3107struct talitos_crypto_alg {
3108 struct list_head entry;
3109 struct device *dev;
acbf7c62 3110 struct talitos_alg_template algt;
9c4a7965
KP		 3111};
3112
89d124cb
JE		 3113static int talitos_init_common(struct talitos_ctx *ctx,
3114 struct talitos_crypto_alg *talitos_alg)
9c4a7965 3115{
5228f0f7 3116 struct talitos_private *priv;
9c4a7965
KP		 3117
3118 /* update context with ptr to dev */
3119 ctx->dev = talitos_alg->dev;
19bbbc63 3120
5228f0f7
KP		 3121 /* assign SEC channel to tfm in round-robin fashion */
3122 priv = dev_get_drvdata(ctx->dev);
3123 ctx->ch = atomic_inc_return(&priv->last_chan) &
3124 (priv->num_channels - 1);
3125
9c4a7965 3126 /* copy descriptor header template value */
acbf7c62 3127 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
9c4a7965 3128
602dba5a
KP		 3129 /* select done notification */
3130 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3131
497f2e6b
LN		 3132 return 0;
3133}
3134
89d124cb
JE		 3135static int talitos_cra_init(struct crypto_tfm *tfm)
3136{
3137 struct crypto_alg *alg = tfm->__crt_alg;
3138 struct talitos_crypto_alg *talitos_alg;
3139 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3140
3141 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
3142 talitos_alg = container_of(__crypto_ahash_alg(alg),
3143 struct talitos_crypto_alg,
3144 algt.alg.hash);
3145 else
3146 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3147 algt.alg.crypto);
3148
3149 return talitos_init_common(ctx, talitos_alg);
3150}
3151
aeb4c132 3152static int talitos_cra_init_aead(struct crypto_aead *tfm)
497f2e6b 3153{
89d124cb
JE		 3154 struct aead_alg *alg = crypto_aead_alg(tfm);
3155 struct talitos_crypto_alg *talitos_alg;
3156 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3157
3158 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3159 algt.alg.aead);
3160
3161 return talitos_init_common(ctx, talitos_alg);
9c4a7965
KP		 3162}
3163
497f2e6b
LN		 3164static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3165{
3166 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3167
3168 talitos_cra_init(tfm);
3169
3170 ctx->keylen = 0;
3171 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3172 sizeof(struct talitos_ahash_req_ctx));
3173
3174 return 0;
3175}
3176
2e13ce08
LC		 3177static void talitos_cra_exit(struct crypto_tfm *tfm)
3178{
3179 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3180 struct device *dev = ctx->dev;
3181
3182 if (ctx->keylen)
3183 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3184}
3185
9c4a7965
KP		 3186/*
3187 * given the alg's descriptor header template, determine whether descriptor
3188 * type and primary/secondary execution units required match the hw
3189 * capabilities description provided in the device tree node.
3190 */
3191static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3192{
3193 struct talitos_private *priv = dev_get_drvdata(dev);
3194 int ret;
3195
3196 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3197 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3198
3199 if (SECONDARY_EU(desc_hdr_template))
3200 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3201 & priv->exec_units);
3202
3203 return ret;
3204}
3205
2dc11581 3206static int talitos_remove(struct platform_device *ofdev)
9c4a7965
KP		 3207{
3208 struct device *dev = &ofdev->dev;
3209 struct talitos_private *priv = dev_get_drvdata(dev);
3210 struct talitos_crypto_alg *t_alg, *n;
3211 int i;
3212
3213 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
acbf7c62
LN		 3214 switch (t_alg->algt.type) {
3215 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62 3216 break;
aeb4c132
HX		 3217 case CRYPTO_ALG_TYPE_AEAD:
3218 crypto_unregister_aead(&t_alg->algt.alg.aead);
acbf7c62
LN		 3219 case CRYPTO_ALG_TYPE_AHASH:
3220 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3221 break;
3222 }
9c4a7965 3223 list_del(&t_alg->entry);
9c4a7965
KP		 3224 }
3225
3226 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3227 talitos_unregister_rng(dev);
3228
c3e337f8 3229 for (i = 0; i < 2; i++)
2cdba3cf 3230 if (priv->irq[i]) {
c3e337f8
KP		 3231 free_irq(priv->irq[i], dev);
3232 irq_dispose_mapping(priv->irq[i]);
3233 }
9c4a7965 3234
c3e337f8 3235 tasklet_kill(&priv->done_task[0]);
2cdba3cf 3236 if (priv->irq[1])
c3e337f8 3237 tasklet_kill(&priv->done_task[1]);
9c4a7965 3238
9c4a7965
KP		 3239 return 0;
3240}
3241
3242static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3243 struct talitos_alg_template
3244 *template)
3245{
60f208d7 3246 struct talitos_private *priv = dev_get_drvdata(dev);
9c4a7965
KP		 3247 struct talitos_crypto_alg *t_alg;
3248 struct crypto_alg *alg;
3249
24b92ff2
LC		 3250 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3251 GFP_KERNEL);
9c4a7965
KP		 3252 if (!t_alg)
3253 return ERR_PTR(-ENOMEM);
3254
acbf7c62
LN		 3255 t_alg->algt = *template;
3256
3257 switch (t_alg->algt.type) {
3258 case CRYPTO_ALG_TYPE_ABLKCIPHER:
497f2e6b
LN		 3259 alg = &t_alg->algt.alg.crypto;
3260 alg->cra_init = talitos_cra_init;
2e13ce08 3261 alg->cra_exit = talitos_cra_exit;
d4cd3283 3262 alg->cra_type = &crypto_ablkcipher_type;
ef7c5c85
HX		 3263 alg->cra_ablkcipher.setkey = alg->cra_ablkcipher.setkey ?:
3264 ablkcipher_setkey;
b286e003
KP		 3265 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3266 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
497f2e6b 3267 break;
acbf7c62 3268 case CRYPTO_ALG_TYPE_AEAD:
aeb4c132 3269 alg = &t_alg->algt.alg.aead.base;
2e13ce08 3270 alg->cra_exit = talitos_cra_exit;
aeb4c132 3271 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
ef7c5c85
HX		 3272 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3273 aead_setkey;
aeb4c132
HX		 3274 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3275 t_alg->algt.alg.aead.decrypt = aead_decrypt;
6cda075a
LC		 3276 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3277 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
24b92ff2 3278 devm_kfree(dev, t_alg);
6cda075a
LC		 3279 return ERR_PTR(-ENOTSUPP);
3280 }
acbf7c62
LN		 3281 break;
3282 case CRYPTO_ALG_TYPE_AHASH:
3283 alg = &t_alg->algt.alg.hash.halg.base;
497f2e6b 3284 alg->cra_init = talitos_cra_init_ahash;
ad4cd51f 3285 alg->cra_exit = talitos_cra_exit;
b286e003
KP		 3286 t_alg->algt.alg.hash.init = ahash_init;
3287 t_alg->algt.alg.hash.update = ahash_update;
3288 t_alg->algt.alg.hash.final = ahash_final;
3289 t_alg->algt.alg.hash.finup = ahash_finup;
3290 t_alg->algt.alg.hash.digest = ahash_digest;
56136631
LC		 3291 if (!strncmp(alg->cra_name, "hmac", 4))
3292 t_alg->algt.alg.hash.setkey = ahash_setkey;
3639ca84
HG		 3293 t_alg->algt.alg.hash.import = ahash_import;
3294 t_alg->algt.alg.hash.export = ahash_export;
b286e003 3295
79b3a418 3296 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
0b2730d8 3297 !strncmp(alg->cra_name, "hmac", 4)) {
24b92ff2 3298 devm_kfree(dev, t_alg);
79b3a418 3299 return ERR_PTR(-ENOTSUPP);
0b2730d8 3300 }
60f208d7 3301 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
79b3a418
LN		 3302 (!strcmp(alg->cra_name, "sha224") ||
3303 !strcmp(alg->cra_name, "hmac(sha224)"))) {
60f208d7
KP		 3304 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3305 t_alg->algt.desc_hdr_template =
3306 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3307 DESC_HDR_SEL0_MDEUA |
3308 DESC_HDR_MODE0_MDEU_SHA256;
3309 }
497f2e6b 3310 break;
1d11911a
KP		 3311 default:
3312 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
24b92ff2 3313 devm_kfree(dev, t_alg);
1d11911a 3314 return ERR_PTR(-EINVAL);
acbf7c62 3315 }
9c4a7965 3316
9c4a7965 3317 alg->cra_module = THIS_MODULE;
b0057763
LC		 3318 if (t_alg->algt.priority)
3319 alg->cra_priority = t_alg->algt.priority;
3320 else
3321 alg->cra_priority = TALITOS_CRA_PRIORITY;
9c4a7965 3322 alg->cra_alignmask = 0;
9c4a7965 3323 alg->cra_ctxsize = sizeof(struct talitos_ctx);
d912bb76 3324 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
9c4a7965 3325
9c4a7965
KP		 3326 t_alg->dev = dev;
3327
3328 return t_alg;
3329}
3330
c3e337f8
KP		 3331static int talitos_probe_irq(struct platform_device *ofdev)
3332{
3333 struct device *dev = &ofdev->dev;
3334 struct device_node *np = ofdev->dev.of_node;
3335 struct talitos_private *priv = dev_get_drvdata(dev);
3336 int err;
dd3c0987 3337 bool is_sec1 = has_ftr_sec1(priv);
c3e337f8
KP		 3338
3339 priv->irq[0] = irq_of_parse_and_map(np, 0);
2cdba3cf 3340 if (!priv->irq[0]) {
c3e337f8
KP		 3341 dev_err(dev, "failed to map irq\n");
3342 return -EINVAL;
3343 }
dd3c0987
LC		 3344 if (is_sec1) {
3345 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3346 dev_driver_string(dev), dev);
3347 goto primary_out;
3348 }
c3e337f8
KP		 3349
3350 priv->irq[1] = irq_of_parse_and_map(np, 1);
3351
3352 /* get the primary irq line */
2cdba3cf 3353 if (!priv->irq[1]) {
dd3c0987 3354 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
c3e337f8
KP		 3355 dev_driver_string(dev), dev);
3356 goto primary_out;
3357 }
3358
dd3c0987 3359 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
c3e337f8
KP		 3360 dev_driver_string(dev), dev);
3361 if (err)
3362 goto primary_out;
3363
3364 /* get the secondary irq line */
dd3c0987 3365 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
c3e337f8
KP		 3366 dev_driver_string(dev), dev);
3367 if (err) {
3368 dev_err(dev, "failed to request secondary irq\n");
3369 irq_dispose_mapping(priv->irq[1]);
2cdba3cf 3370 priv->irq[1] = 0;
c3e337f8
KP		 3371 }
3372
3373 return err;
3374
3375primary_out:
3376 if (err) {
3377 dev_err(dev, "failed to request primary irq\n");
3378 irq_dispose_mapping(priv->irq[0]);
2cdba3cf 3379 priv->irq[0] = 0;
c3e337f8
KP		 3380 }
3381
3382 return err;
3383}
3384
1c48a5c9 3385static int talitos_probe(struct platform_device *ofdev)
9c4a7965
KP		 3386{
3387 struct device *dev = &ofdev->dev;
61c7a080 3388 struct device_node *np = ofdev->dev.of_node;
9c4a7965 3389 struct talitos_private *priv;
9c4a7965 3390 int i, err;
5fa7fa14 3391 int stride;
fd5ea7f0 3392 struct resource *res;
9c4a7965 3393
24b92ff2 3394 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
9c4a7965
KP		 3395 if (!priv)
3396 return -ENOMEM;
3397
f3de9cb1
KH		 3398 INIT_LIST_HEAD(&priv->alg_list);
3399
9c4a7965
KP		 3400 dev_set_drvdata(dev, priv);
3401
3402 priv->ofdev = ofdev;
3403
511d63cb
HG		 3404 spin_lock_init(&priv->reg_lock);
3405
fd5ea7f0
LC		 3406 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3407 if (!res)
3408 return -ENXIO;
3409 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
9c4a7965
KP		 3410 if (!priv->reg) {
3411 dev_err(dev, "failed to of_iomap\n");
3412 err = -ENOMEM;
3413 goto err_out;
3414 }
3415
3416 /* get SEC version capabilities from device tree */
fa14c6cf
LC		 3417 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3418 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3419 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3420 of_property_read_u32(np, "fsl,descriptor-types-mask",
3421 &priv->desc_types);
9c4a7965
KP		 3422
3423 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3424 !priv->exec_units || !priv->desc_types) {
3425 dev_err(dev, "invalid property data in device tree node\n");
3426 err = -EINVAL;
3427 goto err_out;
3428 }
3429
f3c85bc1
LN		 3430 if (of_device_is_compatible(np, "fsl,sec3.0"))
3431 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3432
fe5720e2 3433 if (of_device_is_compatible(np, "fsl,sec2.1"))
60f208d7 3434 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
79b3a418
LN		 3435 TALITOS_FTR_SHA224_HWINIT |
3436 TALITOS_FTR_HMAC_OK;
fe5720e2 3437
21590888
LC		 3438 if (of_device_is_compatible(np, "fsl,sec1.0"))
3439 priv->features |= TALITOS_FTR_SEC1;
3440
5fa7fa14
LC		 3441 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3442 priv->reg_deu = priv->reg + TALITOS12_DEU;
3443 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3444 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3445 stride = TALITOS1_CH_STRIDE;
3446 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3447 priv->reg_deu = priv->reg + TALITOS10_DEU;
3448 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3449 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3450 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3451 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3452 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3453 stride = TALITOS1_CH_STRIDE;
3454 } else {
3455 priv->reg_deu = priv->reg + TALITOS2_DEU;
3456 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3457 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3458 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3459 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3460 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3461 priv->reg_keu = priv->reg + TALITOS2_KEU;
3462 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3463 stride = TALITOS2_CH_STRIDE;
3464 }
3465
dd3c0987
LC		 3466 err = talitos_probe_irq(ofdev);
3467 if (err)
3468 goto err_out;
3469
3470 if (of_device_is_compatible(np, "fsl,sec1.0")) {
9c02e285
LC		 3471 if (priv->num_channels == 1)
3472 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
dd3c0987 3473 (unsigned long)dev);
9c02e285
LC		 3474 else
3475 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3476 (unsigned long)dev);
3477 } else {
3478 if (priv->irq[1]) {
dd3c0987
LC		 3479 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3480 (unsigned long)dev);
3481 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3482 (unsigned long)dev);
9c02e285
LC		 3483 } else if (priv->num_channels == 1) {
3484 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3485 (unsigned long)dev);
3486 } else {
3487 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3488 (unsigned long)dev);
dd3c0987
LC		 3489 }
3490 }
3491
a86854d0
KC		 3492 priv->chan = devm_kcalloc(dev,
3493 priv->num_channels,
3494 sizeof(struct talitos_channel),
3495 GFP_KERNEL);
4b992628
KP		 3496 if (!priv->chan) {
3497 dev_err(dev, "failed to allocate channel management space\n");
9c4a7965
KP		 3498 err = -ENOMEM;
3499 goto err_out;
3500 }
3501
f641dddd
MH		 3502 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3503
c3e337f8 3504 for (i = 0; i < priv->num_channels; i++) {
5fa7fa14 3505 priv->chan[i].reg = priv->reg + stride * (i + 1);
2cdba3cf 3506 if (!priv->irq[1] || !(i & 1))
c3e337f8 3507 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
ad42d5fc 3508
4b992628
KP		 3509 spin_lock_init(&priv->chan[i].head_lock);
3510 spin_lock_init(&priv->chan[i].tail_lock);
9c4a7965 3511
a86854d0
KC		 3512 priv->chan[i].fifo = devm_kcalloc(dev,
3513 priv->fifo_len,
3514 sizeof(struct talitos_request),
3515 GFP_KERNEL);
4b992628 3516 if (!priv->chan[i].fifo) {
9c4a7965
KP		 3517 dev_err(dev, "failed to allocate request fifo %d\n", i);
3518 err = -ENOMEM;
3519 goto err_out;
3520 }
9c4a7965 3521
4b992628
KP		 3522 atomic_set(&priv->chan[i].submit_count,
3523 -(priv->chfifo_len - 1));
f641dddd 3524 }
9c4a7965 3525
81eb024c
KP		 3526 dma_set_mask(dev, DMA_BIT_MASK(36));
3527
9c4a7965
KP		 3528 /* reset and initialize the h/w */
3529 err = init_device(dev);
3530 if (err) {
3531 dev_err(dev, "failed to initialize device\n");
3532 goto err_out;
3533 }
3534
3535 /* register the RNG, if available */
3536 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3537 err = talitos_register_rng(dev);
3538 if (err) {
3539 dev_err(dev, "failed to register hwrng: %d\n", err);
3540 goto err_out;
3541 } else
3542 dev_info(dev, "hwrng\n");
3543 }
3544
3545 /* register crypto algorithms the device supports */
9c4a7965
KP		 3546 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3547 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3548 struct talitos_crypto_alg *t_alg;
aeb4c132 3549 struct crypto_alg *alg = NULL;
9c4a7965
KP		 3550
3551 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3552 if (IS_ERR(t_alg)) {
3553 err = PTR_ERR(t_alg);
0b2730d8 3554 if (err == -ENOTSUPP)
79b3a418 3555 continue;
9c4a7965
KP		 3556 goto err_out;
3557 }
3558
acbf7c62
LN		 3559 switch (t_alg->algt.type) {
3560 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62
LN		 3561 err = crypto_register_alg(
3562 &t_alg->algt.alg.crypto);
aeb4c132 3563 alg = &t_alg->algt.alg.crypto;
acbf7c62 3564 break;
aeb4c132
HX		 3565
3566 case CRYPTO_ALG_TYPE_AEAD:
3567 err = crypto_register_aead(
3568 &t_alg->algt.alg.aead);
3569 alg = &t_alg->algt.alg.aead.base;
3570 break;
3571
acbf7c62
LN		 3572 case CRYPTO_ALG_TYPE_AHASH:
3573 err = crypto_register_ahash(
3574 &t_alg->algt.alg.hash);
aeb4c132 3575 alg = &t_alg->algt.alg.hash.halg.base;
acbf7c62
LN		 3576 break;
3577 }
9c4a7965
KP		 3578 if (err) {
3579 dev_err(dev, "%s alg registration failed\n",
aeb4c132 3580 alg->cra_driver_name);
24b92ff2 3581 devm_kfree(dev, t_alg);
991155ba 3582 } else
9c4a7965 3583 list_add_tail(&t_alg->entry, &priv->alg_list);
9c4a7965
KP		 3584 }
3585 }
5b859b6e
KP		 3586 if (!list_empty(&priv->alg_list))
3587 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3588 (char *)of_get_property(np, "compatible", NULL));
9c4a7965
KP		 3589
3590 return 0;
3591
3592err_out:
3593 talitos_remove(ofdev);
9c4a7965
KP		 3594
3595 return err;
3596}
3597
6c3f975a 3598static const struct of_device_id talitos_match[] = {
0635b7db
LC		 3599#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3600 {
3601 .compatible = "fsl,sec1.0",
3602 },
3603#endif
3604#ifdef CONFIG_CRYPTO_DEV_TALITOS2
9c4a7965
KP		 3605 {
3606 .compatible = "fsl,sec2.0",
3607 },
0635b7db 3608#endif
9c4a7965
KP		 3609 {},
3610};
3611MODULE_DEVICE_TABLE(of, talitos_match);
3612
1c48a5c9 3613static struct platform_driver talitos_driver = {
4018294b
GL		 3614 .driver = {
3615 .name = "talitos",
4018294b
GL		 3616 .of_match_table = talitos_match,
3617 },
9c4a7965 3618 .probe = talitos_probe,
596f1034 3619 .remove = talitos_remove,
9c4a7965
KP		 3620};
3621
741e8c2d 3622module_platform_driver(talitos_driver);
9c4a7965
KP		 3623
3624MODULE_LICENSE("GPL");
3625MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3626MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Linux 6.x block layer and io_uring tree(s)