projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
[linux-2.6-block.git] / drivers / crypto / talitos.c
CommitLineData
9c4a7965
KP		 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
5228f0f7 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
9c4a7965
KP		 5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
5af50730
RH		 35#include <linux/of_address.h>
36#include <linux/of_irq.h>
9c4a7965
KP		 37#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
5a0e3ad6 42#include <linux/slab.h>
9c4a7965
KP		 43
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
3952f17e 46#include <crypto/des.h>
9c4a7965 47#include <crypto/sha.h>
497f2e6b 48#include <crypto/md5.h>
e98014ab 49#include <crypto/internal/aead.h>
9c4a7965 50#include <crypto/authenc.h>
4de9d0b5 51#include <crypto/skcipher.h>
acbf7c62
LN		 52#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
4de9d0b5 54#include <crypto/scatterwalk.h>
9c4a7965
KP		 55
56#include "talitos.h"
57
922f9dc8 58static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
da9de146 59 unsigned int len, bool is_sec1)
81eb024c 60{
edc6bd69 61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
da9de146
LC		 62 if (is_sec1) {
63 ptr->len1 = cpu_to_be16(len);
64 } else {
65 ptr->len = cpu_to_be16(len);
922f9dc8 66 ptr->eptr = upper_32_bits(dma_addr);
da9de146 67 }
81eb024c
KP		 68}
69
340ff60a
HG		 70static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
71 struct talitos_ptr *src_ptr, bool is_sec1)
72{
73 dst_ptr->ptr = src_ptr->ptr;
922f9dc8 74 if (is_sec1) {
da9de146 75 dst_ptr->len1 = src_ptr->len1;
922f9dc8 76 } else {
da9de146
LC		 77 dst_ptr->len = src_ptr->len;
78 dst_ptr->eptr = src_ptr->eptr;
922f9dc8 79 }
538caf83
LC		 80}
81
922f9dc8
LC		 82static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
83 bool is_sec1)
538caf83 84{
922f9dc8
LC		 85 if (is_sec1)
86 return be16_to_cpu(ptr->len1);
87 else
88 return be16_to_cpu(ptr->len);
538caf83
LC		 89}
90
b096b544
LC		 91static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
92 bool is_sec1)
185eb79f 93{
922f9dc8 94 if (!is_sec1)
b096b544
LC		 95 ptr->j_extent = val;
96}
97
98static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
99{
100 if (!is_sec1)
101 ptr->j_extent |= val;
185eb79f
LC		 102}
103
9c4a7965
KP		 104/*
105 * map virtual single (contiguous) pointer to h/w descriptor pointer
106 */
6a4967c3
LC		 107static void __map_single_talitos_ptr(struct device *dev,
108 struct talitos_ptr *ptr,
109 unsigned int len, void *data,
110 enum dma_data_direction dir,
111 unsigned long attrs)
112{
113 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
114 struct talitos_private *priv = dev_get_drvdata(dev);
115 bool is_sec1 = has_ftr_sec1(priv);
116
117 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
118}
119
9c4a7965 120static void map_single_talitos_ptr(struct device *dev,
edc6bd69 121 struct talitos_ptr *ptr,
42e8b0d7 122 unsigned int len, void *data,
9c4a7965
KP		 123 enum dma_data_direction dir)
124{
6a4967c3
LC		 125 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
126}
81eb024c 127
6a4967c3
LC		 128static void map_single_talitos_ptr_nosync(struct device *dev,
129 struct talitos_ptr *ptr,
130 unsigned int len, void *data,
131 enum dma_data_direction dir)
132{
133 __map_single_talitos_ptr(dev, ptr, len, data, dir,
134 DMA_ATTR_SKIP_CPU_SYNC);
9c4a7965
KP		 135}
136
137/*
138 * unmap bus single (contiguous) h/w descriptor pointer
139 */
140static void unmap_single_talitos_ptr(struct device *dev,
edc6bd69 141 struct talitos_ptr *ptr,
9c4a7965
KP		 142 enum dma_data_direction dir)
143{
922f9dc8
LC		 144 struct talitos_private *priv = dev_get_drvdata(dev);
145 bool is_sec1 = has_ftr_sec1(priv);
146
edc6bd69 147 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
922f9dc8 148 from_talitos_ptr_len(ptr, is_sec1), dir);
9c4a7965
KP		 149}
150
151static int reset_channel(struct device *dev, int ch)
152{
153 struct talitos_private *priv = dev_get_drvdata(dev);
154 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 155 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 156
dd3c0987
LC		 157 if (is_sec1) {
158 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
159 TALITOS1_CCCR_LO_RESET);
9c4a7965 160
dd3c0987
LC		 161 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
162 TALITOS1_CCCR_LO_RESET) && --timeout)
163 cpu_relax();
164 } else {
165 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
166 TALITOS2_CCCR_RESET);
167
168 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
169 TALITOS2_CCCR_RESET) && --timeout)
170 cpu_relax();
171 }
9c4a7965
KP		 172
173 if (timeout == 0) {
174 dev_err(dev, "failed to reset channel %d\n", ch);
175 return -EIO;
176 }
177
81eb024c 178 /* set 36-bit addressing, done writeback enable and done IRQ enable */
ad42d5fc 179 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
81eb024c 180 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
37b5e889
LC		 181 /* enable chaining descriptors */
182 if (is_sec1)
183 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
184 TALITOS_CCCR_LO_NE);
9c4a7965 185
fe5720e2
KP		 186 /* and ICCR writeback, if available */
187 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
ad42d5fc 188 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
fe5720e2
KP		 189 TALITOS_CCCR_LO_IWSE);
190
9c4a7965
KP		 191 return 0;
192}
193
194static int reset_device(struct device *dev)
195{
196 struct talitos_private *priv = dev_get_drvdata(dev);
197 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987
LC		 198 bool is_sec1 = has_ftr_sec1(priv);
199 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
9c4a7965 200
c3e337f8 201 setbits32(priv->reg + TALITOS_MCR, mcr);
9c4a7965 202
dd3c0987 203 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
9c4a7965
KP		 204 && --timeout)
205 cpu_relax();
206
2cdba3cf 207 if (priv->irq[1]) {
c3e337f8
KP		 208 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
209 setbits32(priv->reg + TALITOS_MCR, mcr);
210 }
211
9c4a7965
KP		 212 if (timeout == 0) {
213 dev_err(dev, "failed to reset device\n");
214 return -EIO;
215 }
216
217 return 0;
218}
219
220/*
221 * Reset and initialize the device
222 */
223static int init_device(struct device *dev)
224{
225 struct talitos_private *priv = dev_get_drvdata(dev);
226 int ch, err;
dd3c0987 227 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965
KP		 228
229 /*
230 * Master reset
231 * errata documentation: warning: certain SEC interrupts
232 * are not fully cleared by writing the MCR:SWR bit,
233 * set bit twice to completely reset
234 */
235 err = reset_device(dev);
236 if (err)
237 return err;
238
239 err = reset_device(dev);
240 if (err)
241 return err;
242
243 /* reset channels */
244 for (ch = 0; ch < priv->num_channels; ch++) {
245 err = reset_channel(dev, ch);
246 if (err)
247 return err;
248 }
249
250 /* enable channel done and error interrupts */
dd3c0987
LC		 251 if (is_sec1) {
252 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
253 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
254 /* disable parity error check in DEU (erroneous? test vect.) */
255 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
256 } else {
257 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
258 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
259 }
9c4a7965 260
fe5720e2
KP		 261 /* disable integrity check error interrupts (use writeback instead) */
262 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
5fa7fa14 263 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
fe5720e2
KP		 264 TALITOS_MDEUICR_LO_ICE);
265
9c4a7965
KP		 266 return 0;
267}
268
269/**
270 * talitos_submit - submits a descriptor to the device for processing
271 * @dev: the SEC device to be used
5228f0f7 272 * @ch: the SEC device channel to be used
9c4a7965
KP		 273 * @desc: the descriptor to be processed by the device
274 * @callback: whom to call when processing is complete
275 * @context: a handle for use by caller (optional)
276 *
277 * desc must contain valid dma-mapped (bus physical) address pointers.
278 * callback must check err and feedback in descriptor header
279 * for device processing status.
280 */
865d5061
HG		 281int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
282 void (*callback)(struct device *dev,
283 struct talitos_desc *desc,
284 void *context, int error),
285 void *context)
9c4a7965
KP		 286{
287 struct talitos_private *priv = dev_get_drvdata(dev);
288 struct talitos_request *request;
5228f0f7 289 unsigned long flags;
9c4a7965 290 int head;
7d607c6a 291 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 292
4b992628 293 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
9c4a7965 294
4b992628 295 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
ec6644d6 296 /* h/w fifo is full */
4b992628 297 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 298 return -EAGAIN;
299 }
300
4b992628
KP		 301 head = priv->chan[ch].head;
302 request = &priv->chan[ch].fifo[head];
ec6644d6 303
9c4a7965 304 /* map descriptor and save caller data */
7d607c6a
LC		 305 if (is_sec1) {
306 desc->hdr1 = desc->hdr;
7d607c6a
LC		 307 request->dma_desc = dma_map_single(dev, &desc->hdr1,
308 TALITOS_DESC_SIZE,
309 DMA_BIDIRECTIONAL);
310 } else {
311 request->dma_desc = dma_map_single(dev, desc,
312 TALITOS_DESC_SIZE,
313 DMA_BIDIRECTIONAL);
314 }
9c4a7965
KP		 315 request->callback = callback;
316 request->context = context;
317
318 /* increment fifo head */
4b992628 319 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
9c4a7965
KP		 320
321 smp_wmb();
322 request->desc = desc;
323
324 /* GO! */
325 wmb();
ad42d5fc
KP		 326 out_be32(priv->chan[ch].reg + TALITOS_FF,
327 upper_32_bits(request->dma_desc));
328 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
a752447a 329 lower_32_bits(request->dma_desc));
9c4a7965 330
4b992628 331 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 332
333 return -EINPROGRESS;
334}
865d5061 335EXPORT_SYMBOL(talitos_submit);
9c4a7965
KP		 336
337/*
338 * process what was done, notify callback of error if not
339 */
340static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
341{
342 struct talitos_private *priv = dev_get_drvdata(dev);
343 struct talitos_request *request, saved_req;
344 unsigned long flags;
345 int tail, status;
7d607c6a 346 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 347
4b992628 348 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
9c4a7965 349
4b992628
KP		 350 tail = priv->chan[ch].tail;
351 while (priv->chan[ch].fifo[tail].desc) {
7d607c6a
LC		 352 __be32 hdr;
353
4b992628 354 request = &priv->chan[ch].fifo[tail];
9c4a7965
KP		 355
356 /* descriptors with their done bits set don't get the error */
357 rmb();
37b5e889
LC		 358 if (!is_sec1)
359 hdr = request->desc->hdr;
360 else if (request->desc->next_desc)
361 hdr = (request->desc + 1)->hdr1;
362 else
363 hdr = request->desc->hdr1;
7d607c6a
LC		 364
365 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
9c4a7965 366 status = 0;
ca38a814 367 else
9c4a7965
KP		 368 if (!error)
369 break;
370 else
371 status = error;
372
373 dma_unmap_single(dev, request->dma_desc,
7d607c6a 374 TALITOS_DESC_SIZE,
e938e465 375 DMA_BIDIRECTIONAL);
9c4a7965
KP		 376
377 /* copy entries so we can call callback outside lock */
378 saved_req.desc = request->desc;
379 saved_req.callback = request->callback;
380 saved_req.context = request->context;
381
382 /* release request entry in fifo */
383 smp_wmb();
384 request->desc = NULL;
385
386 /* increment fifo tail */
4b992628 387 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
9c4a7965 388
4b992628 389 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
ec6644d6 390
4b992628 391 atomic_dec(&priv->chan[ch].submit_count);
ec6644d6 392
9c4a7965
KP		 393 saved_req.callback(dev, saved_req.desc, saved_req.context,
394 status);
395 /* channel may resume processing in single desc error case */
396 if (error && !reset_ch && status == error)
397 return;
4b992628
KP		 398 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
399 tail = priv->chan[ch].tail;
9c4a7965
KP		 400 }
401
4b992628 402 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
9c4a7965
KP		 403}
404
405/*
406 * process completed requests for channels that have done status
407 */
dd3c0987
LC		 408#define DEF_TALITOS1_DONE(name, ch_done_mask) \
409static void talitos1_done_##name(unsigned long data) \
410{ \
411 struct device *dev = (struct device *)data; \
412 struct talitos_private *priv = dev_get_drvdata(dev); \
413 unsigned long flags; \
414 \
415 if (ch_done_mask & 0x10000000) \
416 flush_channel(dev, 0, 0, 0); \
dd3c0987
LC		 417 if (ch_done_mask & 0x40000000) \
418 flush_channel(dev, 1, 0, 0); \
419 if (ch_done_mask & 0x00010000) \
420 flush_channel(dev, 2, 0, 0); \
421 if (ch_done_mask & 0x00040000) \
422 flush_channel(dev, 3, 0, 0); \
423 \
dd3c0987
LC		 424 /* At this point, all completed channels have been processed */ \
425 /* Unmask done interrupts for channels completed later on. */ \
426 spin_lock_irqsave(&priv->reg_lock, flags); \
427 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
428 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
429 spin_unlock_irqrestore(&priv->reg_lock, flags); \
430}
431
432DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
9c02e285 433DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
dd3c0987
LC		 434
435#define DEF_TALITOS2_DONE(name, ch_done_mask) \
436static void talitos2_done_##name(unsigned long data) \
c3e337f8
KP		 437{ \
438 struct device *dev = (struct device *)data; \
439 struct talitos_private *priv = dev_get_drvdata(dev); \
511d63cb 440 unsigned long flags; \
c3e337f8
KP		 441 \
442 if (ch_done_mask & 1) \
443 flush_channel(dev, 0, 0, 0); \
c3e337f8
KP		 444 if (ch_done_mask & (1 << 2)) \
445 flush_channel(dev, 1, 0, 0); \
446 if (ch_done_mask & (1 << 4)) \
447 flush_channel(dev, 2, 0, 0); \
448 if (ch_done_mask & (1 << 6)) \
449 flush_channel(dev, 3, 0, 0); \
450 \
c3e337f8
KP		 451 /* At this point, all completed channels have been processed */ \
452 /* Unmask done interrupts for channels completed later on. */ \
511d63cb 453 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8 454 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
dd3c0987 455 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
511d63cb 456 spin_unlock_irqrestore(&priv->reg_lock, flags); \
9c4a7965 457}
dd3c0987
LC		 458
459DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
9c02e285 460DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
dd3c0987
LC		 461DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
462DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
9c4a7965
KP		 463
464/*
465 * locate current (offending) descriptor
466 */
3e721aeb 467static u32 current_desc_hdr(struct device *dev, int ch)
9c4a7965
KP		 468{
469 struct talitos_private *priv = dev_get_drvdata(dev);
b62ffd8c 470 int tail, iter;
9c4a7965
KP		 471 dma_addr_t cur_desc;
472
b62ffd8c
HG		 473 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
474 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
9c4a7965 475
b62ffd8c
HG		 476 if (!cur_desc) {
477 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
478 return 0;
479 }
480
481 tail = priv->chan[ch].tail;
482
483 iter = tail;
37b5e889
LC		 484 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
485 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) {
b62ffd8c
HG		 486 iter = (iter + 1) & (priv->fifo_len - 1);
487 if (iter == tail) {
9c4a7965 488 dev_err(dev, "couldn't locate current descriptor\n");
3e721aeb 489 return 0;
9c4a7965
KP		 490 }
491 }
492
37b5e889
LC		 493 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
494 return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
495
b62ffd8c 496 return priv->chan[ch].fifo[iter].desc->hdr;
9c4a7965
KP		 497}
498
499/*
500 * user diagnostics; report root cause of error based on execution unit status
501 */
3e721aeb 502static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
9c4a7965
KP		 503{
504 struct talitos_private *priv = dev_get_drvdata(dev);
505 int i;
506
3e721aeb 507 if (!desc_hdr)
ad42d5fc 508 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
3e721aeb
KP		 509
510 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
9c4a7965
KP		 511 case DESC_HDR_SEL0_AFEU:
512 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 513 in_be32(priv->reg_afeu + TALITOS_EUISR),
514 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
9c4a7965
KP		 515 break;
516 case DESC_HDR_SEL0_DEU:
517 dev_err(dev, "DEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 518 in_be32(priv->reg_deu + TALITOS_EUISR),
519 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
9c4a7965
KP		 520 break;
521 case DESC_HDR_SEL0_MDEUA:
522 case DESC_HDR_SEL0_MDEUB:
523 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 524 in_be32(priv->reg_mdeu + TALITOS_EUISR),
525 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 526 break;
527 case DESC_HDR_SEL0_RNG:
528 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
5fa7fa14
LC		 529 in_be32(priv->reg_rngu + TALITOS_ISR),
530 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
9c4a7965
KP		 531 break;
532 case DESC_HDR_SEL0_PKEU:
533 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 534 in_be32(priv->reg_pkeu + TALITOS_EUISR),
535 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 536 break;
537 case DESC_HDR_SEL0_AESU:
538 dev_err(dev, "AESUISR 0x%08x_%08x\n",
5fa7fa14
LC		 539 in_be32(priv->reg_aesu + TALITOS_EUISR),
540 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
9c4a7965
KP		 541 break;
542 case DESC_HDR_SEL0_CRCU:
543 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 544 in_be32(priv->reg_crcu + TALITOS_EUISR),
545 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 546 break;
547 case DESC_HDR_SEL0_KEU:
548 dev_err(dev, "KEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 549 in_be32(priv->reg_pkeu + TALITOS_EUISR),
550 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 551 break;
552 }
553
3e721aeb 554 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
9c4a7965
KP		 555 case DESC_HDR_SEL1_MDEUA:
556 case DESC_HDR_SEL1_MDEUB:
557 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 558 in_be32(priv->reg_mdeu + TALITOS_EUISR),
559 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 560 break;
561 case DESC_HDR_SEL1_CRCU:
562 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 563 in_be32(priv->reg_crcu + TALITOS_EUISR),
564 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 565 break;
566 }
567
568 for (i = 0; i < 8; i++)
569 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
ad42d5fc
KP		 570 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
571 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
9c4a7965
KP		 572}
573
574/*
575 * recover from error interrupts
576 */
5e718a09 577static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
9c4a7965 578{
9c4a7965
KP		 579 struct talitos_private *priv = dev_get_drvdata(dev);
580 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 581 int ch, error, reset_dev = 0;
42e8b0d7 582 u32 v_lo;
dd3c0987
LC		 583 bool is_sec1 = has_ftr_sec1(priv);
584 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
9c4a7965
KP		 585
586 for (ch = 0; ch < priv->num_channels; ch++) {
587 /* skip channels without errors */
dd3c0987
LC		 588 if (is_sec1) {
589 /* bits 29, 31, 17, 19 */
590 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
591 continue;
592 } else {
593 if (!(isr & (1 << (ch * 2 + 1))))
594 continue;
595 }
9c4a7965
KP		 596
597 error = -EINVAL;
598
ad42d5fc 599 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
9c4a7965
KP		 600
601 if (v_lo & TALITOS_CCPSR_LO_DOF) {
602 dev_err(dev, "double fetch fifo overflow error\n");
603 error = -EAGAIN;
604 reset_ch = 1;
605 }
606 if (v_lo & TALITOS_CCPSR_LO_SOF) {
607 /* h/w dropped descriptor */
608 dev_err(dev, "single fetch fifo overflow error\n");
609 error = -EAGAIN;
610 }
611 if (v_lo & TALITOS_CCPSR_LO_MDTE)
612 dev_err(dev, "master data transfer error\n");
613 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
4d9b3a5b 614 dev_err(dev, is_sec1 ? "pointer not complete error\n"
dd3c0987 615 : "s/g data length zero error\n");
9c4a7965 616 if (v_lo & TALITOS_CCPSR_LO_FPZ)
dd3c0987
LC		 617 dev_err(dev, is_sec1 ? "parity error\n"
618 : "fetch pointer zero error\n");
9c4a7965
KP		 619 if (v_lo & TALITOS_CCPSR_LO_IDH)
620 dev_err(dev, "illegal descriptor header error\n");
621 if (v_lo & TALITOS_CCPSR_LO_IEU)
dd3c0987
LC		 622 dev_err(dev, is_sec1 ? "static assignment error\n"
623 : "invalid exec unit error\n");
9c4a7965 624 if (v_lo & TALITOS_CCPSR_LO_EU)
3e721aeb 625 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
dd3c0987
LC		 626 if (!is_sec1) {
627 if (v_lo & TALITOS_CCPSR_LO_GB)
628 dev_err(dev, "gather boundary error\n");
629 if (v_lo & TALITOS_CCPSR_LO_GRL)
630 dev_err(dev, "gather return/length error\n");
631 if (v_lo & TALITOS_CCPSR_LO_SB)
632 dev_err(dev, "scatter boundary error\n");
633 if (v_lo & TALITOS_CCPSR_LO_SRL)
634 dev_err(dev, "scatter return/length error\n");
635 }
9c4a7965
KP		 636
637 flush_channel(dev, ch, error, reset_ch);
638
639 if (reset_ch) {
640 reset_channel(dev, ch);
641 } else {
ad42d5fc 642 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
dd3c0987 643 TALITOS2_CCCR_CONT);
ad42d5fc
KP		 644 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
645 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
dd3c0987 646 TALITOS2_CCCR_CONT) && --timeout)
9c4a7965
KP		 647 cpu_relax();
648 if (timeout == 0) {
649 dev_err(dev, "failed to restart channel %d\n",
650 ch);
651 reset_dev = 1;
652 }
653 }
654 }
dd3c0987
LC		 655 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
656 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
657 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
658 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
659 isr, isr_lo);
660 else
661 dev_err(dev, "done overflow, internal time out, or "
662 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
9c4a7965
KP		 663
664 /* purge request queues */
665 for (ch = 0; ch < priv->num_channels; ch++)
666 flush_channel(dev, ch, -EIO, 1);
667
668 /* reset and reinitialize the device */
669 init_device(dev);
670 }
671}
672
dd3c0987
LC		 673#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
674static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
675{ \
676 struct device *dev = data; \
677 struct talitos_private *priv = dev_get_drvdata(dev); \
678 u32 isr, isr_lo; \
679 unsigned long flags; \
680 \
681 spin_lock_irqsave(&priv->reg_lock, flags); \
682 isr = in_be32(priv->reg + TALITOS_ISR); \
683 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
684 /* Acknowledge interrupt */ \
685 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
686 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
687 \
688 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
689 spin_unlock_irqrestore(&priv->reg_lock, flags); \
690 talitos_error(dev, isr & ch_err_mask, isr_lo); \
691 } \
692 else { \
693 if (likely(isr & ch_done_mask)) { \
694 /* mask further done interrupts. */ \
695 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
696 /* done_task will unmask done interrupts at exit */ \
697 tasklet_schedule(&priv->done_task[tlet]); \
698 } \
699 spin_unlock_irqrestore(&priv->reg_lock, flags); \
700 } \
701 \
702 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
703 IRQ_NONE; \
704}
705
706DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
707
708#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
709static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
c3e337f8
KP		 710{ \
711 struct device *dev = data; \
712 struct talitos_private *priv = dev_get_drvdata(dev); \
713 u32 isr, isr_lo; \
511d63cb 714 unsigned long flags; \
c3e337f8 715 \
511d63cb 716 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8
KP		 717 isr = in_be32(priv->reg + TALITOS_ISR); \
718 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
719 /* Acknowledge interrupt */ \
720 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
721 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
722 \
511d63cb
HG		 723 if (unlikely(isr & ch_err_mask || isr_lo)) { \
724 spin_unlock_irqrestore(&priv->reg_lock, flags); \
725 talitos_error(dev, isr & ch_err_mask, isr_lo); \
726 } \
727 else { \
c3e337f8
KP		 728 if (likely(isr & ch_done_mask)) { \
729 /* mask further done interrupts. */ \
730 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
731 /* done_task will unmask done interrupts at exit */ \
732 tasklet_schedule(&priv->done_task[tlet]); \
733 } \
511d63cb
HG		 734 spin_unlock_irqrestore(&priv->reg_lock, flags); \
735 } \
c3e337f8
KP		 736 \
737 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
738 IRQ_NONE; \
9c4a7965 739}
dd3c0987
LC		 740
741DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
742DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
743 0)
744DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
745 1)
9c4a7965
KP		 746
747/*
748 * hwrng
749 */
750static int talitos_rng_data_present(struct hwrng *rng, int wait)
751{
752 struct device *dev = (struct device *)rng->priv;
753 struct talitos_private *priv = dev_get_drvdata(dev);
754 u32 ofl;
755 int i;
756
757 for (i = 0; i < 20; i++) {
5fa7fa14 758 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
9c4a7965
KP		 759 TALITOS_RNGUSR_LO_OFL;
760 if (ofl || !wait)
761 break;
762 udelay(10);
763 }
764
765 return !!ofl;
766}
767
768static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
769{
770 struct device *dev = (struct device *)rng->priv;
771 struct talitos_private *priv = dev_get_drvdata(dev);
772
773 /* rng fifo requires 64-bit accesses */
5fa7fa14
LC		 774 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
775 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
9c4a7965
KP		 776
777 return sizeof(u32);
778}
779
780static int talitos_rng_init(struct hwrng *rng)
781{
782 struct device *dev = (struct device *)rng->priv;
783 struct talitos_private *priv = dev_get_drvdata(dev);
784 unsigned int timeout = TALITOS_TIMEOUT;
785
5fa7fa14
LC		 786 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
787 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
788 & TALITOS_RNGUSR_LO_RD)
9c4a7965
KP		 789 && --timeout)
790 cpu_relax();
791 if (timeout == 0) {
792 dev_err(dev, "failed to reset rng hw\n");
793 return -ENODEV;
794 }
795
796 /* start generating */
5fa7fa14 797 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
9c4a7965
KP		 798
799 return 0;
800}
801
802static int talitos_register_rng(struct device *dev)
803{
804 struct talitos_private *priv = dev_get_drvdata(dev);
35a3bb3d 805 int err;
9c4a7965
KP		 806
807 priv->rng.name = dev_driver_string(dev),
808 priv->rng.init = talitos_rng_init,
809 priv->rng.data_present = talitos_rng_data_present,
810 priv->rng.data_read = talitos_rng_data_read,
811 priv->rng.priv = (unsigned long)dev;
812
35a3bb3d
AS		 813 err = hwrng_register(&priv->rng);
814 if (!err)
815 priv->rng_registered = true;
816
817 return err;
9c4a7965
KP		 818}
819
820static void talitos_unregister_rng(struct device *dev)
821{
822 struct talitos_private *priv = dev_get_drvdata(dev);
823
35a3bb3d
AS		 824 if (!priv->rng_registered)
825 return;
826
9c4a7965 827 hwrng_unregister(&priv->rng);
35a3bb3d 828 priv->rng_registered = false;
9c4a7965
KP		 829}
830
831/*
832 * crypto alg
833 */
834#define TALITOS_CRA_PRIORITY 3000
7405c8d7
LC		 835/*
836 * Defines a priority for doing AEAD with descriptors type
837 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
838 */
839#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
03d2c511 840#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
3952f17e 841#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
70bcaca7 842
9c4a7965
KP		 843struct talitos_ctx {
844 struct device *dev;
5228f0f7 845 int ch;
9c4a7965
KP		 846 __be32 desc_hdr_template;
847 u8 key[TALITOS_MAX_KEY_SIZE];
70bcaca7 848 u8 iv[TALITOS_MAX_IV_LENGTH];
2e13ce08 849 dma_addr_t dma_key;
9c4a7965
KP		 850 unsigned int keylen;
851 unsigned int enckeylen;
852 unsigned int authkeylen;
9c4a7965
KP		 853};
854
497f2e6b
LN		 855#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
856#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
857
858struct talitos_ahash_req_ctx {
60f208d7 859 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
497f2e6b 860 unsigned int hw_context_size;
3c0dd190
LC		 861 u8 buf[2][HASH_MAX_BLOCK_SIZE];
862 int buf_idx;
60f208d7 863 unsigned int swinit;
497f2e6b
LN		 864 unsigned int first;
865 unsigned int last;
866 unsigned int to_hash_later;
42e8b0d7 867 unsigned int nbuf;
497f2e6b
LN		 868 struct scatterlist bufsl[2];
869 struct scatterlist *psrc;
870};
871
3639ca84
HG		 872struct talitos_export_state {
873 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
874 u8 buf[HASH_MAX_BLOCK_SIZE];
875 unsigned int swinit;
876 unsigned int first;
877 unsigned int last;
878 unsigned int to_hash_later;
879 unsigned int nbuf;
880};
881
56af8cd4
LN		 882static int aead_setkey(struct crypto_aead *authenc,
883 const u8 *key, unsigned int keylen)
9c4a7965
KP		 884{
885 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
2e13ce08 886 struct device *dev = ctx->dev;
c306a98d 887 struct crypto_authenc_keys keys;
9c4a7965 888
c306a98d 889 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
9c4a7965
KP		 890 goto badkey;
891
c306a98d 892 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
9c4a7965
KP		 893 goto badkey;
894
2e13ce08
LC		 895 if (ctx->keylen)
896 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
897
c306a98d
MK		 898 memcpy(ctx->key, keys.authkey, keys.authkeylen);
899 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
9c4a7965 900
c306a98d
MK		 901 ctx->keylen = keys.authkeylen + keys.enckeylen;
902 ctx->enckeylen = keys.enckeylen;
903 ctx->authkeylen = keys.authkeylen;
2e13ce08
LC		 904 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
905 DMA_TO_DEVICE);
9c4a7965 906
8f0691fc 907 memzero_explicit(&keys, sizeof(keys));
9c4a7965
KP		 908 return 0;
909
910badkey:
911 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
8f0691fc 912 memzero_explicit(&keys, sizeof(keys));
9c4a7965
KP		 913 return -EINVAL;
914}
915
916/*
56af8cd4 917 * talitos_edesc - s/w-extended descriptor
9c4a7965
KP		 918 * @src_nents: number of segments in input scatterlist
919 * @dst_nents: number of segments in output scatterlist
aeb4c132 920 * @icv_ool: whether ICV is out-of-line
79fd31d3 921 * @iv_dma: dma address of iv for checking continuity and link table
9c4a7965 922 * @dma_len: length of dma mapped link_tbl space
6f65f6ac 923 * @dma_link_tbl: bus physical address of link_tbl/buf
9c4a7965 924 * @desc: h/w descriptor
6f65f6ac
LC		 925 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
926 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
9c4a7965
KP		 927 *
928 * if decrypting (with authcheck), or either one of src_nents or dst_nents
929 * is greater than 1, an integrity check value is concatenated to the end
930 * of link_tbl data
931 */
56af8cd4 932struct talitos_edesc {
9c4a7965
KP		 933 int src_nents;
934 int dst_nents;
aeb4c132 935 bool icv_ool;
79fd31d3 936 dma_addr_t iv_dma;
9c4a7965
KP		 937 int dma_len;
938 dma_addr_t dma_link_tbl;
939 struct talitos_desc desc;
6f65f6ac
LC		 940 union {
941 struct talitos_ptr link_tbl[0];
942 u8 buf[0];
943 };
9c4a7965
KP		 944};
945
4de9d0b5
LN		 946static void talitos_sg_unmap(struct device *dev,
947 struct talitos_edesc *edesc,
948 struct scatterlist *src,
6a1e8d14
LC		 949 struct scatterlist *dst,
950 unsigned int len, unsigned int offset)
4de9d0b5 951{
6a1e8d14
LC		 952 struct talitos_private *priv = dev_get_drvdata(dev);
953 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 954 unsigned int src_nents = edesc->src_nents ? : 1;
955 unsigned int dst_nents = edesc->dst_nents ? : 1;
956
6a1e8d14
LC		 957 if (is_sec1 && dst && dst_nents > 1) {
958 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
959 len, DMA_FROM_DEVICE);
960 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
961 offset);
962 }
4de9d0b5 963 if (src != dst) {
6a1e8d14
LC		 964 if (src_nents == 1 || !is_sec1)
965 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
4de9d0b5 966
6a1e8d14 967 if (dst && (dst_nents == 1 || !is_sec1))
b8a011d4 968 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
6a1e8d14 969 } else if (src_nents == 1 || !is_sec1) {
b8a011d4 970 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
246a87cd
LC		 971 }
972}
973
9c4a7965 974static void ipsec_esp_unmap(struct device *dev,
56af8cd4 975 struct talitos_edesc *edesc,
9c4a7965
KP		 976 struct aead_request *areq)
977{
549bd8bc
LC		 978 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
979 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
980 unsigned int ivsize = crypto_aead_ivsize(aead);
9a655608
LC		 981 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
982 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
549bd8bc 983
9a655608 984 if (is_ipsec_esp)
549bd8bc
LC		 985 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
986 DMA_FROM_DEVICE);
9a655608 987 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
9c4a7965 988
6a1e8d14
LC		 989 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
990 areq->assoclen);
9c4a7965
KP		 991
992 if (edesc->dma_len)
993 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
994 DMA_BIDIRECTIONAL);
549bd8bc 995
9a655608 996 if (!is_ipsec_esp) {
549bd8bc
LC		 997 unsigned int dst_nents = edesc->dst_nents ? : 1;
998
999 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1000 areq->assoclen + areq->cryptlen - ivsize);
1001 }
9c4a7965
KP		 1002}
1003
1004/*
1005 * ipsec_esp descriptor callbacks
1006 */
1007static void ipsec_esp_encrypt_done(struct device *dev,
1008 struct talitos_desc *desc, void *context,
1009 int err)
1010{
549bd8bc
LC		 1011 struct talitos_private *priv = dev_get_drvdata(dev);
1012 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 1013 struct aead_request *areq = context;
9c4a7965 1014 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 1015 unsigned int authsize = crypto_aead_authsize(authenc);
2e13ce08 1016 unsigned int ivsize = crypto_aead_ivsize(authenc);
19bbbc63 1017 struct talitos_edesc *edesc;
9c4a7965
KP		 1018 struct scatterlist *sg;
1019 void *icvdata;
1020
19bbbc63
KP		 1021 edesc = container_of(desc, struct talitos_edesc, desc);
1022
9c4a7965
KP		 1023 ipsec_esp_unmap(dev, edesc, areq);
1024
1025 /* copy the generated ICV to dst */
aeb4c132 1026 if (edesc->icv_ool) {
549bd8bc
LC		 1027 if (is_sec1)
1028 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
1029 else
1030 icvdata = &edesc->link_tbl[edesc->src_nents +
1031 edesc->dst_nents + 2];
9c4a7965 1032 sg = sg_last(areq->dst, edesc->dst_nents);
aeb4c132
HX		 1033 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1034 icvdata, authsize);
9c4a7965
KP		 1035 }
1036
2e13ce08
LC		 1037 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1038
9c4a7965
KP		 1039 kfree(edesc);
1040
1041 aead_request_complete(areq, err);
1042}
1043
fe5720e2 1044static void ipsec_esp_decrypt_swauth_done(struct device *dev,
e938e465
KP		 1045 struct talitos_desc *desc,
1046 void *context, int err)
9c4a7965
KP		 1047{
1048 struct aead_request *req = context;
9c4a7965 1049 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1050 unsigned int authsize = crypto_aead_authsize(authenc);
19bbbc63 1051 struct talitos_edesc *edesc;
9c4a7965 1052 struct scatterlist *sg;
aeb4c132 1053 char *oicv, *icv;
549bd8bc
LC		 1054 struct talitos_private *priv = dev_get_drvdata(dev);
1055 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 1056
19bbbc63
KP		 1057 edesc = container_of(desc, struct talitos_edesc, desc);
1058
9c4a7965
KP		 1059 ipsec_esp_unmap(dev, edesc, req);
1060
1061 if (!err) {
1062 /* auth check */
9c4a7965 1063 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
aeb4c132
HX		 1064 icv = (char *)sg_virt(sg) + sg->length - authsize;
1065
1066 if (edesc->dma_len) {
549bd8bc
LC		 1067 if (is_sec1)
1068 oicv = (char *)&edesc->dma_link_tbl +
1069 req->assoclen + req->cryptlen;
1070 else
1071 oicv = (char *)
1072 &edesc->link_tbl[edesc->src_nents +
aeb4c132
HX		 1073 edesc->dst_nents + 2];
1074 if (edesc->icv_ool)
1075 icv = oicv + authsize;
1076 } else
1077 oicv = (char *)&edesc->link_tbl[0];
1078
79960943 1079 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
9c4a7965
KP		 1080 }
1081
1082 kfree(edesc);
1083
1084 aead_request_complete(req, err);
1085}
1086
fe5720e2 1087static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
e938e465
KP		 1088 struct talitos_desc *desc,
1089 void *context, int err)
fe5720e2
KP		 1090{
1091 struct aead_request *req = context;
19bbbc63
KP		 1092 struct talitos_edesc *edesc;
1093
1094 edesc = container_of(desc, struct talitos_edesc, desc);
fe5720e2
KP		 1095
1096 ipsec_esp_unmap(dev, edesc, req);
1097
1098 /* check ICV auth status */
e938e465
KP		 1099 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1100 DESC_HDR_LO_ICCR1_PASS))
1101 err = -EBADMSG;
fe5720e2
KP		 1102
1103 kfree(edesc);
1104
1105 aead_request_complete(req, err);
1106}
1107
9c4a7965
KP		 1108/*
1109 * convert scatterlist to SEC h/w link table format
1110 * stop at cryptlen bytes
1111 */
aeb4c132
HX		 1112static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1113 unsigned int offset, int cryptlen,
1114 struct talitos_ptr *link_tbl_ptr)
9c4a7965 1115{
70bcaca7 1116 int n_sg = sg_count;
aeb4c132 1117 int count = 0;
70bcaca7 1118
aeb4c132
HX		 1119 while (cryptlen && sg && n_sg--) {
1120 unsigned int len = sg_dma_len(sg);
9c4a7965 1121
aeb4c132
HX		 1122 if (offset >= len) {
1123 offset -= len;
1124 goto next;
1125 }
1126
1127 len -= offset;
1128
1129 if (len > cryptlen)
1130 len = cryptlen;
1131
1132 to_talitos_ptr(link_tbl_ptr + count,
da9de146 1133 sg_dma_address(sg) + offset, len, 0);
b096b544 1134 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
aeb4c132
HX		 1135 count++;
1136 cryptlen -= len;
1137 offset = 0;
1138
1139next:
1140 sg = sg_next(sg);
70bcaca7 1141 }
9c4a7965
KP		 1142
1143 /* tag end of link table */
aeb4c132 1144 if (count > 0)
b096b544
LC		 1145 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1146 DESC_PTR_LNKTBL_RETURN, 0);
70bcaca7 1147
aeb4c132
HX		 1148 return count;
1149}
1150
2b122730
LC		 1151static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1152 unsigned int len, struct talitos_edesc *edesc,
1153 struct talitos_ptr *ptr, int sg_count,
1154 unsigned int offset, int tbl_off, int elen)
246a87cd 1155{
246a87cd
LC		 1156 struct talitos_private *priv = dev_get_drvdata(dev);
1157 bool is_sec1 = has_ftr_sec1(priv);
1158
87a81dce
LC		 1159 if (!src) {
1160 to_talitos_ptr(ptr, 0, 0, is_sec1);
1161 return 1;
1162 }
2b122730 1163 to_talitos_ptr_ext_set(ptr, elen, is_sec1);
6a1e8d14 1164 if (sg_count == 1) {
da9de146 1165 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
6a1e8d14 1166 return sg_count;
246a87cd 1167 }
246a87cd 1168 if (is_sec1) {
da9de146 1169 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
6a1e8d14 1170 return sg_count;
246a87cd 1171 }
2b122730 1172 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen,
6a1e8d14
LC		 1173 &edesc->link_tbl[tbl_off]);
1174 if (sg_count == 1) {
1175 /* Only one segment now, so no link tbl needed*/
1176 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1177 return sg_count;
1178 }
1179 to_talitos_ptr(ptr, edesc->dma_link_tbl +
da9de146 1180 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
6a1e8d14
LC		 1181 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1182
1183 return sg_count;
246a87cd
LC		 1184}
1185
2b122730
LC		 1186static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1187 unsigned int len, struct talitos_edesc *edesc,
1188 struct talitos_ptr *ptr, int sg_count,
1189 unsigned int offset, int tbl_off)
1190{
1191 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1192 tbl_off, 0);
1193}
1194
9c4a7965
KP		 1195/*
1196 * fill in and submit ipsec_esp descriptor
1197 */
56af8cd4 1198static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
aeb4c132
HX		 1199 void (*callback)(struct device *dev,
1200 struct talitos_desc *desc,
1201 void *context, int error))
9c4a7965
KP		 1202{
1203 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
aeb4c132 1204 unsigned int authsize = crypto_aead_authsize(aead);
9c4a7965
KP		 1205 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1206 struct device *dev = ctx->dev;
1207 struct talitos_desc *desc = &edesc->desc;
1208 unsigned int cryptlen = areq->cryptlen;
e41256f1 1209 unsigned int ivsize = crypto_aead_ivsize(aead);
aeb4c132 1210 int tbl_off = 0;
fa86a267 1211 int sg_count, ret;
2b122730 1212 int elen = 0;
549bd8bc
LC		 1213 bool sync_needed = false;
1214 struct talitos_private *priv = dev_get_drvdata(dev);
1215 bool is_sec1 = has_ftr_sec1(priv);
9a655608
LC		 1216 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1217 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1218 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
9c4a7965
KP		 1219
1220 /* hmac key */
2e13ce08 1221 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
79fd31d3 1222
549bd8bc
LC		 1223 sg_count = edesc->src_nents ?: 1;
1224 if (is_sec1 && sg_count > 1)
1225 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1226 areq->assoclen + cryptlen);
1227 else
1228 sg_count = dma_map_sg(dev, areq->src, sg_count,
1229 (areq->src == areq->dst) ?
1230 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
79fd31d3 1231
549bd8bc
LC		 1232 /* hmac data */
1233 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1234 &desc->ptr[1], sg_count, 0, tbl_off);
340ff60a 1235
549bd8bc 1236 if (ret > 1) {
340ff60a 1237 tbl_off += ret;
549bd8bc 1238 sync_needed = true;
79fd31d3
HG		 1239 }
1240
9c4a7965 1241 /* cipher iv */
9a655608 1242 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
9c4a7965
KP		 1243
1244 /* cipher key */
2e13ce08
LC		 1245 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1246 ctx->enckeylen, is_sec1);
9c4a7965
KP		 1247
1248 /*
1249 * cipher in
1250 * map and adjust cipher len to aead request cryptlen.
1251 * extent is bytes of HMAC postpended to ciphertext,
1252 * typically 12 for ipsec
1253 */
2b122730
LC		 1254 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1255 elen = authsize;
9c4a7965 1256
2b122730
LC		 1257 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1258 sg_count, areq->assoclen, tbl_off, elen);
549bd8bc 1259
ec8c7d14
LC		 1260 if (ret > 1) {
1261 tbl_off += ret;
549bd8bc
LC		 1262 sync_needed = true;
1263 }
9c4a7965 1264
549bd8bc
LC		 1265 /* cipher out */
1266 if (areq->src != areq->dst) {
1267 sg_count = edesc->dst_nents ? : 1;
1268 if (!is_sec1 || sg_count == 1)
1269 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1270 }
9c4a7965 1271
e04a61be
LC		 1272 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1273 sg_count, areq->assoclen, tbl_off);
aeb4c132 1274
9a655608 1275 if (is_ipsec_esp)
549bd8bc 1276 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
aeb4c132 1277
e04a61be
LC		 1278 /* ICV data */
1279 if (ret > 1) {
1280 tbl_off += ret;
aeb4c132 1281 edesc->icv_ool = true;
549bd8bc
LC		 1282 sync_needed = true;
1283
9a655608 1284 if (is_ipsec_esp) {
549bd8bc
LC		 1285 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1286 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1287 sizeof(struct talitos_ptr) + authsize;
1288
1289 /* Add an entry to the link table for ICV data */
e04a61be 1290 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
549bd8bc
LC		 1291 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1292 is_sec1);
549bd8bc
LC		 1293
1294 /* icv data follows link tables */
1295 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
da9de146 1296 authsize, is_sec1);
e04a61be
LC		 1297 } else {
1298 dma_addr_t addr = edesc->dma_link_tbl;
1299
1300 if (is_sec1)
1301 addr += areq->assoclen + cryptlen;
1302 else
1303 addr += sizeof(struct talitos_ptr) * tbl_off;
1304
da9de146 1305 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
e04a61be 1306 }
9a655608 1307 } else if (!is_ipsec_esp) {
e04a61be
LC		 1308 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1309 &desc->ptr[6], sg_count, areq->assoclen +
1310 cryptlen,
1311 tbl_off);
1312 if (ret > 1) {
1313 tbl_off += ret;
1314 edesc->icv_ool = true;
1315 sync_needed = true;
1316 } else {
1317 edesc->icv_ool = false;
549bd8bc 1318 }
340ff60a 1319 } else {
549bd8bc
LC		 1320 edesc->icv_ool = false;
1321 }
1322
9c4a7965 1323 /* iv out */
9a655608 1324 if (is_ipsec_esp)
549bd8bc
LC		 1325 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1326 DMA_FROM_DEVICE);
1327
1328 if (sync_needed)
1329 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1330 edesc->dma_len,
1331 DMA_BIDIRECTIONAL);
9c4a7965 1332
5228f0f7 1333 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
fa86a267
KP		 1334 if (ret != -EINPROGRESS) {
1335 ipsec_esp_unmap(dev, edesc, areq);
1336 kfree(edesc);
1337 }
1338 return ret;
9c4a7965
KP		 1339}
1340
9c4a7965 1341/*
56af8cd4 1342 * allocate and map the extended descriptor
9c4a7965 1343 */
4de9d0b5
LN		 1344static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1345 struct scatterlist *src,
1346 struct scatterlist *dst,
79fd31d3
HG		 1347 u8 *iv,
1348 unsigned int assoclen,
4de9d0b5
LN		 1349 unsigned int cryptlen,
1350 unsigned int authsize,
79fd31d3 1351 unsigned int ivsize,
4de9d0b5 1352 int icv_stashing,
62293a37
HG		 1353 u32 cryptoflags,
1354 bool encrypt)
9c4a7965 1355{
56af8cd4 1356 struct talitos_edesc *edesc;
6a1e8d14 1357 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
79fd31d3 1358 dma_addr_t iv_dma = 0;
4de9d0b5 1359 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
586725f8 1360 GFP_ATOMIC;
6f65f6ac
LC		 1361 struct talitos_private *priv = dev_get_drvdata(dev);
1362 bool is_sec1 = has_ftr_sec1(priv);
1363 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
8e409fe1 1364 void *err;
9c4a7965 1365
6f65f6ac 1366 if (cryptlen + authsize > max_len) {
4de9d0b5 1367 dev_err(dev, "length exceeds h/w max limit\n");
9c4a7965
KP		 1368 return ERR_PTR(-EINVAL);
1369 }
1370
935e99a3 1371 if (ivsize)
79fd31d3
HG		 1372 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1373
62293a37 1374 if (!dst || dst == src) {
6a1e8d14
LC		 1375 src_len = assoclen + cryptlen + authsize;
1376 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1377 if (src_nents < 0) {
1378 dev_err(dev, "Invalid number of src SG.\n");
1379 err = ERR_PTR(-EINVAL);
1380 goto error_sg;
1381 }
62293a37
HG		 1382 src_nents = (src_nents == 1) ? 0 : src_nents;
1383 dst_nents = dst ? src_nents : 0;
6a1e8d14 1384 dst_len = 0;
62293a37 1385 } else { /* dst && dst != src*/
6a1e8d14
LC		 1386 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1387 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1388 if (src_nents < 0) {
1389 dev_err(dev, "Invalid number of src SG.\n");
1390 err = ERR_PTR(-EINVAL);
1391 goto error_sg;
1392 }
62293a37 1393 src_nents = (src_nents == 1) ? 0 : src_nents;
6a1e8d14
LC		 1394 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1395 dst_nents = sg_nents_for_len(dst, dst_len);
8e409fe1
LC		 1396 if (dst_nents < 0) {
1397 dev_err(dev, "Invalid number of dst SG.\n");
1398 err = ERR_PTR(-EINVAL);
1399 goto error_sg;
1400 }
62293a37 1401 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
9c4a7965
KP		 1402 }
1403
1404 /*
1405 * allocate space for base edesc plus the link tables,
aeb4c132
HX		 1406 * allowing for two separate entries for AD and generated ICV (+ 2),
1407 * and space for two sets of ICVs (stashed and generated)
9c4a7965 1408 */
56af8cd4 1409 alloc_len = sizeof(struct talitos_edesc);
aeb4c132 1410 if (src_nents || dst_nents) {
6f65f6ac 1411 if (is_sec1)
6a1e8d14
LC		 1412 dma_len = (src_nents ? src_len : 0) +
1413 (dst_nents ? dst_len : 0);
6f65f6ac 1414 else
aeb4c132
HX		 1415 dma_len = (src_nents + dst_nents + 2) *
1416 sizeof(struct talitos_ptr) + authsize * 2;
9c4a7965
KP		 1417 alloc_len += dma_len;
1418 } else {
1419 dma_len = 0;
4de9d0b5 1420 alloc_len += icv_stashing ? authsize : 0;
9c4a7965
KP		 1421 }
1422
37b5e889
LC		 1423 /* if its a ahash, add space for a second desc next to the first one */
1424 if (is_sec1 && !dst)
1425 alloc_len += sizeof(struct talitos_desc);
1426
586725f8 1427 edesc = kmalloc(alloc_len, GFP_DMA | flags);
9c4a7965 1428 if (!edesc) {
8e409fe1
LC		 1429 err = ERR_PTR(-ENOMEM);
1430 goto error_sg;
9c4a7965 1431 }
e4a647c4 1432 memset(&edesc->desc, 0, sizeof(edesc->desc));
9c4a7965
KP		 1433
1434 edesc->src_nents = src_nents;
1435 edesc->dst_nents = dst_nents;
79fd31d3 1436 edesc->iv_dma = iv_dma;
9c4a7965 1437 edesc->dma_len = dma_len;
37b5e889
LC		 1438 if (dma_len) {
1439 void *addr = &edesc->link_tbl[0];
1440
1441 if (is_sec1 && !dst)
1442 addr += sizeof(struct talitos_desc);
1443 edesc->dma_link_tbl = dma_map_single(dev, addr,
497f2e6b
LN		 1444 edesc->dma_len,
1445 DMA_BIDIRECTIONAL);
37b5e889 1446 }
9c4a7965 1447 return edesc;
8e409fe1
LC		 1448error_sg:
1449 if (iv_dma)
1450 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1451 return err;
9c4a7965
KP		 1452}
1453
79fd31d3 1454static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
62293a37 1455 int icv_stashing, bool encrypt)
4de9d0b5
LN		 1456{
1457 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 1458 unsigned int authsize = crypto_aead_authsize(authenc);
4de9d0b5 1459 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
79fd31d3 1460 unsigned int ivsize = crypto_aead_ivsize(authenc);
4de9d0b5 1461
aeb4c132 1462 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1463 iv, areq->assoclen, areq->cryptlen,
aeb4c132 1464 authsize, ivsize, icv_stashing,
62293a37 1465 areq->base.flags, encrypt);
4de9d0b5
LN		 1466}
1467
56af8cd4 1468static int aead_encrypt(struct aead_request *req)
9c4a7965
KP		 1469{
1470 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1471 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
56af8cd4 1472 struct talitos_edesc *edesc;
9c4a7965
KP		 1473
1474 /* allocate extended descriptor */
62293a37 1475 edesc = aead_edesc_alloc(req, req->iv, 0, true);
9c4a7965
KP		 1476 if (IS_ERR(edesc))
1477 return PTR_ERR(edesc);
1478
1479 /* set encrypt */
70bcaca7 1480 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965 1481
aeb4c132 1482 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
9c4a7965
KP		 1483}
1484
56af8cd4 1485static int aead_decrypt(struct aead_request *req)
9c4a7965
KP		 1486{
1487 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1488 unsigned int authsize = crypto_aead_authsize(authenc);
9c4a7965 1489 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
fe5720e2 1490 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
56af8cd4 1491 struct talitos_edesc *edesc;
9c4a7965
KP		 1492 struct scatterlist *sg;
1493 void *icvdata;
1494
1495 req->cryptlen -= authsize;
1496
1497 /* allocate extended descriptor */
62293a37 1498 edesc = aead_edesc_alloc(req, req->iv, 1, false);
9c4a7965
KP		 1499 if (IS_ERR(edesc))
1500 return PTR_ERR(edesc);
1501
fe5720e2 1502 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
e938e465
KP		 1503 ((!edesc->src_nents && !edesc->dst_nents) ||
1504 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
9c4a7965 1505
fe5720e2 1506 /* decrypt and check the ICV */
e938e465
KP		 1507 edesc->desc.hdr = ctx->desc_hdr_template |
1508 DESC_HDR_DIR_INBOUND |
fe5720e2 1509 DESC_HDR_MODE1_MDEU_CICV;
9c4a7965 1510
fe5720e2 1511 /* reset integrity check result bits */
9c4a7965 1512
aeb4c132 1513 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
e938e465 1514 }
fe5720e2 1515
e938e465
KP		 1516 /* Have to check the ICV with software */
1517 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
fe5720e2 1518
e938e465
KP		 1519 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1520 if (edesc->dma_len)
aeb4c132
HX		 1521 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1522 edesc->dst_nents + 2];
e938e465
KP		 1523 else
1524 icvdata = &edesc->link_tbl[0];
fe5720e2 1525
e938e465 1526 sg = sg_last(req->src, edesc->src_nents ? : 1);
fe5720e2 1527
aeb4c132 1528 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
9c4a7965 1529
aeb4c132 1530 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
9c4a7965
KP		 1531}
1532
4de9d0b5
LN		 1533static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1534 const u8 *key, unsigned int keylen)
1535{
1536 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
2e13ce08 1537 struct device *dev = ctx->dev;
f384cdc4 1538 u32 tmp[DES_EXPKEY_WORDS];
4de9d0b5 1539
03d2c511
MH		 1540 if (keylen > TALITOS_MAX_KEY_SIZE) {
1541 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1542 return -EINVAL;
1543 }
1544
f384cdc4
LC		 1545 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1546 CRYPTO_TFM_REQ_WEAK_KEY) &&
1547 !des_ekey(tmp, key)) {
1548 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1549 return -EINVAL;
1550 }
1551
2e13ce08
LC		 1552 if (ctx->keylen)
1553 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1554
4de9d0b5
LN		 1555 memcpy(&ctx->key, key, keylen);
1556 ctx->keylen = keylen;
1557
2e13ce08
LC		 1558 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1559
4de9d0b5 1560 return 0;
4de9d0b5
LN		 1561}
1562
1563static void common_nonsnoop_unmap(struct device *dev,
1564 struct talitos_edesc *edesc,
1565 struct ablkcipher_request *areq)
1566{
1567 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
032d197e 1568
6a1e8d14 1569 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
4de9d0b5
LN		 1570 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1571
4de9d0b5
LN		 1572 if (edesc->dma_len)
1573 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1574 DMA_BIDIRECTIONAL);
1575}
1576
1577static void ablkcipher_done(struct device *dev,
1578 struct talitos_desc *desc, void *context,
1579 int err)
1580{
1581 struct ablkcipher_request *areq = context;
19bbbc63
KP		 1582 struct talitos_edesc *edesc;
1583
1584 edesc = container_of(desc, struct talitos_edesc, desc);
4de9d0b5
LN		 1585
1586 common_nonsnoop_unmap(dev, edesc, areq);
1587
1588 kfree(edesc);
1589
1590 areq->base.complete(&areq->base, err);
1591}
1592
1593static int common_nonsnoop(struct talitos_edesc *edesc,
1594 struct ablkcipher_request *areq,
4de9d0b5
LN		 1595 void (*callback) (struct device *dev,
1596 struct talitos_desc *desc,
1597 void *context, int error))
1598{
1599 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1600 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1601 struct device *dev = ctx->dev;
1602 struct talitos_desc *desc = &edesc->desc;
1603 unsigned int cryptlen = areq->nbytes;
79fd31d3 1604 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1605 int sg_count, ret;
6a1e8d14 1606 bool sync_needed = false;
922f9dc8
LC		 1607 struct talitos_private *priv = dev_get_drvdata(dev);
1608 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 1609
1610 /* first DWORD empty */
4de9d0b5
LN		 1611
1612 /* cipher iv */
da9de146 1613 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
4de9d0b5
LN		 1614
1615 /* cipher key */
2e13ce08 1616 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
4de9d0b5 1617
6a1e8d14
LC		 1618 sg_count = edesc->src_nents ?: 1;
1619 if (is_sec1 && sg_count > 1)
1620 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1621 cryptlen);
1622 else
1623 sg_count = dma_map_sg(dev, areq->src, sg_count,
1624 (areq->src == areq->dst) ?
1625 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
4de9d0b5
LN		 1626 /*
1627 * cipher in
1628 */
6a1e8d14
LC		 1629 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1630 &desc->ptr[3], sg_count, 0, 0);
1631 if (sg_count > 1)
1632 sync_needed = true;
4de9d0b5
LN		 1633
1634 /* cipher out */
6a1e8d14
LC		 1635 if (areq->src != areq->dst) {
1636 sg_count = edesc->dst_nents ? : 1;
1637 if (!is_sec1 || sg_count == 1)
1638 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1639 }
1640
1641 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1642 sg_count, 0, (edesc->src_nents + 1));
1643 if (ret > 1)
1644 sync_needed = true;
4de9d0b5
LN		 1645
1646 /* iv out */
a2b35aa8 1647 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
4de9d0b5
LN		 1648 DMA_FROM_DEVICE);
1649
1650 /* last DWORD empty */
4de9d0b5 1651
6a1e8d14
LC		 1652 if (sync_needed)
1653 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1654 edesc->dma_len, DMA_BIDIRECTIONAL);
1655
5228f0f7 1656 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
4de9d0b5
LN		 1657 if (ret != -EINPROGRESS) {
1658 common_nonsnoop_unmap(dev, edesc, areq);
1659 kfree(edesc);
1660 }
1661 return ret;
1662}
1663
e938e465 1664static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
62293a37 1665 areq, bool encrypt)
4de9d0b5
LN		 1666{
1667 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1668 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
79fd31d3 1669 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1670
aeb4c132 1671 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1672 areq->info, 0, areq->nbytes, 0, ivsize, 0,
62293a37 1673 areq->base.flags, encrypt);
4de9d0b5
LN		 1674}
1675
1676static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1677{
1678 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1679 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1680 struct talitos_edesc *edesc;
1681
1682 /* allocate extended descriptor */
62293a37 1683 edesc = ablkcipher_edesc_alloc(areq, true);
4de9d0b5
LN		 1684 if (IS_ERR(edesc))
1685 return PTR_ERR(edesc);
1686
1687 /* set encrypt */
1688 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1689
febec542 1690 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1691}
1692
1693static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1694{
1695 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1696 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1697 struct talitos_edesc *edesc;
1698
1699 /* allocate extended descriptor */
62293a37 1700 edesc = ablkcipher_edesc_alloc(areq, false);
4de9d0b5
LN		 1701 if (IS_ERR(edesc))
1702 return PTR_ERR(edesc);
1703
1704 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1705
febec542 1706 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1707}
1708
497f2e6b
LN		 1709static void common_nonsnoop_hash_unmap(struct device *dev,
1710 struct talitos_edesc *edesc,
1711 struct ahash_request *areq)
1712{
1713 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
ad4cd51f
LC		 1714 struct talitos_private *priv = dev_get_drvdata(dev);
1715 bool is_sec1 = has_ftr_sec1(priv);
1716 struct talitos_desc *desc = &edesc->desc;
1717 struct talitos_desc *desc2 = desc + 1;
1718
1719 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1720 if (desc->next_desc &&
1721 desc->ptr[5].ptr != desc2->ptr[5].ptr)
1722 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
497f2e6b 1723
6a1e8d14 1724 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
032d197e 1725
ad4cd51f
LC		 1726 /* When using hashctx-in, must unmap it. */
1727 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1728 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1729 DMA_TO_DEVICE);
1730 else if (desc->next_desc)
1731 unmap_single_talitos_ptr(dev, &desc2->ptr[1],
1732 DMA_TO_DEVICE);
1733
1734 if (is_sec1 && req_ctx->nbuf)
1735 unmap_single_talitos_ptr(dev, &desc->ptr[3],
1736 DMA_TO_DEVICE);
1737
497f2e6b
LN		 1738 if (edesc->dma_len)
1739 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1740 DMA_BIDIRECTIONAL);
1741
37b5e889
LC		 1742 if (edesc->desc.next_desc)
1743 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1744 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
497f2e6b
LN		 1745}
1746
1747static void ahash_done(struct device *dev,
1748 struct talitos_desc *desc, void *context,
1749 int err)
1750{
1751 struct ahash_request *areq = context;
1752 struct talitos_edesc *edesc =
1753 container_of(desc, struct talitos_edesc, desc);
1754 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1755
1756 if (!req_ctx->last && req_ctx->to_hash_later) {
1757 /* Position any partial block for next update/final/finup */
3c0dd190 1758 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
5e833bc4 1759 req_ctx->nbuf = req_ctx->to_hash_later;
497f2e6b
LN		 1760 }
1761 common_nonsnoop_hash_unmap(dev, edesc, areq);
1762
1763 kfree(edesc);
1764
1765 areq->base.complete(&areq->base, err);
1766}
1767
2d02905e
LC		 1768/*
1769 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1770 * ourself and submit a padded block
1771 */
5b2cf268 1772static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
2d02905e
LC		 1773 struct talitos_edesc *edesc,
1774 struct talitos_ptr *ptr)
1775{
1776 static u8 padded_hash[64] = {
1777 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1778 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1779 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1780 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1781 };
1782
1783 pr_err_once("Bug in SEC1, padding ourself\n");
1784 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1785 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1786 (char *)padded_hash, DMA_TO_DEVICE);
1787}
1788
497f2e6b
LN		 1789static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1790 struct ahash_request *areq, unsigned int length,
37b5e889 1791 unsigned int offset,
497f2e6b
LN		 1792 void (*callback) (struct device *dev,
1793 struct talitos_desc *desc,
1794 void *context, int error))
1795{
1796 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1797 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1798 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1799 struct device *dev = ctx->dev;
1800 struct talitos_desc *desc = &edesc->desc;
032d197e 1801 int ret;
6a1e8d14 1802 bool sync_needed = false;
922f9dc8
LC		 1803 struct talitos_private *priv = dev_get_drvdata(dev);
1804 bool is_sec1 = has_ftr_sec1(priv);
6a1e8d14 1805 int sg_count;
497f2e6b
LN		 1806
1807 /* first DWORD empty */
497f2e6b 1808
60f208d7
KP		 1809 /* hash context in */
1810 if (!req_ctx->first || req_ctx->swinit) {
6a4967c3
LC		 1811 map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1812 req_ctx->hw_context_size,
1813 req_ctx->hw_context,
1814 DMA_TO_DEVICE);
60f208d7 1815 req_ctx->swinit = 0;
497f2e6b 1816 }
afd62fa2
LC		 1817 /* Indicate next op is not the first. */
1818 req_ctx->first = 0;
497f2e6b
LN		 1819
1820 /* HMAC key */
1821 if (ctx->keylen)
2e13ce08
LC		 1822 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1823 is_sec1);
497f2e6b 1824
37b5e889
LC		 1825 if (is_sec1 && req_ctx->nbuf)
1826 length -= req_ctx->nbuf;
1827
6a1e8d14
LC		 1828 sg_count = edesc->src_nents ?: 1;
1829 if (is_sec1 && sg_count > 1)
37b5e889
LC		 1830 sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
1831 edesc->buf + sizeof(struct talitos_desc),
1832 length, req_ctx->nbuf);
1833 else if (length)
6a1e8d14
LC		 1834 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1835 DMA_TO_DEVICE);
497f2e6b
LN		 1836 /*
1837 * data in
1838 */
37b5e889 1839 if (is_sec1 && req_ctx->nbuf) {
ad4cd51f
LC		 1840 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
1841 req_ctx->buf[req_ctx->buf_idx],
1842 DMA_TO_DEVICE);
37b5e889
LC		 1843 } else {
1844 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1845 &desc->ptr[3], sg_count, offset, 0);
1846 if (sg_count > 1)
1847 sync_needed = true;
1848 }
497f2e6b
LN		 1849
1850 /* fifth DWORD empty */
497f2e6b
LN		 1851
1852 /* hash/HMAC out -or- hash context out */
1853 if (req_ctx->last)
1854 map_single_talitos_ptr(dev, &desc->ptr[5],
1855 crypto_ahash_digestsize(tfm),
a2b35aa8 1856 areq->result, DMA_FROM_DEVICE);
497f2e6b 1857 else
6a4967c3
LC		 1858 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1859 req_ctx->hw_context_size,
1860 req_ctx->hw_context,
1861 DMA_FROM_DEVICE);
497f2e6b
LN		 1862
1863 /* last DWORD empty */
497f2e6b 1864
2d02905e
LC		 1865 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1866 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1867
37b5e889
LC		 1868 if (is_sec1 && req_ctx->nbuf && length) {
1869 struct talitos_desc *desc2 = desc + 1;
1870 dma_addr_t next_desc;
1871
1872 memset(desc2, 0, sizeof(*desc2));
1873 desc2->hdr = desc->hdr;
1874 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1875 desc2->hdr1 = desc2->hdr;
1876 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1877 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1878 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1879
ad4cd51f
LC		 1880 if (desc->ptr[1].ptr)
1881 copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
1882 is_sec1);
1883 else
6a4967c3
LC		 1884 map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
1885 req_ctx->hw_context_size,
1886 req_ctx->hw_context,
1887 DMA_TO_DEVICE);
37b5e889
LC		 1888 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1889 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1890 &desc2->ptr[3], sg_count, offset, 0);
1891 if (sg_count > 1)
1892 sync_needed = true;
1893 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1894 if (req_ctx->last)
6a4967c3
LC		 1895 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1896 req_ctx->hw_context_size,
1897 req_ctx->hw_context,
1898 DMA_FROM_DEVICE);
37b5e889
LC		 1899
1900 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1901 DMA_BIDIRECTIONAL);
1902 desc->next_desc = cpu_to_be32(next_desc);
1903 }
1904
6a1e8d14
LC		 1905 if (sync_needed)
1906 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1907 edesc->dma_len, DMA_BIDIRECTIONAL);
1908
5228f0f7 1909 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
497f2e6b
LN		 1910 if (ret != -EINPROGRESS) {
1911 common_nonsnoop_hash_unmap(dev, edesc, areq);
1912 kfree(edesc);
1913 }
1914 return ret;
1915}
1916
1917static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1918 unsigned int nbytes)
1919{
1920 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1921 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1922 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
37b5e889
LC		 1923 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1924 bool is_sec1 = has_ftr_sec1(priv);
1925
1926 if (is_sec1)
1927 nbytes -= req_ctx->nbuf;
497f2e6b 1928
aeb4c132 1929 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
62293a37 1930 nbytes, 0, 0, 0, areq->base.flags, false);
497f2e6b
LN		 1931}
1932
1933static int ahash_init(struct ahash_request *areq)
1934{
1935 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
6a4967c3
LC		 1936 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1937 struct device *dev = ctx->dev;
497f2e6b 1938 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
49f9783b 1939 unsigned int size;
6a4967c3 1940 dma_addr_t dma;
497f2e6b
LN		 1941
1942 /* Initialize the context */
3c0dd190 1943 req_ctx->buf_idx = 0;
5e833bc4 1944 req_ctx->nbuf = 0;
60f208d7
KP		 1945 req_ctx->first = 1; /* first indicates h/w must init its context */
1946 req_ctx->swinit = 0; /* assume h/w init of context */
49f9783b 1947 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
497f2e6b
LN		 1948 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1949 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
49f9783b 1950 req_ctx->hw_context_size = size;
497f2e6b 1951
6a4967c3
LC		 1952 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
1953 DMA_TO_DEVICE);
1954 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
1955
497f2e6b
LN		 1956 return 0;
1957}
1958
60f208d7
KP		 1959/*
1960 * on h/w without explicit sha224 support, we initialize h/w context
1961 * manually with sha224 constants, and tell it to run sha256.
1962 */
1963static int ahash_init_sha224_swinit(struct ahash_request *areq)
1964{
1965 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1966
a752447a
KP		 1967 req_ctx->hw_context[0] = SHA224_H0;
1968 req_ctx->hw_context[1] = SHA224_H1;
1969 req_ctx->hw_context[2] = SHA224_H2;
1970 req_ctx->hw_context[3] = SHA224_H3;
1971 req_ctx->hw_context[4] = SHA224_H4;
1972 req_ctx->hw_context[5] = SHA224_H5;
1973 req_ctx->hw_context[6] = SHA224_H6;
1974 req_ctx->hw_context[7] = SHA224_H7;
60f208d7
KP		 1975
1976 /* init 64-bit count */
1977 req_ctx->hw_context[8] = 0;
1978 req_ctx->hw_context[9] = 0;
1979
6a4967c3
LC		 1980 ahash_init(areq);
1981 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1982
60f208d7
KP		 1983 return 0;
1984}
1985
497f2e6b
LN		 1986static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1987{
1988 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1989 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1990 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1991 struct talitos_edesc *edesc;
1992 unsigned int blocksize =
1993 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1994 unsigned int nbytes_to_hash;
1995 unsigned int to_hash_later;
5e833bc4 1996 unsigned int nsg;
8e409fe1 1997 int nents;
37b5e889
LC		 1998 struct device *dev = ctx->dev;
1999 struct talitos_private *priv = dev_get_drvdata(dev);
2000 bool is_sec1 = has_ftr_sec1(priv);
2001 int offset = 0;
3c0dd190 2002 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
497f2e6b 2003
5e833bc4
LN		 2004 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
2005 /* Buffer up to one whole block */
8e409fe1
LC		 2006 nents = sg_nents_for_len(areq->src, nbytes);
2007 if (nents < 0) {
2008 dev_err(ctx->dev, "Invalid number of src SG.\n");
2009 return nents;
2010 }
2011 sg_copy_to_buffer(areq->src, nents,
3c0dd190 2012 ctx_buf + req_ctx->nbuf, nbytes);
5e833bc4 2013 req_ctx->nbuf += nbytes;
497f2e6b
LN		 2014 return 0;
2015 }
2016
5e833bc4
LN		 2017 /* At least (blocksize + 1) bytes are available to hash */
2018 nbytes_to_hash = nbytes + req_ctx->nbuf;
2019 to_hash_later = nbytes_to_hash & (blocksize - 1);
2020
2021 if (req_ctx->last)
2022 to_hash_later = 0;
2023 else if (to_hash_later)
2024 /* There is a partial block. Hash the full block(s) now */
2025 nbytes_to_hash -= to_hash_later;
2026 else {
2027 /* Keep one block buffered */
2028 nbytes_to_hash -= blocksize;
2029 to_hash_later = blocksize;
2030 }
2031
2032 /* Chain in any previously buffered data */
37b5e889 2033 if (!is_sec1 && req_ctx->nbuf) {
5e833bc4
LN		 2034 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2035 sg_init_table(req_ctx->bufsl, nsg);
3c0dd190 2036 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
5e833bc4 2037 if (nsg > 1)
c56f6d12 2038 sg_chain(req_ctx->bufsl, 2, areq->src);
497f2e6b 2039 req_ctx->psrc = req_ctx->bufsl;
37b5e889
LC		 2040 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2041 if (nbytes_to_hash > blocksize)
2042 offset = blocksize - req_ctx->nbuf;
2043 else
2044 offset = nbytes_to_hash - req_ctx->nbuf;
2045 nents = sg_nents_for_len(areq->src, offset);
2046 if (nents < 0) {
2047 dev_err(ctx->dev, "Invalid number of src SG.\n");
2048 return nents;
2049 }
2050 sg_copy_to_buffer(areq->src, nents,
3c0dd190 2051 ctx_buf + req_ctx->nbuf, offset);
37b5e889
LC		 2052 req_ctx->nbuf += offset;
2053 req_ctx->psrc = areq->src;
5e833bc4 2054 } else
497f2e6b 2055 req_ctx->psrc = areq->src;
5e833bc4
LN		 2056
2057 if (to_hash_later) {
8e409fe1
LC		 2058 nents = sg_nents_for_len(areq->src, nbytes);
2059 if (nents < 0) {
2060 dev_err(ctx->dev, "Invalid number of src SG.\n");
2061 return nents;
2062 }
d0525723 2063 sg_pcopy_to_buffer(areq->src, nents,
3c0dd190 2064 req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
5e833bc4
LN		 2065 to_hash_later,
2066 nbytes - to_hash_later);
497f2e6b 2067 }
5e833bc4 2068 req_ctx->to_hash_later = to_hash_later;
497f2e6b 2069
5e833bc4 2070 /* Allocate extended descriptor */
497f2e6b
LN		 2071 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2072 if (IS_ERR(edesc))
2073 return PTR_ERR(edesc);
2074
2075 edesc->desc.hdr = ctx->desc_hdr_template;
2076
2077 /* On last one, request SEC to pad; otherwise continue */
2078 if (req_ctx->last)
2079 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2080 else
2081 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2082
60f208d7
KP		 2083 /* request SEC to INIT hash. */
2084 if (req_ctx->first && !req_ctx->swinit)
497f2e6b
LN		 2085 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2086
2087 /* When the tfm context has a keylen, it's an HMAC.
2088 * A first or last (ie. not middle) descriptor must request HMAC.
2089 */
2090 if (ctx->keylen && (req_ctx->first || req_ctx->last))
2091 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2092
37b5e889 2093 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
497f2e6b
LN		 2094 ahash_done);
2095}
2096
2097static int ahash_update(struct ahash_request *areq)
2098{
2099 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2100
2101 req_ctx->last = 0;
2102
2103 return ahash_process_req(areq, areq->nbytes);
2104}
2105
2106static int ahash_final(struct ahash_request *areq)
2107{
2108 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2109
2110 req_ctx->last = 1;
2111
2112 return ahash_process_req(areq, 0);
2113}
2114
2115static int ahash_finup(struct ahash_request *areq)
2116{
2117 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2118
2119 req_ctx->last = 1;
2120
2121 return ahash_process_req(areq, areq->nbytes);
2122}
2123
2124static int ahash_digest(struct ahash_request *areq)
2125{
2126 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
60f208d7 2127 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
497f2e6b 2128
60f208d7 2129 ahash->init(areq);
497f2e6b
LN		 2130 req_ctx->last = 1;
2131
2132 return ahash_process_req(areq, areq->nbytes);
2133}
2134
3639ca84
HG		 2135static int ahash_export(struct ahash_request *areq, void *out)
2136{
2137 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2138 struct talitos_export_state *export = out;
6a4967c3
LC		 2139 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2140 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2141 struct device *dev = ctx->dev;
2142 dma_addr_t dma;
2143
2144 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2145 DMA_FROM_DEVICE);
2146 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
3639ca84
HG		 2147
2148 memcpy(export->hw_context, req_ctx->hw_context,
2149 req_ctx->hw_context_size);
3c0dd190 2150 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
3639ca84
HG		 2151 export->swinit = req_ctx->swinit;
2152 export->first = req_ctx->first;
2153 export->last = req_ctx->last;
2154 export->to_hash_later = req_ctx->to_hash_later;
2155 export->nbuf = req_ctx->nbuf;
2156
2157 return 0;
2158}
2159
2160static int ahash_import(struct ahash_request *areq, const void *in)
2161{
2162 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2163 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
6a4967c3
LC		 2164 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2165 struct device *dev = ctx->dev;
3639ca84 2166 const struct talitos_export_state *export = in;
49f9783b 2167 unsigned int size;
6a4967c3 2168 dma_addr_t dma;
3639ca84
HG		 2169
2170 memset(req_ctx, 0, sizeof(*req_ctx));
49f9783b 2171 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
3639ca84
HG		 2172 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2173 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
49f9783b 2174 req_ctx->hw_context_size = size;
49f9783b 2175 memcpy(req_ctx->hw_context, export->hw_context, size);
3c0dd190 2176 memcpy(req_ctx->buf[0], export->buf, export->nbuf);
3639ca84
HG		 2177 req_ctx->swinit = export->swinit;
2178 req_ctx->first = export->first;
2179 req_ctx->last = export->last;
2180 req_ctx->to_hash_later = export->to_hash_later;
2181 req_ctx->nbuf = export->nbuf;
2182
6a4967c3
LC		 2183 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2184 DMA_TO_DEVICE);
2185 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2186
3639ca84
HG		 2187 return 0;
2188}
2189
79b3a418
LN		 2190static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2191 u8 *hash)
2192{
2193 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2194
2195 struct scatterlist sg[1];
2196 struct ahash_request *req;
f1c90ac3 2197 struct crypto_wait wait;
79b3a418
LN		 2198 int ret;
2199
f1c90ac3 2200 crypto_init_wait(&wait);
79b3a418
LN		 2201
2202 req = ahash_request_alloc(tfm, GFP_KERNEL);
2203 if (!req)
2204 return -ENOMEM;
2205
2206 /* Keep tfm keylen == 0 during hash of the long key */
2207 ctx->keylen = 0;
2208 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
f1c90ac3 2209 crypto_req_done, &wait);
79b3a418
LN		 2210
2211 sg_init_one(&sg[0], key, keylen);
2212
2213 ahash_request_set_crypt(req, sg, hash, keylen);
f1c90ac3
GBY		 2214 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2215
79b3a418
LN		 2216 ahash_request_free(req);
2217
2218 return ret;
2219}
2220
2221static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2222 unsigned int keylen)
2223{
2224 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2e13ce08 2225 struct device *dev = ctx->dev;
79b3a418
LN		 2226 unsigned int blocksize =
2227 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2228 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2229 unsigned int keysize = keylen;
2230 u8 hash[SHA512_DIGEST_SIZE];
2231 int ret;
2232
2233 if (keylen <= blocksize)
2234 memcpy(ctx->key, key, keysize);
2235 else {
2236 /* Must get the hash of the long key */
2237 ret = keyhash(tfm, key, keylen, hash);
2238
2239 if (ret) {
2240 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2241 return -EINVAL;
2242 }
2243
2244 keysize = digestsize;
2245 memcpy(ctx->key, hash, digestsize);
2246 }
2247
2e13ce08
LC		 2248 if (ctx->keylen)
2249 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2250
79b3a418 2251 ctx->keylen = keysize;
2e13ce08 2252 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
79b3a418
LN		 2253
2254 return 0;
2255}
2256
2257
9c4a7965 2258struct talitos_alg_template {
d5e4aaef 2259 u32 type;
b0057763 2260 u32 priority;
d5e4aaef
LN		 2261 union {
2262 struct crypto_alg crypto;
acbf7c62 2263 struct ahash_alg hash;
aeb4c132 2264 struct aead_alg aead;
d5e4aaef 2265 } alg;
9c4a7965
KP		 2266 __be32 desc_hdr_template;
2267};
2268
2269static struct talitos_alg_template driver_algs[] = {
991155ba 2270 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
d5e4aaef 2271 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2272 .alg.aead = {
2273 .base = {
2274 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2275 .cra_driver_name = "authenc-hmac-sha1-"
2276 "cbc-aes-talitos",
2277 .cra_blocksize = AES_BLOCK_SIZE,
2278 .cra_flags = CRYPTO_ALG_ASYNC,
2279 },
2280 .ivsize = AES_BLOCK_SIZE,
2281 .maxauthsize = SHA1_DIGEST_SIZE,
56af8cd4 2282 },
9c4a7965
KP		 2283 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2284 DESC_HDR_SEL0_AESU |
2285 DESC_HDR_MODE0_AESU_CBC |
2286 DESC_HDR_SEL1_MDEUA |
2287 DESC_HDR_MODE1_MDEU_INIT |
2288 DESC_HDR_MODE1_MDEU_PAD |
2289 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
70bcaca7 2290 },
7405c8d7
LC		 2291 { .type = CRYPTO_ALG_TYPE_AEAD,
2292 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2293 .alg.aead = {
2294 .base = {
2295 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2296 .cra_driver_name = "authenc-hmac-sha1-"
2297 "cbc-aes-talitos",
2298 .cra_blocksize = AES_BLOCK_SIZE,
2299 .cra_flags = CRYPTO_ALG_ASYNC,
2300 },
2301 .ivsize = AES_BLOCK_SIZE,
2302 .maxauthsize = SHA1_DIGEST_SIZE,
2303 },
2304 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2305 DESC_HDR_SEL0_AESU |
2306 DESC_HDR_MODE0_AESU_CBC |
2307 DESC_HDR_SEL1_MDEUA |
2308 DESC_HDR_MODE1_MDEU_INIT |
2309 DESC_HDR_MODE1_MDEU_PAD |
2310 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2311 },
d5e4aaef 2312 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2313 .alg.aead = {
2314 .base = {
2315 .cra_name = "authenc(hmac(sha1),"
2316 "cbc(des3_ede))",
2317 .cra_driver_name = "authenc-hmac-sha1-"
2318 "cbc-3des-talitos",
2319 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2320 .cra_flags = CRYPTO_ALG_ASYNC,
2321 },
2322 .ivsize = DES3_EDE_BLOCK_SIZE,
2323 .maxauthsize = SHA1_DIGEST_SIZE,
56af8cd4 2324 },
70bcaca7
LN		 2325 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2326 DESC_HDR_SEL0_DEU |
2327 DESC_HDR_MODE0_DEU_CBC |
2328 DESC_HDR_MODE0_DEU_3DES |
2329 DESC_HDR_SEL1_MDEUA |
2330 DESC_HDR_MODE1_MDEU_INIT |
2331 DESC_HDR_MODE1_MDEU_PAD |
2332 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
3952f17e 2333 },
7405c8d7
LC		 2334 { .type = CRYPTO_ALG_TYPE_AEAD,
2335 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2336 .alg.aead = {
2337 .base = {
2338 .cra_name = "authenc(hmac(sha1),"
2339 "cbc(des3_ede))",
2340 .cra_driver_name = "authenc-hmac-sha1-"
2341 "cbc-3des-talitos",
2342 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2343 .cra_flags = CRYPTO_ALG_ASYNC,
2344 },
2345 .ivsize = DES3_EDE_BLOCK_SIZE,
2346 .maxauthsize = SHA1_DIGEST_SIZE,
2347 },
2348 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2349 DESC_HDR_SEL0_DEU |
2350 DESC_HDR_MODE0_DEU_CBC |
2351 DESC_HDR_MODE0_DEU_3DES |
2352 DESC_HDR_SEL1_MDEUA |
2353 DESC_HDR_MODE1_MDEU_INIT |
2354 DESC_HDR_MODE1_MDEU_PAD |
2355 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2356 },
357fb605 2357 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2358 .alg.aead = {
2359 .base = {
2360 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2361 .cra_driver_name = "authenc-hmac-sha224-"
2362 "cbc-aes-talitos",
2363 .cra_blocksize = AES_BLOCK_SIZE,
2364 .cra_flags = CRYPTO_ALG_ASYNC,
2365 },
2366 .ivsize = AES_BLOCK_SIZE,
2367 .maxauthsize = SHA224_DIGEST_SIZE,
357fb605
HG		 2368 },
2369 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2370 DESC_HDR_SEL0_AESU |
2371 DESC_HDR_MODE0_AESU_CBC |
2372 DESC_HDR_SEL1_MDEUA |
2373 DESC_HDR_MODE1_MDEU_INIT |
2374 DESC_HDR_MODE1_MDEU_PAD |
2375 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2376 },
7405c8d7
LC		 2377 { .type = CRYPTO_ALG_TYPE_AEAD,
2378 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2379 .alg.aead = {
2380 .base = {
2381 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2382 .cra_driver_name = "authenc-hmac-sha224-"
2383 "cbc-aes-talitos",
2384 .cra_blocksize = AES_BLOCK_SIZE,
2385 .cra_flags = CRYPTO_ALG_ASYNC,
2386 },
2387 .ivsize = AES_BLOCK_SIZE,
2388 .maxauthsize = SHA224_DIGEST_SIZE,
2389 },
2390 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2391 DESC_HDR_SEL0_AESU |
2392 DESC_HDR_MODE0_AESU_CBC |
2393 DESC_HDR_SEL1_MDEUA |
2394 DESC_HDR_MODE1_MDEU_INIT |
2395 DESC_HDR_MODE1_MDEU_PAD |
2396 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2397 },
357fb605 2398 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2399 .alg.aead = {
2400 .base = {
2401 .cra_name = "authenc(hmac(sha224),"
2402 "cbc(des3_ede))",
2403 .cra_driver_name = "authenc-hmac-sha224-"
2404 "cbc-3des-talitos",
2405 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2406 .cra_flags = CRYPTO_ALG_ASYNC,
2407 },
2408 .ivsize = DES3_EDE_BLOCK_SIZE,
2409 .maxauthsize = SHA224_DIGEST_SIZE,
357fb605
HG		 2410 },
2411 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2412 DESC_HDR_SEL0_DEU |
2413 DESC_HDR_MODE0_DEU_CBC |
2414 DESC_HDR_MODE0_DEU_3DES |
2415 DESC_HDR_SEL1_MDEUA |
2416 DESC_HDR_MODE1_MDEU_INIT |
2417 DESC_HDR_MODE1_MDEU_PAD |
2418 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2419 },
7405c8d7
LC		 2420 { .type = CRYPTO_ALG_TYPE_AEAD,
2421 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2422 .alg.aead = {
2423 .base = {
2424 .cra_name = "authenc(hmac(sha224),"
2425 "cbc(des3_ede))",
2426 .cra_driver_name = "authenc-hmac-sha224-"
2427 "cbc-3des-talitos",
2428 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2429 .cra_flags = CRYPTO_ALG_ASYNC,
2430 },
2431 .ivsize = DES3_EDE_BLOCK_SIZE,
2432 .maxauthsize = SHA224_DIGEST_SIZE,
2433 },
2434 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2435 DESC_HDR_SEL0_DEU |
2436 DESC_HDR_MODE0_DEU_CBC |
2437 DESC_HDR_MODE0_DEU_3DES |
2438 DESC_HDR_SEL1_MDEUA |
2439 DESC_HDR_MODE1_MDEU_INIT |
2440 DESC_HDR_MODE1_MDEU_PAD |
2441 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2442 },
d5e4aaef 2443 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2444 .alg.aead = {
2445 .base = {
2446 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2447 .cra_driver_name = "authenc-hmac-sha256-"
2448 "cbc-aes-talitos",
2449 .cra_blocksize = AES_BLOCK_SIZE,
2450 .cra_flags = CRYPTO_ALG_ASYNC,
2451 },
2452 .ivsize = AES_BLOCK_SIZE,
2453 .maxauthsize = SHA256_DIGEST_SIZE,
56af8cd4 2454 },
3952f17e
LN		 2455 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2456 DESC_HDR_SEL0_AESU |
2457 DESC_HDR_MODE0_AESU_CBC |
2458 DESC_HDR_SEL1_MDEUA |
2459 DESC_HDR_MODE1_MDEU_INIT |
2460 DESC_HDR_MODE1_MDEU_PAD |
2461 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2462 },
7405c8d7
LC		 2463 { .type = CRYPTO_ALG_TYPE_AEAD,
2464 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2465 .alg.aead = {
2466 .base = {
2467 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2468 .cra_driver_name = "authenc-hmac-sha256-"
2469 "cbc-aes-talitos",
2470 .cra_blocksize = AES_BLOCK_SIZE,
2471 .cra_flags = CRYPTO_ALG_ASYNC,
2472 },
2473 .ivsize = AES_BLOCK_SIZE,
2474 .maxauthsize = SHA256_DIGEST_SIZE,
2475 },
2476 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2477 DESC_HDR_SEL0_AESU |
2478 DESC_HDR_MODE0_AESU_CBC |
2479 DESC_HDR_SEL1_MDEUA |
2480 DESC_HDR_MODE1_MDEU_INIT |
2481 DESC_HDR_MODE1_MDEU_PAD |
2482 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2483 },
d5e4aaef 2484 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2485 .alg.aead = {
2486 .base = {
2487 .cra_name = "authenc(hmac(sha256),"
2488 "cbc(des3_ede))",
2489 .cra_driver_name = "authenc-hmac-sha256-"
2490 "cbc-3des-talitos",
2491 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2492 .cra_flags = CRYPTO_ALG_ASYNC,
2493 },
2494 .ivsize = DES3_EDE_BLOCK_SIZE,
2495 .maxauthsize = SHA256_DIGEST_SIZE,
56af8cd4 2496 },
3952f17e
LN		 2497 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2498 DESC_HDR_SEL0_DEU |
2499 DESC_HDR_MODE0_DEU_CBC |
2500 DESC_HDR_MODE0_DEU_3DES |
2501 DESC_HDR_SEL1_MDEUA |
2502 DESC_HDR_MODE1_MDEU_INIT |
2503 DESC_HDR_MODE1_MDEU_PAD |
2504 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2505 },
7405c8d7
LC		 2506 { .type = CRYPTO_ALG_TYPE_AEAD,
2507 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2508 .alg.aead = {
2509 .base = {
2510 .cra_name = "authenc(hmac(sha256),"
2511 "cbc(des3_ede))",
2512 .cra_driver_name = "authenc-hmac-sha256-"
2513 "cbc-3des-talitos",
2514 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2515 .cra_flags = CRYPTO_ALG_ASYNC,
2516 },
2517 .ivsize = DES3_EDE_BLOCK_SIZE,
2518 .maxauthsize = SHA256_DIGEST_SIZE,
2519 },
2520 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2521 DESC_HDR_SEL0_DEU |
2522 DESC_HDR_MODE0_DEU_CBC |
2523 DESC_HDR_MODE0_DEU_3DES |
2524 DESC_HDR_SEL1_MDEUA |
2525 DESC_HDR_MODE1_MDEU_INIT |
2526 DESC_HDR_MODE1_MDEU_PAD |
2527 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2528 },
d5e4aaef 2529 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2530 .alg.aead = {
2531 .base = {
2532 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2533 .cra_driver_name = "authenc-hmac-sha384-"
2534 "cbc-aes-talitos",
2535 .cra_blocksize = AES_BLOCK_SIZE,
2536 .cra_flags = CRYPTO_ALG_ASYNC,
2537 },
2538 .ivsize = AES_BLOCK_SIZE,
2539 .maxauthsize = SHA384_DIGEST_SIZE,
357fb605
HG		 2540 },
2541 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2542 DESC_HDR_SEL0_AESU |
2543 DESC_HDR_MODE0_AESU_CBC |
2544 DESC_HDR_SEL1_MDEUB |
2545 DESC_HDR_MODE1_MDEU_INIT |
2546 DESC_HDR_MODE1_MDEU_PAD |
2547 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2548 },
2549 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2550 .alg.aead = {
2551 .base = {
2552 .cra_name = "authenc(hmac(sha384),"
2553 "cbc(des3_ede))",
2554 .cra_driver_name = "authenc-hmac-sha384-"
2555 "cbc-3des-talitos",
2556 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2557 .cra_flags = CRYPTO_ALG_ASYNC,
2558 },
2559 .ivsize = DES3_EDE_BLOCK_SIZE,
2560 .maxauthsize = SHA384_DIGEST_SIZE,
357fb605
HG		 2561 },
2562 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2563 DESC_HDR_SEL0_DEU |
2564 DESC_HDR_MODE0_DEU_CBC |
2565 DESC_HDR_MODE0_DEU_3DES |
2566 DESC_HDR_SEL1_MDEUB |
2567 DESC_HDR_MODE1_MDEU_INIT |
2568 DESC_HDR_MODE1_MDEU_PAD |
2569 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2570 },
2571 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2572 .alg.aead = {
2573 .base = {
2574 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2575 .cra_driver_name = "authenc-hmac-sha512-"
2576 "cbc-aes-talitos",
2577 .cra_blocksize = AES_BLOCK_SIZE,
2578 .cra_flags = CRYPTO_ALG_ASYNC,
2579 },
2580 .ivsize = AES_BLOCK_SIZE,
2581 .maxauthsize = SHA512_DIGEST_SIZE,
357fb605
HG		 2582 },
2583 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2584 DESC_HDR_SEL0_AESU |
2585 DESC_HDR_MODE0_AESU_CBC |
2586 DESC_HDR_SEL1_MDEUB |
2587 DESC_HDR_MODE1_MDEU_INIT |
2588 DESC_HDR_MODE1_MDEU_PAD |
2589 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2590 },
2591 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2592 .alg.aead = {
2593 .base = {
2594 .cra_name = "authenc(hmac(sha512),"
2595 "cbc(des3_ede))",
2596 .cra_driver_name = "authenc-hmac-sha512-"
2597 "cbc-3des-talitos",
2598 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2599 .cra_flags = CRYPTO_ALG_ASYNC,
2600 },
2601 .ivsize = DES3_EDE_BLOCK_SIZE,
2602 .maxauthsize = SHA512_DIGEST_SIZE,
357fb605
HG		 2603 },
2604 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2605 DESC_HDR_SEL0_DEU |
2606 DESC_HDR_MODE0_DEU_CBC |
2607 DESC_HDR_MODE0_DEU_3DES |
2608 DESC_HDR_SEL1_MDEUB |
2609 DESC_HDR_MODE1_MDEU_INIT |
2610 DESC_HDR_MODE1_MDEU_PAD |
2611 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2612 },
2613 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2614 .alg.aead = {
2615 .base = {
2616 .cra_name = "authenc(hmac(md5),cbc(aes))",
2617 .cra_driver_name = "authenc-hmac-md5-"
2618 "cbc-aes-talitos",
2619 .cra_blocksize = AES_BLOCK_SIZE,
2620 .cra_flags = CRYPTO_ALG_ASYNC,
2621 },
2622 .ivsize = AES_BLOCK_SIZE,
2623 .maxauthsize = MD5_DIGEST_SIZE,
56af8cd4 2624 },
3952f17e
LN		 2625 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2626 DESC_HDR_SEL0_AESU |
2627 DESC_HDR_MODE0_AESU_CBC |
2628 DESC_HDR_SEL1_MDEUA |
2629 DESC_HDR_MODE1_MDEU_INIT |
2630 DESC_HDR_MODE1_MDEU_PAD |
2631 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2632 },
7405c8d7
LC		 2633 { .type = CRYPTO_ALG_TYPE_AEAD,
2634 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2635 .alg.aead = {
2636 .base = {
2637 .cra_name = "authenc(hmac(md5),cbc(aes))",
2638 .cra_driver_name = "authenc-hmac-md5-"
2639 "cbc-aes-talitos",
2640 .cra_blocksize = AES_BLOCK_SIZE,
2641 .cra_flags = CRYPTO_ALG_ASYNC,
2642 },
2643 .ivsize = AES_BLOCK_SIZE,
2644 .maxauthsize = MD5_DIGEST_SIZE,
2645 },
2646 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2647 DESC_HDR_SEL0_AESU |
2648 DESC_HDR_MODE0_AESU_CBC |
2649 DESC_HDR_SEL1_MDEUA |
2650 DESC_HDR_MODE1_MDEU_INIT |
2651 DESC_HDR_MODE1_MDEU_PAD |
2652 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2653 },
d5e4aaef 2654 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2655 .alg.aead = {
2656 .base = {
2657 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2658 .cra_driver_name = "authenc-hmac-md5-"
2659 "cbc-3des-talitos",
2660 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2661 .cra_flags = CRYPTO_ALG_ASYNC,
2662 },
2663 .ivsize = DES3_EDE_BLOCK_SIZE,
2664 .maxauthsize = MD5_DIGEST_SIZE,
56af8cd4 2665 },
3952f17e
LN		 2666 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2667 DESC_HDR_SEL0_DEU |
2668 DESC_HDR_MODE0_DEU_CBC |
2669 DESC_HDR_MODE0_DEU_3DES |
2670 DESC_HDR_SEL1_MDEUA |
2671 DESC_HDR_MODE1_MDEU_INIT |
2672 DESC_HDR_MODE1_MDEU_PAD |
2673 DESC_HDR_MODE1_MDEU_MD5_HMAC,
4de9d0b5 2674 },
7405c8d7
LC		 2675 { .type = CRYPTO_ALG_TYPE_AEAD,
2676 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2677 .alg.aead = {
2678 .base = {
2679 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2680 .cra_driver_name = "authenc-hmac-md5-"
2681 "cbc-3des-talitos",
2682 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2683 .cra_flags = CRYPTO_ALG_ASYNC,
2684 },
2685 .ivsize = DES3_EDE_BLOCK_SIZE,
2686 .maxauthsize = MD5_DIGEST_SIZE,
2687 },
2688 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2689 DESC_HDR_SEL0_DEU |
2690 DESC_HDR_MODE0_DEU_CBC |
2691 DESC_HDR_MODE0_DEU_3DES |
2692 DESC_HDR_SEL1_MDEUA |
2693 DESC_HDR_MODE1_MDEU_INIT |
2694 DESC_HDR_MODE1_MDEU_PAD |
2695 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2696 },
4de9d0b5 2697 /* ABLKCIPHER algorithms. */
5e75ae1b
LC		 2698 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2699 .alg.crypto = {
2700 .cra_name = "ecb(aes)",
2701 .cra_driver_name = "ecb-aes-talitos",
2702 .cra_blocksize = AES_BLOCK_SIZE,
2703 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2704 CRYPTO_ALG_ASYNC,
2705 .cra_ablkcipher = {
2706 .min_keysize = AES_MIN_KEY_SIZE,
2707 .max_keysize = AES_MAX_KEY_SIZE,
2708 .ivsize = AES_BLOCK_SIZE,
2709 }
2710 },
2711 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2712 DESC_HDR_SEL0_AESU,
2713 },
d5e4aaef
LN		 2714 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2715 .alg.crypto = {
4de9d0b5
LN		 2716 .cra_name = "cbc(aes)",
2717 .cra_driver_name = "cbc-aes-talitos",
2718 .cra_blocksize = AES_BLOCK_SIZE,
2719 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2720 CRYPTO_ALG_ASYNC,
4de9d0b5 2721 .cra_ablkcipher = {
4de9d0b5
LN		 2722 .min_keysize = AES_MIN_KEY_SIZE,
2723 .max_keysize = AES_MAX_KEY_SIZE,
2724 .ivsize = AES_BLOCK_SIZE,
2725 }
2726 },
2727 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2728 DESC_HDR_SEL0_AESU |
2729 DESC_HDR_MODE0_AESU_CBC,
2730 },
5e75ae1b
LC		 2731 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2732 .alg.crypto = {
2733 .cra_name = "ctr(aes)",
2734 .cra_driver_name = "ctr-aes-talitos",
2735 .cra_blocksize = AES_BLOCK_SIZE,
2736 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2737 CRYPTO_ALG_ASYNC,
2738 .cra_ablkcipher = {
2739 .min_keysize = AES_MIN_KEY_SIZE,
2740 .max_keysize = AES_MAX_KEY_SIZE,
2741 .ivsize = AES_BLOCK_SIZE,
2742 }
2743 },
70d355cc 2744 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
5e75ae1b
LC		 2745 DESC_HDR_SEL0_AESU |
2746 DESC_HDR_MODE0_AESU_CTR,
2747 },
2748 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2749 .alg.crypto = {
2750 .cra_name = "ecb(des)",
2751 .cra_driver_name = "ecb-des-talitos",
2752 .cra_blocksize = DES_BLOCK_SIZE,
2753 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2754 CRYPTO_ALG_ASYNC,
2755 .cra_ablkcipher = {
2756 .min_keysize = DES_KEY_SIZE,
2757 .max_keysize = DES_KEY_SIZE,
2758 .ivsize = DES_BLOCK_SIZE,
2759 }
2760 },
2761 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2762 DESC_HDR_SEL0_DEU,
2763 },
2764 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2765 .alg.crypto = {
2766 .cra_name = "cbc(des)",
2767 .cra_driver_name = "cbc-des-talitos",
2768 .cra_blocksize = DES_BLOCK_SIZE,
2769 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2770 CRYPTO_ALG_ASYNC,
2771 .cra_ablkcipher = {
2772 .min_keysize = DES_KEY_SIZE,
2773 .max_keysize = DES_KEY_SIZE,
2774 .ivsize = DES_BLOCK_SIZE,
2775 }
2776 },
2777 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2778 DESC_HDR_SEL0_DEU |
2779 DESC_HDR_MODE0_DEU_CBC,
2780 },
2781 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2782 .alg.crypto = {
2783 .cra_name = "ecb(des3_ede)",
2784 .cra_driver_name = "ecb-3des-talitos",
2785 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2786 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2787 CRYPTO_ALG_ASYNC,
2788 .cra_ablkcipher = {
2789 .min_keysize = DES3_EDE_KEY_SIZE,
2790 .max_keysize = DES3_EDE_KEY_SIZE,
2791 .ivsize = DES3_EDE_BLOCK_SIZE,
2792 }
2793 },
2794 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2795 DESC_HDR_SEL0_DEU |
2796 DESC_HDR_MODE0_DEU_3DES,
2797 },
d5e4aaef
LN		 2798 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2799 .alg.crypto = {
4de9d0b5
LN		 2800 .cra_name = "cbc(des3_ede)",
2801 .cra_driver_name = "cbc-3des-talitos",
2802 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2803 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2804 CRYPTO_ALG_ASYNC,
4de9d0b5 2805 .cra_ablkcipher = {
4de9d0b5
LN		 2806 .min_keysize = DES3_EDE_KEY_SIZE,
2807 .max_keysize = DES3_EDE_KEY_SIZE,
2808 .ivsize = DES3_EDE_BLOCK_SIZE,
2809 }
2810 },
2811 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2812 DESC_HDR_SEL0_DEU |
2813 DESC_HDR_MODE0_DEU_CBC |
2814 DESC_HDR_MODE0_DEU_3DES,
497f2e6b
LN		 2815 },
2816 /* AHASH algorithms. */
2817 { .type = CRYPTO_ALG_TYPE_AHASH,
2818 .alg.hash = {
497f2e6b 2819 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 2820 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2821 .halg.base = {
2822 .cra_name = "md5",
2823 .cra_driver_name = "md5-talitos",
b3988618 2824 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
6a38f622 2825 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2826 }
2827 },
2828 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2829 DESC_HDR_SEL0_MDEUA |
2830 DESC_HDR_MODE0_MDEU_MD5,
2831 },
2832 { .type = CRYPTO_ALG_TYPE_AHASH,
2833 .alg.hash = {
497f2e6b 2834 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 2835 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2836 .halg.base = {
2837 .cra_name = "sha1",
2838 .cra_driver_name = "sha1-talitos",
2839 .cra_blocksize = SHA1_BLOCK_SIZE,
6a38f622 2840 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2841 }
2842 },
2843 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2844 DESC_HDR_SEL0_MDEUA |
2845 DESC_HDR_MODE0_MDEU_SHA1,
2846 },
60f208d7
KP		 2847 { .type = CRYPTO_ALG_TYPE_AHASH,
2848 .alg.hash = {
60f208d7 2849 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 2850 .halg.statesize = sizeof(struct talitos_export_state),
60f208d7
KP		 2851 .halg.base = {
2852 .cra_name = "sha224",
2853 .cra_driver_name = "sha224-talitos",
2854 .cra_blocksize = SHA224_BLOCK_SIZE,
6a38f622 2855 .cra_flags = CRYPTO_ALG_ASYNC,
60f208d7
KP		 2856 }
2857 },
2858 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2859 DESC_HDR_SEL0_MDEUA |
2860 DESC_HDR_MODE0_MDEU_SHA224,
2861 },
497f2e6b
LN		 2862 { .type = CRYPTO_ALG_TYPE_AHASH,
2863 .alg.hash = {
497f2e6b 2864 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 2865 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2866 .halg.base = {
2867 .cra_name = "sha256",
2868 .cra_driver_name = "sha256-talitos",
2869 .cra_blocksize = SHA256_BLOCK_SIZE,
6a38f622 2870 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2871 }
2872 },
2873 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2874 DESC_HDR_SEL0_MDEUA |
2875 DESC_HDR_MODE0_MDEU_SHA256,
2876 },
2877 { .type = CRYPTO_ALG_TYPE_AHASH,
2878 .alg.hash = {
497f2e6b 2879 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 2880 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2881 .halg.base = {
2882 .cra_name = "sha384",
2883 .cra_driver_name = "sha384-talitos",
2884 .cra_blocksize = SHA384_BLOCK_SIZE,
6a38f622 2885 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2886 }
2887 },
2888 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2889 DESC_HDR_SEL0_MDEUB |
2890 DESC_HDR_MODE0_MDEUB_SHA384,
2891 },
2892 { .type = CRYPTO_ALG_TYPE_AHASH,
2893 .alg.hash = {
497f2e6b 2894 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 2895 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2896 .halg.base = {
2897 .cra_name = "sha512",
2898 .cra_driver_name = "sha512-talitos",
2899 .cra_blocksize = SHA512_BLOCK_SIZE,
6a38f622 2900 .cra_flags = CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2901 }
2902 },
2903 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2904 DESC_HDR_SEL0_MDEUB |
2905 DESC_HDR_MODE0_MDEUB_SHA512,
2906 },
79b3a418
LN		 2907 { .type = CRYPTO_ALG_TYPE_AHASH,
2908 .alg.hash = {
79b3a418 2909 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 2910 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2911 .halg.base = {
2912 .cra_name = "hmac(md5)",
2913 .cra_driver_name = "hmac-md5-talitos",
b3988618 2914 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
6a38f622 2915 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2916 }
2917 },
2918 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2919 DESC_HDR_SEL0_MDEUA |
2920 DESC_HDR_MODE0_MDEU_MD5,
2921 },
2922 { .type = CRYPTO_ALG_TYPE_AHASH,
2923 .alg.hash = {
79b3a418 2924 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 2925 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2926 .halg.base = {
2927 .cra_name = "hmac(sha1)",
2928 .cra_driver_name = "hmac-sha1-talitos",
2929 .cra_blocksize = SHA1_BLOCK_SIZE,
6a38f622 2930 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2931 }
2932 },
2933 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2934 DESC_HDR_SEL0_MDEUA |
2935 DESC_HDR_MODE0_MDEU_SHA1,
2936 },
2937 { .type = CRYPTO_ALG_TYPE_AHASH,
2938 .alg.hash = {
79b3a418 2939 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 2940 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2941 .halg.base = {
2942 .cra_name = "hmac(sha224)",
2943 .cra_driver_name = "hmac-sha224-talitos",
2944 .cra_blocksize = SHA224_BLOCK_SIZE,
6a38f622 2945 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2946 }
2947 },
2948 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2949 DESC_HDR_SEL0_MDEUA |
2950 DESC_HDR_MODE0_MDEU_SHA224,
2951 },
2952 { .type = CRYPTO_ALG_TYPE_AHASH,
2953 .alg.hash = {
79b3a418 2954 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 2955 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2956 .halg.base = {
2957 .cra_name = "hmac(sha256)",
2958 .cra_driver_name = "hmac-sha256-talitos",
2959 .cra_blocksize = SHA256_BLOCK_SIZE,
6a38f622 2960 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2961 }
2962 },
2963 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2964 DESC_HDR_SEL0_MDEUA |
2965 DESC_HDR_MODE0_MDEU_SHA256,
2966 },
2967 { .type = CRYPTO_ALG_TYPE_AHASH,
2968 .alg.hash = {
79b3a418 2969 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 2970 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2971 .halg.base = {
2972 .cra_name = "hmac(sha384)",
2973 .cra_driver_name = "hmac-sha384-talitos",
2974 .cra_blocksize = SHA384_BLOCK_SIZE,
6a38f622 2975 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2976 }
2977 },
2978 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2979 DESC_HDR_SEL0_MDEUB |
2980 DESC_HDR_MODE0_MDEUB_SHA384,
2981 },
2982 { .type = CRYPTO_ALG_TYPE_AHASH,
2983 .alg.hash = {
79b3a418 2984 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 2985 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2986 .halg.base = {
2987 .cra_name = "hmac(sha512)",
2988 .cra_driver_name = "hmac-sha512-talitos",
2989 .cra_blocksize = SHA512_BLOCK_SIZE,
6a38f622 2990 .cra_flags = CRYPTO_ALG_ASYNC,
79b3a418
LN		 2991 }
2992 },
2993 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2994 DESC_HDR_SEL0_MDEUB |
2995 DESC_HDR_MODE0_MDEUB_SHA512,
2996 }
9c4a7965
KP		 2997};
2998
2999struct talitos_crypto_alg {
3000 struct list_head entry;
3001 struct device *dev;
acbf7c62 3002 struct talitos_alg_template algt;
9c4a7965
KP		 3003};
3004
89d124cb
JE		 3005static int talitos_init_common(struct talitos_ctx *ctx,
3006 struct talitos_crypto_alg *talitos_alg)
9c4a7965 3007{
5228f0f7 3008 struct talitos_private *priv;
9c4a7965
KP		 3009
3010 /* update context with ptr to dev */
3011 ctx->dev = talitos_alg->dev;
19bbbc63 3012
5228f0f7
KP		 3013 /* assign SEC channel to tfm in round-robin fashion */
3014 priv = dev_get_drvdata(ctx->dev);
3015 ctx->ch = atomic_inc_return(&priv->last_chan) &
3016 (priv->num_channels - 1);
3017
9c4a7965 3018 /* copy descriptor header template value */
acbf7c62 3019 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
9c4a7965 3020
602dba5a
KP		 3021 /* select done notification */
3022 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3023
497f2e6b
LN		 3024 return 0;
3025}
3026
89d124cb
JE		 3027static int talitos_cra_init(struct crypto_tfm *tfm)
3028{
3029 struct crypto_alg *alg = tfm->__crt_alg;
3030 struct talitos_crypto_alg *talitos_alg;
3031 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3032
3033 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
3034 talitos_alg = container_of(__crypto_ahash_alg(alg),
3035 struct talitos_crypto_alg,
3036 algt.alg.hash);
3037 else
3038 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3039 algt.alg.crypto);
3040
3041 return talitos_init_common(ctx, talitos_alg);
3042}
3043
aeb4c132 3044static int talitos_cra_init_aead(struct crypto_aead *tfm)
497f2e6b 3045{
89d124cb
JE		 3046 struct aead_alg *alg = crypto_aead_alg(tfm);
3047 struct talitos_crypto_alg *talitos_alg;
3048 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3049
3050 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3051 algt.alg.aead);
3052
3053 return talitos_init_common(ctx, talitos_alg);
9c4a7965
KP		 3054}
3055
497f2e6b
LN		 3056static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3057{
3058 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3059
3060 talitos_cra_init(tfm);
3061
3062 ctx->keylen = 0;
3063 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3064 sizeof(struct talitos_ahash_req_ctx));
3065
3066 return 0;
3067}
3068
2e13ce08
LC		 3069static void talitos_cra_exit(struct crypto_tfm *tfm)
3070{
3071 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3072 struct device *dev = ctx->dev;
3073
3074 if (ctx->keylen)
3075 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3076}
3077
9c4a7965
KP		 3078/*
3079 * given the alg's descriptor header template, determine whether descriptor
3080 * type and primary/secondary execution units required match the hw
3081 * capabilities description provided in the device tree node.
3082 */
3083static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3084{
3085 struct talitos_private *priv = dev_get_drvdata(dev);
3086 int ret;
3087
3088 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3089 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3090
3091 if (SECONDARY_EU(desc_hdr_template))
3092 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3093 & priv->exec_units);
3094
3095 return ret;
3096}
3097
2dc11581 3098static int talitos_remove(struct platform_device *ofdev)
9c4a7965
KP		 3099{
3100 struct device *dev = &ofdev->dev;
3101 struct talitos_private *priv = dev_get_drvdata(dev);
3102 struct talitos_crypto_alg *t_alg, *n;
3103 int i;
3104
3105 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
acbf7c62
LN		 3106 switch (t_alg->algt.type) {
3107 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62 3108 break;
aeb4c132
HX		 3109 case CRYPTO_ALG_TYPE_AEAD:
3110 crypto_unregister_aead(&t_alg->algt.alg.aead);
acbf7c62
LN		 3111 case CRYPTO_ALG_TYPE_AHASH:
3112 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3113 break;
3114 }
9c4a7965 3115 list_del(&t_alg->entry);
9c4a7965
KP		 3116 }
3117
3118 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3119 talitos_unregister_rng(dev);
3120
c3e337f8 3121 for (i = 0; i < 2; i++)
2cdba3cf 3122 if (priv->irq[i]) {
c3e337f8
KP		 3123 free_irq(priv->irq[i], dev);
3124 irq_dispose_mapping(priv->irq[i]);
3125 }
9c4a7965 3126
c3e337f8 3127 tasklet_kill(&priv->done_task[0]);
2cdba3cf 3128 if (priv->irq[1])
c3e337f8 3129 tasklet_kill(&priv->done_task[1]);
9c4a7965 3130
9c4a7965
KP		 3131 return 0;
3132}
3133
3134static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3135 struct talitos_alg_template
3136 *template)
3137{
60f208d7 3138 struct talitos_private *priv = dev_get_drvdata(dev);
9c4a7965
KP		 3139 struct talitos_crypto_alg *t_alg;
3140 struct crypto_alg *alg;
3141
24b92ff2
LC		 3142 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3143 GFP_KERNEL);
9c4a7965
KP		 3144 if (!t_alg)
3145 return ERR_PTR(-ENOMEM);
3146
acbf7c62
LN		 3147 t_alg->algt = *template;
3148
3149 switch (t_alg->algt.type) {
3150 case CRYPTO_ALG_TYPE_ABLKCIPHER:
497f2e6b
LN		 3151 alg = &t_alg->algt.alg.crypto;
3152 alg->cra_init = talitos_cra_init;
2e13ce08 3153 alg->cra_exit = talitos_cra_exit;
d4cd3283 3154 alg->cra_type = &crypto_ablkcipher_type;
b286e003
KP		 3155 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3156 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3157 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3158 alg->cra_ablkcipher.geniv = "eseqiv";
497f2e6b 3159 break;
acbf7c62 3160 case CRYPTO_ALG_TYPE_AEAD:
aeb4c132 3161 alg = &t_alg->algt.alg.aead.base;
2e13ce08 3162 alg->cra_exit = talitos_cra_exit;
aeb4c132
HX		 3163 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3164 t_alg->algt.alg.aead.setkey = aead_setkey;
3165 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3166 t_alg->algt.alg.aead.decrypt = aead_decrypt;
6cda075a
LC		 3167 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3168 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
24b92ff2 3169 devm_kfree(dev, t_alg);
6cda075a
LC		 3170 return ERR_PTR(-ENOTSUPP);
3171 }
acbf7c62
LN		 3172 break;
3173 case CRYPTO_ALG_TYPE_AHASH:
3174 alg = &t_alg->algt.alg.hash.halg.base;
497f2e6b 3175 alg->cra_init = talitos_cra_init_ahash;
ad4cd51f 3176 alg->cra_exit = talitos_cra_exit;
b286e003
KP		 3177 t_alg->algt.alg.hash.init = ahash_init;
3178 t_alg->algt.alg.hash.update = ahash_update;
3179 t_alg->algt.alg.hash.final = ahash_final;
3180 t_alg->algt.alg.hash.finup = ahash_finup;
3181 t_alg->algt.alg.hash.digest = ahash_digest;
56136631
LC		 3182 if (!strncmp(alg->cra_name, "hmac", 4))
3183 t_alg->algt.alg.hash.setkey = ahash_setkey;
3639ca84
HG		 3184 t_alg->algt.alg.hash.import = ahash_import;
3185 t_alg->algt.alg.hash.export = ahash_export;
b286e003 3186
79b3a418 3187 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
0b2730d8 3188 !strncmp(alg->cra_name, "hmac", 4)) {
24b92ff2 3189 devm_kfree(dev, t_alg);
79b3a418 3190 return ERR_PTR(-ENOTSUPP);
0b2730d8 3191 }
60f208d7 3192 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
79b3a418
LN		 3193 (!strcmp(alg->cra_name, "sha224") ||
3194 !strcmp(alg->cra_name, "hmac(sha224)"))) {
60f208d7
KP		 3195 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3196 t_alg->algt.desc_hdr_template =
3197 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3198 DESC_HDR_SEL0_MDEUA |
3199 DESC_HDR_MODE0_MDEU_SHA256;
3200 }
497f2e6b 3201 break;
1d11911a
KP		 3202 default:
3203 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
24b92ff2 3204 devm_kfree(dev, t_alg);
1d11911a 3205 return ERR_PTR(-EINVAL);
acbf7c62 3206 }
9c4a7965 3207
9c4a7965 3208 alg->cra_module = THIS_MODULE;
b0057763
LC		 3209 if (t_alg->algt.priority)
3210 alg->cra_priority = t_alg->algt.priority;
3211 else
3212 alg->cra_priority = TALITOS_CRA_PRIORITY;
9c4a7965 3213 alg->cra_alignmask = 0;
9c4a7965 3214 alg->cra_ctxsize = sizeof(struct talitos_ctx);
d912bb76 3215 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
9c4a7965 3216
9c4a7965
KP		 3217 t_alg->dev = dev;
3218
3219 return t_alg;
3220}
3221
c3e337f8
KP		 3222static int talitos_probe_irq(struct platform_device *ofdev)
3223{
3224 struct device *dev = &ofdev->dev;
3225 struct device_node *np = ofdev->dev.of_node;
3226 struct talitos_private *priv = dev_get_drvdata(dev);
3227 int err;
dd3c0987 3228 bool is_sec1 = has_ftr_sec1(priv);
c3e337f8
KP		 3229
3230 priv->irq[0] = irq_of_parse_and_map(np, 0);
2cdba3cf 3231 if (!priv->irq[0]) {
c3e337f8
KP		 3232 dev_err(dev, "failed to map irq\n");
3233 return -EINVAL;
3234 }
dd3c0987
LC		 3235 if (is_sec1) {
3236 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3237 dev_driver_string(dev), dev);
3238 goto primary_out;
3239 }
c3e337f8
KP		 3240
3241 priv->irq[1] = irq_of_parse_and_map(np, 1);
3242
3243 /* get the primary irq line */
2cdba3cf 3244 if (!priv->irq[1]) {
dd3c0987 3245 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
c3e337f8
KP		 3246 dev_driver_string(dev), dev);
3247 goto primary_out;
3248 }
3249
dd3c0987 3250 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
c3e337f8
KP		 3251 dev_driver_string(dev), dev);
3252 if (err)
3253 goto primary_out;
3254
3255 /* get the secondary irq line */
dd3c0987 3256 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
c3e337f8
KP		 3257 dev_driver_string(dev), dev);
3258 if (err) {
3259 dev_err(dev, "failed to request secondary irq\n");
3260 irq_dispose_mapping(priv->irq[1]);
2cdba3cf 3261 priv->irq[1] = 0;
c3e337f8
KP		 3262 }
3263
3264 return err;
3265
3266primary_out:
3267 if (err) {
3268 dev_err(dev, "failed to request primary irq\n");
3269 irq_dispose_mapping(priv->irq[0]);
2cdba3cf 3270 priv->irq[0] = 0;
c3e337f8
KP		 3271 }
3272
3273 return err;
3274}
3275
1c48a5c9 3276static int talitos_probe(struct platform_device *ofdev)
9c4a7965
KP		 3277{
3278 struct device *dev = &ofdev->dev;
61c7a080 3279 struct device_node *np = ofdev->dev.of_node;
9c4a7965 3280 struct talitos_private *priv;
9c4a7965 3281 int i, err;
5fa7fa14 3282 int stride;
fd5ea7f0 3283 struct resource *res;
9c4a7965 3284
24b92ff2 3285 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
9c4a7965
KP		 3286 if (!priv)
3287 return -ENOMEM;
3288
f3de9cb1
KH		 3289 INIT_LIST_HEAD(&priv->alg_list);
3290
9c4a7965
KP		 3291 dev_set_drvdata(dev, priv);
3292
3293 priv->ofdev = ofdev;
3294
511d63cb
HG		 3295 spin_lock_init(&priv->reg_lock);
3296
fd5ea7f0
LC		 3297 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3298 if (!res)
3299 return -ENXIO;
3300 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
9c4a7965
KP		 3301 if (!priv->reg) {
3302 dev_err(dev, "failed to of_iomap\n");
3303 err = -ENOMEM;
3304 goto err_out;
3305 }
3306
3307 /* get SEC version capabilities from device tree */
fa14c6cf
LC		 3308 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3309 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3310 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3311 of_property_read_u32(np, "fsl,descriptor-types-mask",
3312 &priv->desc_types);
9c4a7965
KP		 3313
3314 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3315 !priv->exec_units || !priv->desc_types) {
3316 dev_err(dev, "invalid property data in device tree node\n");
3317 err = -EINVAL;
3318 goto err_out;
3319 }
3320
f3c85bc1
LN		 3321 if (of_device_is_compatible(np, "fsl,sec3.0"))
3322 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3323
fe5720e2 3324 if (of_device_is_compatible(np, "fsl,sec2.1"))
60f208d7 3325 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
79b3a418
LN		 3326 TALITOS_FTR_SHA224_HWINIT |
3327 TALITOS_FTR_HMAC_OK;
fe5720e2 3328
21590888
LC		 3329 if (of_device_is_compatible(np, "fsl,sec1.0"))
3330 priv->features |= TALITOS_FTR_SEC1;
3331
5fa7fa14
LC		 3332 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3333 priv->reg_deu = priv->reg + TALITOS12_DEU;
3334 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3335 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3336 stride = TALITOS1_CH_STRIDE;
3337 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3338 priv->reg_deu = priv->reg + TALITOS10_DEU;
3339 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3340 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3341 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3342 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3343 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3344 stride = TALITOS1_CH_STRIDE;
3345 } else {
3346 priv->reg_deu = priv->reg + TALITOS2_DEU;
3347 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3348 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3349 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3350 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3351 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3352 priv->reg_keu = priv->reg + TALITOS2_KEU;
3353 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3354 stride = TALITOS2_CH_STRIDE;
3355 }
3356
dd3c0987
LC		 3357 err = talitos_probe_irq(ofdev);
3358 if (err)
3359 goto err_out;
3360
3361 if (of_device_is_compatible(np, "fsl,sec1.0")) {
9c02e285
LC		 3362 if (priv->num_channels == 1)
3363 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
dd3c0987 3364 (unsigned long)dev);
9c02e285
LC		 3365 else
3366 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3367 (unsigned long)dev);
3368 } else {
3369 if (priv->irq[1]) {
dd3c0987
LC		 3370 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3371 (unsigned long)dev);
3372 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3373 (unsigned long)dev);
9c02e285
LC		 3374 } else if (priv->num_channels == 1) {
3375 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3376 (unsigned long)dev);
3377 } else {
3378 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3379 (unsigned long)dev);
dd3c0987
LC		 3380 }
3381 }
3382
a86854d0
KC		 3383 priv->chan = devm_kcalloc(dev,
3384 priv->num_channels,
3385 sizeof(struct talitos_channel),
3386 GFP_KERNEL);
4b992628
KP		 3387 if (!priv->chan) {
3388 dev_err(dev, "failed to allocate channel management space\n");
9c4a7965
KP		 3389 err = -ENOMEM;
3390 goto err_out;
3391 }
3392
f641dddd
MH		 3393 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3394
c3e337f8 3395 for (i = 0; i < priv->num_channels; i++) {
5fa7fa14 3396 priv->chan[i].reg = priv->reg + stride * (i + 1);
2cdba3cf 3397 if (!priv->irq[1] || !(i & 1))
c3e337f8 3398 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
ad42d5fc 3399
4b992628
KP		 3400 spin_lock_init(&priv->chan[i].head_lock);
3401 spin_lock_init(&priv->chan[i].tail_lock);
9c4a7965 3402
a86854d0
KC		 3403 priv->chan[i].fifo = devm_kcalloc(dev,
3404 priv->fifo_len,
3405 sizeof(struct talitos_request),
3406 GFP_KERNEL);
4b992628 3407 if (!priv->chan[i].fifo) {
9c4a7965
KP		 3408 dev_err(dev, "failed to allocate request fifo %d\n", i);
3409 err = -ENOMEM;
3410 goto err_out;
3411 }
9c4a7965 3412
4b992628
KP		 3413 atomic_set(&priv->chan[i].submit_count,
3414 -(priv->chfifo_len - 1));
f641dddd 3415 }
9c4a7965 3416
81eb024c
KP		 3417 dma_set_mask(dev, DMA_BIT_MASK(36));
3418
9c4a7965
KP		 3419 /* reset and initialize the h/w */
3420 err = init_device(dev);
3421 if (err) {
3422 dev_err(dev, "failed to initialize device\n");
3423 goto err_out;
3424 }
3425
3426 /* register the RNG, if available */
3427 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3428 err = talitos_register_rng(dev);
3429 if (err) {
3430 dev_err(dev, "failed to register hwrng: %d\n", err);
3431 goto err_out;
3432 } else
3433 dev_info(dev, "hwrng\n");
3434 }
3435
3436 /* register crypto algorithms the device supports */
9c4a7965
KP		 3437 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3438 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3439 struct talitos_crypto_alg *t_alg;
aeb4c132 3440 struct crypto_alg *alg = NULL;
9c4a7965
KP		 3441
3442 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3443 if (IS_ERR(t_alg)) {
3444 err = PTR_ERR(t_alg);
0b2730d8 3445 if (err == -ENOTSUPP)
79b3a418 3446 continue;
9c4a7965
KP		 3447 goto err_out;
3448 }
3449
acbf7c62
LN		 3450 switch (t_alg->algt.type) {
3451 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62
LN		 3452 err = crypto_register_alg(
3453 &t_alg->algt.alg.crypto);
aeb4c132 3454 alg = &t_alg->algt.alg.crypto;
acbf7c62 3455 break;
aeb4c132
HX		 3456
3457 case CRYPTO_ALG_TYPE_AEAD:
3458 err = crypto_register_aead(
3459 &t_alg->algt.alg.aead);
3460 alg = &t_alg->algt.alg.aead.base;
3461 break;
3462
acbf7c62
LN		 3463 case CRYPTO_ALG_TYPE_AHASH:
3464 err = crypto_register_ahash(
3465 &t_alg->algt.alg.hash);
aeb4c132 3466 alg = &t_alg->algt.alg.hash.halg.base;
acbf7c62
LN		 3467 break;
3468 }
9c4a7965
KP		 3469 if (err) {
3470 dev_err(dev, "%s alg registration failed\n",
aeb4c132 3471 alg->cra_driver_name);
24b92ff2 3472 devm_kfree(dev, t_alg);
991155ba 3473 } else
9c4a7965 3474 list_add_tail(&t_alg->entry, &priv->alg_list);
9c4a7965
KP		 3475 }
3476 }
5b859b6e
KP		 3477 if (!list_empty(&priv->alg_list))
3478 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3479 (char *)of_get_property(np, "compatible", NULL));
9c4a7965
KP		 3480
3481 return 0;
3482
3483err_out:
3484 talitos_remove(ofdev);
9c4a7965
KP		 3485
3486 return err;
3487}
3488
6c3f975a 3489static const struct of_device_id talitos_match[] = {
0635b7db
LC		 3490#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3491 {
3492 .compatible = "fsl,sec1.0",
3493 },
3494#endif
3495#ifdef CONFIG_CRYPTO_DEV_TALITOS2
9c4a7965
KP		 3496 {
3497 .compatible = "fsl,sec2.0",
3498 },
0635b7db 3499#endif
9c4a7965
KP		 3500 {},
3501};
3502MODULE_DEVICE_TABLE(of, talitos_match);
3503
1c48a5c9 3504static struct platform_driver talitos_driver = {
4018294b
GL		 3505 .driver = {
3506 .name = "talitos",
4018294b
GL		 3507 .of_match_table = talitos_match,
3508 },
9c4a7965 3509 .probe = talitos_probe,
596f1034 3510 .remove = talitos_remove,
9c4a7965
KP		 3511};
3512
741e8c2d 3513module_platform_driver(talitos_driver);
9c4a7965
KP		 3514
3515MODULE_LICENSE("GPL");
3516MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3517MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Linux 6.x block layer and io_uring tree(s)