projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: talitos - use devm_kmalloc()
[linux-2.6-block.git] / drivers / crypto / talitos.c
CommitLineData
9c4a7965
KP		 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
5228f0f7 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
9c4a7965
KP		 5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
5af50730
RH		 35#include <linux/of_address.h>
36#include <linux/of_irq.h>
9c4a7965
KP		 37#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
5a0e3ad6 42#include <linux/slab.h>
9c4a7965
KP		 43
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
3952f17e 46#include <crypto/des.h>
9c4a7965 47#include <crypto/sha.h>
497f2e6b 48#include <crypto/md5.h>
e98014ab 49#include <crypto/internal/aead.h>
9c4a7965 50#include <crypto/authenc.h>
4de9d0b5 51#include <crypto/skcipher.h>
acbf7c62
LN		 52#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
4de9d0b5 54#include <crypto/scatterwalk.h>
9c4a7965
KP		 55
56#include "talitos.h"
57
922f9dc8
LC		 58static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 bool is_sec1)
81eb024c 60{
edc6bd69 61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
922f9dc8
LC		 62 if (!is_sec1)
63 ptr->eptr = upper_32_bits(dma_addr);
81eb024c
KP		 64}
65
340ff60a
HG		 66static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
67 struct talitos_ptr *src_ptr, bool is_sec1)
68{
69 dst_ptr->ptr = src_ptr->ptr;
70 if (!is_sec1)
71 dst_ptr->eptr = src_ptr->eptr;
72}
73
42e8b0d7 74static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
922f9dc8 75 bool is_sec1)
538caf83 76{
922f9dc8 77 if (is_sec1) {
922f9dc8
LC		 78 ptr->len1 = cpu_to_be16(len);
79 } else {
80 ptr->len = cpu_to_be16(len);
81 }
538caf83
LC		 82}
83
922f9dc8
LC		 84static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
85 bool is_sec1)
538caf83 86{
922f9dc8
LC		 87 if (is_sec1)
88 return be16_to_cpu(ptr->len1);
89 else
90 return be16_to_cpu(ptr->len);
538caf83
LC		 91}
92
b096b544
LC		 93static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
94 bool is_sec1)
185eb79f 95{
922f9dc8 96 if (!is_sec1)
b096b544
LC		 97 ptr->j_extent = val;
98}
99
100static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
101{
102 if (!is_sec1)
103 ptr->j_extent |= val;
185eb79f
LC		 104}
105
9c4a7965
KP		 106/*
107 * map virtual single (contiguous) pointer to h/w descriptor pointer
108 */
109static void map_single_talitos_ptr(struct device *dev,
edc6bd69 110 struct talitos_ptr *ptr,
42e8b0d7 111 unsigned int len, void *data,
9c4a7965
KP		 112 enum dma_data_direction dir)
113{
81eb024c 114 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
922f9dc8
LC		 115 struct talitos_private *priv = dev_get_drvdata(dev);
116 bool is_sec1 = has_ftr_sec1(priv);
81eb024c 117
922f9dc8
LC		 118 to_talitos_ptr_len(ptr, len, is_sec1);
119 to_talitos_ptr(ptr, dma_addr, is_sec1);
9c4a7965
KP		 120}
121
122/*
123 * unmap bus single (contiguous) h/w descriptor pointer
124 */
125static void unmap_single_talitos_ptr(struct device *dev,
edc6bd69 126 struct talitos_ptr *ptr,
9c4a7965
KP		 127 enum dma_data_direction dir)
128{
922f9dc8
LC		 129 struct talitos_private *priv = dev_get_drvdata(dev);
130 bool is_sec1 = has_ftr_sec1(priv);
131
edc6bd69 132 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
922f9dc8 133 from_talitos_ptr_len(ptr, is_sec1), dir);
9c4a7965
KP		 134}
135
136static int reset_channel(struct device *dev, int ch)
137{
138 struct talitos_private *priv = dev_get_drvdata(dev);
139 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 140 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 141
dd3c0987
LC		 142 if (is_sec1) {
143 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
144 TALITOS1_CCCR_LO_RESET);
9c4a7965 145
dd3c0987
LC		 146 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
147 TALITOS1_CCCR_LO_RESET) && --timeout)
148 cpu_relax();
149 } else {
150 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
151 TALITOS2_CCCR_RESET);
152
153 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
154 TALITOS2_CCCR_RESET) && --timeout)
155 cpu_relax();
156 }
9c4a7965
KP		 157
158 if (timeout == 0) {
159 dev_err(dev, "failed to reset channel %d\n", ch);
160 return -EIO;
161 }
162
81eb024c 163 /* set 36-bit addressing, done writeback enable and done IRQ enable */
ad42d5fc 164 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
81eb024c 165 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
9c4a7965 166
fe5720e2
KP		 167 /* and ICCR writeback, if available */
168 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
ad42d5fc 169 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
fe5720e2
KP		 170 TALITOS_CCCR_LO_IWSE);
171
9c4a7965
KP		 172 return 0;
173}
174
175static int reset_device(struct device *dev)
176{
177 struct talitos_private *priv = dev_get_drvdata(dev);
178 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987
LC		 179 bool is_sec1 = has_ftr_sec1(priv);
180 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
9c4a7965 181
c3e337f8 182 setbits32(priv->reg + TALITOS_MCR, mcr);
9c4a7965 183
dd3c0987 184 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
9c4a7965
KP		 185 && --timeout)
186 cpu_relax();
187
2cdba3cf 188 if (priv->irq[1]) {
c3e337f8
KP		 189 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
190 setbits32(priv->reg + TALITOS_MCR, mcr);
191 }
192
9c4a7965
KP		 193 if (timeout == 0) {
194 dev_err(dev, "failed to reset device\n");
195 return -EIO;
196 }
197
198 return 0;
199}
200
201/*
202 * Reset and initialize the device
203 */
204static int init_device(struct device *dev)
205{
206 struct talitos_private *priv = dev_get_drvdata(dev);
207 int ch, err;
dd3c0987 208 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965
KP		 209
210 /*
211 * Master reset
212 * errata documentation: warning: certain SEC interrupts
213 * are not fully cleared by writing the MCR:SWR bit,
214 * set bit twice to completely reset
215 */
216 err = reset_device(dev);
217 if (err)
218 return err;
219
220 err = reset_device(dev);
221 if (err)
222 return err;
223
224 /* reset channels */
225 for (ch = 0; ch < priv->num_channels; ch++) {
226 err = reset_channel(dev, ch);
227 if (err)
228 return err;
229 }
230
231 /* enable channel done and error interrupts */
dd3c0987
LC		 232 if (is_sec1) {
233 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
234 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
235 /* disable parity error check in DEU (erroneous? test vect.) */
236 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
237 } else {
238 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
239 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
240 }
9c4a7965 241
fe5720e2
KP		 242 /* disable integrity check error interrupts (use writeback instead) */
243 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
5fa7fa14 244 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
fe5720e2
KP		 245 TALITOS_MDEUICR_LO_ICE);
246
9c4a7965
KP		 247 return 0;
248}
249
250/**
251 * talitos_submit - submits a descriptor to the device for processing
252 * @dev: the SEC device to be used
5228f0f7 253 * @ch: the SEC device channel to be used
9c4a7965
KP		 254 * @desc: the descriptor to be processed by the device
255 * @callback: whom to call when processing is complete
256 * @context: a handle for use by caller (optional)
257 *
258 * desc must contain valid dma-mapped (bus physical) address pointers.
259 * callback must check err and feedback in descriptor header
260 * for device processing status.
261 */
865d5061
HG		 262int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
263 void (*callback)(struct device *dev,
264 struct talitos_desc *desc,
265 void *context, int error),
266 void *context)
9c4a7965
KP		 267{
268 struct talitos_private *priv = dev_get_drvdata(dev);
269 struct talitos_request *request;
5228f0f7 270 unsigned long flags;
9c4a7965 271 int head;
7d607c6a 272 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 273
4b992628 274 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
9c4a7965 275
4b992628 276 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
ec6644d6 277 /* h/w fifo is full */
4b992628 278 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 279 return -EAGAIN;
280 }
281
4b992628
KP		 282 head = priv->chan[ch].head;
283 request = &priv->chan[ch].fifo[head];
ec6644d6 284
9c4a7965 285 /* map descriptor and save caller data */
7d607c6a
LC		 286 if (is_sec1) {
287 desc->hdr1 = desc->hdr;
7d607c6a
LC		 288 request->dma_desc = dma_map_single(dev, &desc->hdr1,
289 TALITOS_DESC_SIZE,
290 DMA_BIDIRECTIONAL);
291 } else {
292 request->dma_desc = dma_map_single(dev, desc,
293 TALITOS_DESC_SIZE,
294 DMA_BIDIRECTIONAL);
295 }
9c4a7965
KP		 296 request->callback = callback;
297 request->context = context;
298
299 /* increment fifo head */
4b992628 300 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
9c4a7965
KP		 301
302 smp_wmb();
303 request->desc = desc;
304
305 /* GO! */
306 wmb();
ad42d5fc
KP		 307 out_be32(priv->chan[ch].reg + TALITOS_FF,
308 upper_32_bits(request->dma_desc));
309 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
a752447a 310 lower_32_bits(request->dma_desc));
9c4a7965 311
4b992628 312 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 313
314 return -EINPROGRESS;
315}
865d5061 316EXPORT_SYMBOL(talitos_submit);
9c4a7965
KP		 317
318/*
319 * process what was done, notify callback of error if not
320 */
321static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
322{
323 struct talitos_private *priv = dev_get_drvdata(dev);
324 struct talitos_request *request, saved_req;
325 unsigned long flags;
326 int tail, status;
7d607c6a 327 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 328
4b992628 329 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
9c4a7965 330
4b992628
KP		 331 tail = priv->chan[ch].tail;
332 while (priv->chan[ch].fifo[tail].desc) {
7d607c6a
LC		 333 __be32 hdr;
334
4b992628 335 request = &priv->chan[ch].fifo[tail];
9c4a7965
KP		 336
337 /* descriptors with their done bits set don't get the error */
338 rmb();
7d607c6a
LC		 339 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
340
341 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
9c4a7965 342 status = 0;
ca38a814 343 else
9c4a7965
KP		 344 if (!error)
345 break;
346 else
347 status = error;
348
349 dma_unmap_single(dev, request->dma_desc,
7d607c6a 350 TALITOS_DESC_SIZE,
e938e465 351 DMA_BIDIRECTIONAL);
9c4a7965
KP		 352
353 /* copy entries so we can call callback outside lock */
354 saved_req.desc = request->desc;
355 saved_req.callback = request->callback;
356 saved_req.context = request->context;
357
358 /* release request entry in fifo */
359 smp_wmb();
360 request->desc = NULL;
361
362 /* increment fifo tail */
4b992628 363 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
9c4a7965 364
4b992628 365 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
ec6644d6 366
4b992628 367 atomic_dec(&priv->chan[ch].submit_count);
ec6644d6 368
9c4a7965
KP		 369 saved_req.callback(dev, saved_req.desc, saved_req.context,
370 status);
371 /* channel may resume processing in single desc error case */
372 if (error && !reset_ch && status == error)
373 return;
4b992628
KP		 374 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
375 tail = priv->chan[ch].tail;
9c4a7965
KP		 376 }
377
4b992628 378 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
9c4a7965
KP		 379}
380
381/*
382 * process completed requests for channels that have done status
383 */
dd3c0987
LC		 384#define DEF_TALITOS1_DONE(name, ch_done_mask) \
385static void talitos1_done_##name(unsigned long data) \
386{ \
387 struct device *dev = (struct device *)data; \
388 struct talitos_private *priv = dev_get_drvdata(dev); \
389 unsigned long flags; \
390 \
391 if (ch_done_mask & 0x10000000) \
392 flush_channel(dev, 0, 0, 0); \
393 if (priv->num_channels == 1) \
394 goto out; \
395 if (ch_done_mask & 0x40000000) \
396 flush_channel(dev, 1, 0, 0); \
397 if (ch_done_mask & 0x00010000) \
398 flush_channel(dev, 2, 0, 0); \
399 if (ch_done_mask & 0x00040000) \
400 flush_channel(dev, 3, 0, 0); \
401 \
402out: \
403 /* At this point, all completed channels have been processed */ \
404 /* Unmask done interrupts for channels completed later on. */ \
405 spin_lock_irqsave(&priv->reg_lock, flags); \
406 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
407 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
408 spin_unlock_irqrestore(&priv->reg_lock, flags); \
409}
410
411DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
412
413#define DEF_TALITOS2_DONE(name, ch_done_mask) \
414static void talitos2_done_##name(unsigned long data) \
c3e337f8
KP		 415{ \
416 struct device *dev = (struct device *)data; \
417 struct talitos_private *priv = dev_get_drvdata(dev); \
511d63cb 418 unsigned long flags; \
c3e337f8
KP		 419 \
420 if (ch_done_mask & 1) \
421 flush_channel(dev, 0, 0, 0); \
422 if (priv->num_channels == 1) \
423 goto out; \
424 if (ch_done_mask & (1 << 2)) \
425 flush_channel(dev, 1, 0, 0); \
426 if (ch_done_mask & (1 << 4)) \
427 flush_channel(dev, 2, 0, 0); \
428 if (ch_done_mask & (1 << 6)) \
429 flush_channel(dev, 3, 0, 0); \
430 \
431out: \
432 /* At this point, all completed channels have been processed */ \
433 /* Unmask done interrupts for channels completed later on. */ \
511d63cb 434 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8 435 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
dd3c0987 436 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
511d63cb 437 spin_unlock_irqrestore(&priv->reg_lock, flags); \
9c4a7965 438}
dd3c0987
LC		 439
440DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
441DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
442DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
9c4a7965
KP		 443
444/*
445 * locate current (offending) descriptor
446 */
3e721aeb 447static u32 current_desc_hdr(struct device *dev, int ch)
9c4a7965
KP		 448{
449 struct talitos_private *priv = dev_get_drvdata(dev);
b62ffd8c 450 int tail, iter;
9c4a7965
KP		 451 dma_addr_t cur_desc;
452
b62ffd8c
HG		 453 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
454 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
9c4a7965 455
b62ffd8c
HG		 456 if (!cur_desc) {
457 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
458 return 0;
459 }
460
461 tail = priv->chan[ch].tail;
462
463 iter = tail;
464 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
465 iter = (iter + 1) & (priv->fifo_len - 1);
466 if (iter == tail) {
9c4a7965 467 dev_err(dev, "couldn't locate current descriptor\n");
3e721aeb 468 return 0;
9c4a7965
KP		 469 }
470 }
471
b62ffd8c 472 return priv->chan[ch].fifo[iter].desc->hdr;
9c4a7965
KP		 473}
474
475/*
476 * user diagnostics; report root cause of error based on execution unit status
477 */
3e721aeb 478static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
9c4a7965
KP		 479{
480 struct talitos_private *priv = dev_get_drvdata(dev);
481 int i;
482
3e721aeb 483 if (!desc_hdr)
ad42d5fc 484 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
3e721aeb
KP		 485
486 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
9c4a7965
KP		 487 case DESC_HDR_SEL0_AFEU:
488 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 489 in_be32(priv->reg_afeu + TALITOS_EUISR),
490 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
9c4a7965
KP		 491 break;
492 case DESC_HDR_SEL0_DEU:
493 dev_err(dev, "DEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 494 in_be32(priv->reg_deu + TALITOS_EUISR),
495 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
9c4a7965
KP		 496 break;
497 case DESC_HDR_SEL0_MDEUA:
498 case DESC_HDR_SEL0_MDEUB:
499 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 500 in_be32(priv->reg_mdeu + TALITOS_EUISR),
501 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 502 break;
503 case DESC_HDR_SEL0_RNG:
504 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
5fa7fa14
LC		 505 in_be32(priv->reg_rngu + TALITOS_ISR),
506 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
9c4a7965
KP		 507 break;
508 case DESC_HDR_SEL0_PKEU:
509 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 510 in_be32(priv->reg_pkeu + TALITOS_EUISR),
511 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 512 break;
513 case DESC_HDR_SEL0_AESU:
514 dev_err(dev, "AESUISR 0x%08x_%08x\n",
5fa7fa14
LC		 515 in_be32(priv->reg_aesu + TALITOS_EUISR),
516 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
9c4a7965
KP		 517 break;
518 case DESC_HDR_SEL0_CRCU:
519 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 520 in_be32(priv->reg_crcu + TALITOS_EUISR),
521 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 522 break;
523 case DESC_HDR_SEL0_KEU:
524 dev_err(dev, "KEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 525 in_be32(priv->reg_pkeu + TALITOS_EUISR),
526 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
9c4a7965
KP		 527 break;
528 }
529
3e721aeb 530 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
9c4a7965
KP		 531 case DESC_HDR_SEL1_MDEUA:
532 case DESC_HDR_SEL1_MDEUB:
533 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
5fa7fa14
LC		 534 in_be32(priv->reg_mdeu + TALITOS_EUISR),
535 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
9c4a7965
KP		 536 break;
537 case DESC_HDR_SEL1_CRCU:
538 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
5fa7fa14
LC		 539 in_be32(priv->reg_crcu + TALITOS_EUISR),
540 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
9c4a7965
KP		 541 break;
542 }
543
544 for (i = 0; i < 8; i++)
545 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
ad42d5fc
KP		 546 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
547 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
9c4a7965
KP		 548}
549
550/*
551 * recover from error interrupts
552 */
5e718a09 553static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
9c4a7965 554{
9c4a7965
KP		 555 struct talitos_private *priv = dev_get_drvdata(dev);
556 unsigned int timeout = TALITOS_TIMEOUT;
dd3c0987 557 int ch, error, reset_dev = 0;
42e8b0d7 558 u32 v_lo;
dd3c0987
LC		 559 bool is_sec1 = has_ftr_sec1(priv);
560 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
9c4a7965
KP		 561
562 for (ch = 0; ch < priv->num_channels; ch++) {
563 /* skip channels without errors */
dd3c0987
LC		 564 if (is_sec1) {
565 /* bits 29, 31, 17, 19 */
566 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
567 continue;
568 } else {
569 if (!(isr & (1 << (ch * 2 + 1))))
570 continue;
571 }
9c4a7965
KP		 572
573 error = -EINVAL;
574
ad42d5fc 575 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
9c4a7965
KP		 576
577 if (v_lo & TALITOS_CCPSR_LO_DOF) {
578 dev_err(dev, "double fetch fifo overflow error\n");
579 error = -EAGAIN;
580 reset_ch = 1;
581 }
582 if (v_lo & TALITOS_CCPSR_LO_SOF) {
583 /* h/w dropped descriptor */
584 dev_err(dev, "single fetch fifo overflow error\n");
585 error = -EAGAIN;
586 }
587 if (v_lo & TALITOS_CCPSR_LO_MDTE)
588 dev_err(dev, "master data transfer error\n");
589 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
4d9b3a5b 590 dev_err(dev, is_sec1 ? "pointer not complete error\n"
dd3c0987 591 : "s/g data length zero error\n");
9c4a7965 592 if (v_lo & TALITOS_CCPSR_LO_FPZ)
dd3c0987
LC		 593 dev_err(dev, is_sec1 ? "parity error\n"
594 : "fetch pointer zero error\n");
9c4a7965
KP		 595 if (v_lo & TALITOS_CCPSR_LO_IDH)
596 dev_err(dev, "illegal descriptor header error\n");
597 if (v_lo & TALITOS_CCPSR_LO_IEU)
dd3c0987
LC		 598 dev_err(dev, is_sec1 ? "static assignment error\n"
599 : "invalid exec unit error\n");
9c4a7965 600 if (v_lo & TALITOS_CCPSR_LO_EU)
3e721aeb 601 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
dd3c0987
LC		 602 if (!is_sec1) {
603 if (v_lo & TALITOS_CCPSR_LO_GB)
604 dev_err(dev, "gather boundary error\n");
605 if (v_lo & TALITOS_CCPSR_LO_GRL)
606 dev_err(dev, "gather return/length error\n");
607 if (v_lo & TALITOS_CCPSR_LO_SB)
608 dev_err(dev, "scatter boundary error\n");
609 if (v_lo & TALITOS_CCPSR_LO_SRL)
610 dev_err(dev, "scatter return/length error\n");
611 }
9c4a7965
KP		 612
613 flush_channel(dev, ch, error, reset_ch);
614
615 if (reset_ch) {
616 reset_channel(dev, ch);
617 } else {
ad42d5fc 618 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
dd3c0987 619 TALITOS2_CCCR_CONT);
ad42d5fc
KP		 620 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
621 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
dd3c0987 622 TALITOS2_CCCR_CONT) && --timeout)
9c4a7965
KP		 623 cpu_relax();
624 if (timeout == 0) {
625 dev_err(dev, "failed to restart channel %d\n",
626 ch);
627 reset_dev = 1;
628 }
629 }
630 }
dd3c0987
LC		 631 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
632 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
633 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
634 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
635 isr, isr_lo);
636 else
637 dev_err(dev, "done overflow, internal time out, or "
638 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
9c4a7965
KP		 639
640 /* purge request queues */
641 for (ch = 0; ch < priv->num_channels; ch++)
642 flush_channel(dev, ch, -EIO, 1);
643
644 /* reset and reinitialize the device */
645 init_device(dev);
646 }
647}
648
dd3c0987
LC		 649#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
650static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
651{ \
652 struct device *dev = data; \
653 struct talitos_private *priv = dev_get_drvdata(dev); \
654 u32 isr, isr_lo; \
655 unsigned long flags; \
656 \
657 spin_lock_irqsave(&priv->reg_lock, flags); \
658 isr = in_be32(priv->reg + TALITOS_ISR); \
659 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
660 /* Acknowledge interrupt */ \
661 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
662 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
663 \
664 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
665 spin_unlock_irqrestore(&priv->reg_lock, flags); \
666 talitos_error(dev, isr & ch_err_mask, isr_lo); \
667 } \
668 else { \
669 if (likely(isr & ch_done_mask)) { \
670 /* mask further done interrupts. */ \
671 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
672 /* done_task will unmask done interrupts at exit */ \
673 tasklet_schedule(&priv->done_task[tlet]); \
674 } \
675 spin_unlock_irqrestore(&priv->reg_lock, flags); \
676 } \
677 \
678 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
679 IRQ_NONE; \
680}
681
682DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
683
684#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
685static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
c3e337f8
KP		 686{ \
687 struct device *dev = data; \
688 struct talitos_private *priv = dev_get_drvdata(dev); \
689 u32 isr, isr_lo; \
511d63cb 690 unsigned long flags; \
c3e337f8 691 \
511d63cb 692 spin_lock_irqsave(&priv->reg_lock, flags); \
c3e337f8
KP		 693 isr = in_be32(priv->reg + TALITOS_ISR); \
694 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
695 /* Acknowledge interrupt */ \
696 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
697 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
698 \
511d63cb
HG		 699 if (unlikely(isr & ch_err_mask || isr_lo)) { \
700 spin_unlock_irqrestore(&priv->reg_lock, flags); \
701 talitos_error(dev, isr & ch_err_mask, isr_lo); \
702 } \
703 else { \
c3e337f8
KP		 704 if (likely(isr & ch_done_mask)) { \
705 /* mask further done interrupts. */ \
706 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
707 /* done_task will unmask done interrupts at exit */ \
708 tasklet_schedule(&priv->done_task[tlet]); \
709 } \
511d63cb
HG		 710 spin_unlock_irqrestore(&priv->reg_lock, flags); \
711 } \
c3e337f8
KP		 712 \
713 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
714 IRQ_NONE; \
9c4a7965 715}
dd3c0987
LC		 716
717DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
718DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
719 0)
720DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
721 1)
9c4a7965
KP		 722
723/*
724 * hwrng
725 */
726static int talitos_rng_data_present(struct hwrng *rng, int wait)
727{
728 struct device *dev = (struct device *)rng->priv;
729 struct talitos_private *priv = dev_get_drvdata(dev);
730 u32 ofl;
731 int i;
732
733 for (i = 0; i < 20; i++) {
5fa7fa14 734 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
9c4a7965
KP		 735 TALITOS_RNGUSR_LO_OFL;
736 if (ofl || !wait)
737 break;
738 udelay(10);
739 }
740
741 return !!ofl;
742}
743
744static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
745{
746 struct device *dev = (struct device *)rng->priv;
747 struct talitos_private *priv = dev_get_drvdata(dev);
748
749 /* rng fifo requires 64-bit accesses */
5fa7fa14
LC		 750 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
751 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
9c4a7965
KP		 752
753 return sizeof(u32);
754}
755
756static int talitos_rng_init(struct hwrng *rng)
757{
758 struct device *dev = (struct device *)rng->priv;
759 struct talitos_private *priv = dev_get_drvdata(dev);
760 unsigned int timeout = TALITOS_TIMEOUT;
761
5fa7fa14
LC		 762 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
763 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
764 & TALITOS_RNGUSR_LO_RD)
9c4a7965
KP		 765 && --timeout)
766 cpu_relax();
767 if (timeout == 0) {
768 dev_err(dev, "failed to reset rng hw\n");
769 return -ENODEV;
770 }
771
772 /* start generating */
5fa7fa14 773 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
9c4a7965
KP		 774
775 return 0;
776}
777
778static int talitos_register_rng(struct device *dev)
779{
780 struct talitos_private *priv = dev_get_drvdata(dev);
35a3bb3d 781 int err;
9c4a7965
KP		 782
783 priv->rng.name = dev_driver_string(dev),
784 priv->rng.init = talitos_rng_init,
785 priv->rng.data_present = talitos_rng_data_present,
786 priv->rng.data_read = talitos_rng_data_read,
787 priv->rng.priv = (unsigned long)dev;
788
35a3bb3d
AS		 789 err = hwrng_register(&priv->rng);
790 if (!err)
791 priv->rng_registered = true;
792
793 return err;
9c4a7965
KP		 794}
795
796static void talitos_unregister_rng(struct device *dev)
797{
798 struct talitos_private *priv = dev_get_drvdata(dev);
799
35a3bb3d
AS		 800 if (!priv->rng_registered)
801 return;
802
9c4a7965 803 hwrng_unregister(&priv->rng);
35a3bb3d 804 priv->rng_registered = false;
9c4a7965
KP		 805}
806
807/*
808 * crypto alg
809 */
810#define TALITOS_CRA_PRIORITY 3000
7405c8d7
LC		 811/*
812 * Defines a priority for doing AEAD with descriptors type
813 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
814 */
815#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
03d2c511 816#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
3952f17e 817#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
70bcaca7 818
9c4a7965
KP		 819struct talitos_ctx {
820 struct device *dev;
5228f0f7 821 int ch;
9c4a7965
KP		 822 __be32 desc_hdr_template;
823 u8 key[TALITOS_MAX_KEY_SIZE];
70bcaca7 824 u8 iv[TALITOS_MAX_IV_LENGTH];
9c4a7965
KP		 825 unsigned int keylen;
826 unsigned int enckeylen;
827 unsigned int authkeylen;
9c4a7965
KP		 828};
829
497f2e6b
LN		 830#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
831#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
832
833struct talitos_ahash_req_ctx {
60f208d7 834 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
497f2e6b
LN		 835 unsigned int hw_context_size;
836 u8 buf[HASH_MAX_BLOCK_SIZE];
837 u8 bufnext[HASH_MAX_BLOCK_SIZE];
60f208d7 838 unsigned int swinit;
497f2e6b
LN		 839 unsigned int first;
840 unsigned int last;
841 unsigned int to_hash_later;
42e8b0d7 842 unsigned int nbuf;
497f2e6b
LN		 843 struct scatterlist bufsl[2];
844 struct scatterlist *psrc;
845};
846
3639ca84
HG		 847struct talitos_export_state {
848 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
849 u8 buf[HASH_MAX_BLOCK_SIZE];
850 unsigned int swinit;
851 unsigned int first;
852 unsigned int last;
853 unsigned int to_hash_later;
854 unsigned int nbuf;
855};
856
56af8cd4
LN		 857static int aead_setkey(struct crypto_aead *authenc,
858 const u8 *key, unsigned int keylen)
9c4a7965
KP		 859{
860 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
c306a98d 861 struct crypto_authenc_keys keys;
9c4a7965 862
c306a98d 863 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
9c4a7965
KP		 864 goto badkey;
865
c306a98d 866 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
9c4a7965
KP		 867 goto badkey;
868
c306a98d
MK		 869 memcpy(ctx->key, keys.authkey, keys.authkeylen);
870 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
9c4a7965 871
c306a98d
MK		 872 ctx->keylen = keys.authkeylen + keys.enckeylen;
873 ctx->enckeylen = keys.enckeylen;
874 ctx->authkeylen = keys.authkeylen;
9c4a7965
KP		 875
876 return 0;
877
878badkey:
879 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
880 return -EINVAL;
881}
882
883/*
56af8cd4 884 * talitos_edesc - s/w-extended descriptor
9c4a7965
KP		 885 * @src_nents: number of segments in input scatterlist
886 * @dst_nents: number of segments in output scatterlist
aeb4c132 887 * @icv_ool: whether ICV is out-of-line
79fd31d3 888 * @iv_dma: dma address of iv for checking continuity and link table
9c4a7965 889 * @dma_len: length of dma mapped link_tbl space
6f65f6ac 890 * @dma_link_tbl: bus physical address of link_tbl/buf
9c4a7965 891 * @desc: h/w descriptor
6f65f6ac
LC		 892 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
893 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
9c4a7965
KP		 894 *
895 * if decrypting (with authcheck), or either one of src_nents or dst_nents
896 * is greater than 1, an integrity check value is concatenated to the end
897 * of link_tbl data
898 */
56af8cd4 899struct talitos_edesc {
9c4a7965
KP		 900 int src_nents;
901 int dst_nents;
aeb4c132 902 bool icv_ool;
79fd31d3 903 dma_addr_t iv_dma;
9c4a7965
KP		 904 int dma_len;
905 dma_addr_t dma_link_tbl;
906 struct talitos_desc desc;
6f65f6ac
LC		 907 union {
908 struct talitos_ptr link_tbl[0];
909 u8 buf[0];
910 };
9c4a7965
KP		 911};
912
4de9d0b5
LN		 913static void talitos_sg_unmap(struct device *dev,
914 struct talitos_edesc *edesc,
915 struct scatterlist *src,
6a1e8d14
LC		 916 struct scatterlist *dst,
917 unsigned int len, unsigned int offset)
4de9d0b5 918{
6a1e8d14
LC		 919 struct talitos_private *priv = dev_get_drvdata(dev);
920 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 921 unsigned int src_nents = edesc->src_nents ? : 1;
922 unsigned int dst_nents = edesc->dst_nents ? : 1;
923
6a1e8d14
LC		 924 if (is_sec1 && dst && dst_nents > 1) {
925 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
926 len, DMA_FROM_DEVICE);
927 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
928 offset);
929 }
4de9d0b5 930 if (src != dst) {
6a1e8d14
LC		 931 if (src_nents == 1 || !is_sec1)
932 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
4de9d0b5 933
6a1e8d14 934 if (dst && (dst_nents == 1 || !is_sec1))
b8a011d4 935 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
6a1e8d14 936 } else if (src_nents == 1 || !is_sec1) {
b8a011d4 937 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
246a87cd
LC		 938 }
939}
940
9c4a7965 941static void ipsec_esp_unmap(struct device *dev,
56af8cd4 942 struct talitos_edesc *edesc,
9c4a7965
KP		 943 struct aead_request *areq)
944{
549bd8bc
LC		 945 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
946 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
947 unsigned int ivsize = crypto_aead_ivsize(aead);
948
949 if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)
950 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
951 DMA_FROM_DEVICE);
9c4a7965
KP		 952 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
953 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
954 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
955
6a1e8d14
LC		 956 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
957 areq->assoclen);
9c4a7965
KP		 958
959 if (edesc->dma_len)
960 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
961 DMA_BIDIRECTIONAL);
549bd8bc
LC		 962
963 if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
964 unsigned int dst_nents = edesc->dst_nents ? : 1;
965
966 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
967 areq->assoclen + areq->cryptlen - ivsize);
968 }
9c4a7965
KP		 969}
970
971/*
972 * ipsec_esp descriptor callbacks
973 */
974static void ipsec_esp_encrypt_done(struct device *dev,
975 struct talitos_desc *desc, void *context,
976 int err)
977{
549bd8bc
LC		 978 struct talitos_private *priv = dev_get_drvdata(dev);
979 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 980 struct aead_request *areq = context;
9c4a7965 981 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 982 unsigned int authsize = crypto_aead_authsize(authenc);
19bbbc63 983 struct talitos_edesc *edesc;
9c4a7965
KP		 984 struct scatterlist *sg;
985 void *icvdata;
986
19bbbc63
KP		 987 edesc = container_of(desc, struct talitos_edesc, desc);
988
9c4a7965
KP		 989 ipsec_esp_unmap(dev, edesc, areq);
990
991 /* copy the generated ICV to dst */
aeb4c132 992 if (edesc->icv_ool) {
549bd8bc
LC		 993 if (is_sec1)
994 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
995 else
996 icvdata = &edesc->link_tbl[edesc->src_nents +
997 edesc->dst_nents + 2];
9c4a7965 998 sg = sg_last(areq->dst, edesc->dst_nents);
aeb4c132
HX		 999 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1000 icvdata, authsize);
9c4a7965
KP		 1001 }
1002
1003 kfree(edesc);
1004
1005 aead_request_complete(areq, err);
1006}
1007
fe5720e2 1008static void ipsec_esp_decrypt_swauth_done(struct device *dev,
e938e465
KP		 1009 struct talitos_desc *desc,
1010 void *context, int err)
9c4a7965
KP		 1011{
1012 struct aead_request *req = context;
9c4a7965 1013 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1014 unsigned int authsize = crypto_aead_authsize(authenc);
19bbbc63 1015 struct talitos_edesc *edesc;
9c4a7965 1016 struct scatterlist *sg;
aeb4c132 1017 char *oicv, *icv;
549bd8bc
LC		 1018 struct talitos_private *priv = dev_get_drvdata(dev);
1019 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965 1020
19bbbc63
KP		 1021 edesc = container_of(desc, struct talitos_edesc, desc);
1022
9c4a7965
KP		 1023 ipsec_esp_unmap(dev, edesc, req);
1024
1025 if (!err) {
1026 /* auth check */
9c4a7965 1027 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
aeb4c132
HX		 1028 icv = (char *)sg_virt(sg) + sg->length - authsize;
1029
1030 if (edesc->dma_len) {
549bd8bc
LC		 1031 if (is_sec1)
1032 oicv = (char *)&edesc->dma_link_tbl +
1033 req->assoclen + req->cryptlen;
1034 else
1035 oicv = (char *)
1036 &edesc->link_tbl[edesc->src_nents +
aeb4c132
HX		 1037 edesc->dst_nents + 2];
1038 if (edesc->icv_ool)
1039 icv = oicv + authsize;
1040 } else
1041 oicv = (char *)&edesc->link_tbl[0];
1042
79960943 1043 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
9c4a7965
KP		 1044 }
1045
1046 kfree(edesc);
1047
1048 aead_request_complete(req, err);
1049}
1050
fe5720e2 1051static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
e938e465
KP		 1052 struct talitos_desc *desc,
1053 void *context, int err)
fe5720e2
KP		 1054{
1055 struct aead_request *req = context;
19bbbc63
KP		 1056 struct talitos_edesc *edesc;
1057
1058 edesc = container_of(desc, struct talitos_edesc, desc);
fe5720e2
KP		 1059
1060 ipsec_esp_unmap(dev, edesc, req);
1061
1062 /* check ICV auth status */
e938e465
KP		 1063 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1064 DESC_HDR_LO_ICCR1_PASS))
1065 err = -EBADMSG;
fe5720e2
KP		 1066
1067 kfree(edesc);
1068
1069 aead_request_complete(req, err);
1070}
1071
9c4a7965
KP		 1072/*
1073 * convert scatterlist to SEC h/w link table format
1074 * stop at cryptlen bytes
1075 */
aeb4c132
HX		 1076static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1077 unsigned int offset, int cryptlen,
1078 struct talitos_ptr *link_tbl_ptr)
9c4a7965 1079{
70bcaca7 1080 int n_sg = sg_count;
aeb4c132 1081 int count = 0;
70bcaca7 1082
aeb4c132
HX		 1083 while (cryptlen && sg && n_sg--) {
1084 unsigned int len = sg_dma_len(sg);
9c4a7965 1085
aeb4c132
HX		 1086 if (offset >= len) {
1087 offset -= len;
1088 goto next;
1089 }
1090
1091 len -= offset;
1092
1093 if (len > cryptlen)
1094 len = cryptlen;
1095
1096 to_talitos_ptr(link_tbl_ptr + count,
1097 sg_dma_address(sg) + offset, 0);
b096b544
LC		 1098 to_talitos_ptr_len(link_tbl_ptr + count, len, 0);
1099 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
aeb4c132
HX		 1100 count++;
1101 cryptlen -= len;
1102 offset = 0;
1103
1104next:
1105 sg = sg_next(sg);
70bcaca7 1106 }
9c4a7965
KP		 1107
1108 /* tag end of link table */
aeb4c132 1109 if (count > 0)
b096b544
LC		 1110 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1111 DESC_PTR_LNKTBL_RETURN, 0);
70bcaca7 1112
aeb4c132
HX		 1113 return count;
1114}
1115
5b2cf268 1116static int talitos_sg_map(struct device *dev, struct scatterlist *src,
6a1e8d14
LC		 1117 unsigned int len, struct talitos_edesc *edesc,
1118 struct talitos_ptr *ptr,
1119 int sg_count, unsigned int offset, int tbl_off)
246a87cd 1120{
246a87cd
LC		 1121 struct talitos_private *priv = dev_get_drvdata(dev);
1122 bool is_sec1 = has_ftr_sec1(priv);
1123
1124 to_talitos_ptr_len(ptr, len, is_sec1);
1125
6a1e8d14
LC		 1126 if (sg_count == 1) {
1127 to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1);
1128 return sg_count;
246a87cd 1129 }
246a87cd 1130 if (is_sec1) {
6a1e8d14
LC		 1131 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1);
1132 return sg_count;
246a87cd 1133 }
6a1e8d14
LC		 1134 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1135 &edesc->link_tbl[tbl_off]);
1136 if (sg_count == 1) {
1137 /* Only one segment now, so no link tbl needed*/
1138 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1139 return sg_count;
1140 }
1141 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1142 tbl_off * sizeof(struct talitos_ptr), is_sec1);
1143 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1144
1145 return sg_count;
246a87cd
LC		 1146}
1147
9c4a7965
KP		 1148/*
1149 * fill in and submit ipsec_esp descriptor
1150 */
56af8cd4 1151static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
aeb4c132
HX		 1152 void (*callback)(struct device *dev,
1153 struct talitos_desc *desc,
1154 void *context, int error))
9c4a7965
KP		 1155{
1156 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
aeb4c132 1157 unsigned int authsize = crypto_aead_authsize(aead);
9c4a7965
KP		 1158 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1159 struct device *dev = ctx->dev;
1160 struct talitos_desc *desc = &edesc->desc;
1161 unsigned int cryptlen = areq->cryptlen;
e41256f1 1162 unsigned int ivsize = crypto_aead_ivsize(aead);
aeb4c132 1163 int tbl_off = 0;
fa86a267 1164 int sg_count, ret;
fe5720e2 1165 int sg_link_tbl_len;
549bd8bc
LC		 1166 bool sync_needed = false;
1167 struct talitos_private *priv = dev_get_drvdata(dev);
1168 bool is_sec1 = has_ftr_sec1(priv);
9c4a7965
KP		 1169
1170 /* hmac key */
1171 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
a2b35aa8 1172 DMA_TO_DEVICE);
79fd31d3 1173
549bd8bc
LC		 1174 sg_count = edesc->src_nents ?: 1;
1175 if (is_sec1 && sg_count > 1)
1176 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1177 areq->assoclen + cryptlen);
1178 else
1179 sg_count = dma_map_sg(dev, areq->src, sg_count,
1180 (areq->src == areq->dst) ?
1181 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
79fd31d3 1182
549bd8bc
LC		 1183 /* hmac data */
1184 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1185 &desc->ptr[1], sg_count, 0, tbl_off);
340ff60a 1186
549bd8bc 1187 if (ret > 1) {
340ff60a 1188 tbl_off += ret;
549bd8bc 1189 sync_needed = true;
79fd31d3
HG		 1190 }
1191
9c4a7965 1192 /* cipher iv */
549bd8bc
LC		 1193 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1194 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1);
1195 to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1);
549bd8bc
LC		 1196 } else {
1197 to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1);
1198 to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1);
549bd8bc 1199 }
9c4a7965
KP		 1200
1201 /* cipher key */
549bd8bc
LC		 1202 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1203 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1204 (char *)&ctx->key + ctx->authkeylen,
1205 DMA_TO_DEVICE);
1206 else
1207 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen,
1208 (char *)&ctx->key + ctx->authkeylen,
1209 DMA_TO_DEVICE);
9c4a7965
KP		 1210
1211 /*
1212 * cipher in
1213 * map and adjust cipher len to aead request cryptlen.
1214 * extent is bytes of HMAC postpended to ciphertext,
1215 * typically 12 for ipsec
1216 */
549bd8bc 1217 to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1);
9c4a7965 1218
aeb4c132 1219 sg_link_tbl_len = cryptlen;
aeb4c132 1220
549bd8bc
LC		 1221 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1222 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1223
1224 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1225 sg_link_tbl_len += authsize;
340ff60a 1226 }
9c4a7965 1227
fbb22137
LC		 1228 ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc,
1229 &desc->ptr[4], sg_count, areq->assoclen, tbl_off);
549bd8bc 1230
ec8c7d14
LC		 1231 if (ret > 1) {
1232 tbl_off += ret;
549bd8bc
LC		 1233 sync_needed = true;
1234 }
9c4a7965 1235
549bd8bc
LC		 1236 /* cipher out */
1237 if (areq->src != areq->dst) {
1238 sg_count = edesc->dst_nents ? : 1;
1239 if (!is_sec1 || sg_count == 1)
1240 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1241 }
9c4a7965 1242
e04a61be
LC		 1243 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1244 sg_count, areq->assoclen, tbl_off);
aeb4c132 1245
549bd8bc
LC		 1246 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1247 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
aeb4c132 1248
e04a61be
LC		 1249 /* ICV data */
1250 if (ret > 1) {
1251 tbl_off += ret;
aeb4c132 1252 edesc->icv_ool = true;
549bd8bc
LC		 1253 sync_needed = true;
1254
1255 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1256 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1257 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1258 sizeof(struct talitos_ptr) + authsize;
1259
1260 /* Add an entry to the link table for ICV data */
e04a61be 1261 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
549bd8bc
LC		 1262 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1263 is_sec1);
1264 to_talitos_ptr_len(tbl_ptr, authsize, is_sec1);
1265
1266 /* icv data follows link tables */
1267 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1268 is_sec1);
e04a61be
LC		 1269 } else {
1270 dma_addr_t addr = edesc->dma_link_tbl;
1271
1272 if (is_sec1)
1273 addr += areq->assoclen + cryptlen;
1274 else
1275 addr += sizeof(struct talitos_ptr) * tbl_off;
1276
1277 to_talitos_ptr(&desc->ptr[6], addr, is_sec1);
1278 to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1);
1279 }
1280 } else if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
1281 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1282 &desc->ptr[6], sg_count, areq->assoclen +
1283 cryptlen,
1284 tbl_off);
1285 if (ret > 1) {
1286 tbl_off += ret;
1287 edesc->icv_ool = true;
1288 sync_needed = true;
1289 } else {
1290 edesc->icv_ool = false;
549bd8bc 1291 }
340ff60a 1292 } else {
549bd8bc
LC		 1293 edesc->icv_ool = false;
1294 }
1295
9c4a7965 1296 /* iv out */
549bd8bc
LC		 1297 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1298 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1299 DMA_FROM_DEVICE);
1300
1301 if (sync_needed)
1302 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1303 edesc->dma_len,
1304 DMA_BIDIRECTIONAL);
9c4a7965 1305
5228f0f7 1306 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
fa86a267
KP		 1307 if (ret != -EINPROGRESS) {
1308 ipsec_esp_unmap(dev, edesc, areq);
1309 kfree(edesc);
1310 }
1311 return ret;
9c4a7965
KP		 1312}
1313
9c4a7965 1314/*
56af8cd4 1315 * allocate and map the extended descriptor
9c4a7965 1316 */
4de9d0b5
LN		 1317static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1318 struct scatterlist *src,
1319 struct scatterlist *dst,
79fd31d3
HG		 1320 u8 *iv,
1321 unsigned int assoclen,
4de9d0b5
LN		 1322 unsigned int cryptlen,
1323 unsigned int authsize,
79fd31d3 1324 unsigned int ivsize,
4de9d0b5 1325 int icv_stashing,
62293a37
HG		 1326 u32 cryptoflags,
1327 bool encrypt)
9c4a7965 1328{
56af8cd4 1329 struct talitos_edesc *edesc;
6a1e8d14 1330 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
79fd31d3 1331 dma_addr_t iv_dma = 0;
4de9d0b5 1332 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
586725f8 1333 GFP_ATOMIC;
6f65f6ac
LC		 1334 struct talitos_private *priv = dev_get_drvdata(dev);
1335 bool is_sec1 = has_ftr_sec1(priv);
1336 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
8e409fe1 1337 void *err;
9c4a7965 1338
6f65f6ac 1339 if (cryptlen + authsize > max_len) {
4de9d0b5 1340 dev_err(dev, "length exceeds h/w max limit\n");
9c4a7965
KP		 1341 return ERR_PTR(-EINVAL);
1342 }
1343
935e99a3 1344 if (ivsize)
79fd31d3
HG		 1345 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1346
62293a37 1347 if (!dst || dst == src) {
6a1e8d14
LC		 1348 src_len = assoclen + cryptlen + authsize;
1349 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1350 if (src_nents < 0) {
1351 dev_err(dev, "Invalid number of src SG.\n");
1352 err = ERR_PTR(-EINVAL);
1353 goto error_sg;
1354 }
62293a37
HG		 1355 src_nents = (src_nents == 1) ? 0 : src_nents;
1356 dst_nents = dst ? src_nents : 0;
6a1e8d14 1357 dst_len = 0;
62293a37 1358 } else { /* dst && dst != src*/
6a1e8d14
LC		 1359 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1360 src_nents = sg_nents_for_len(src, src_len);
8e409fe1
LC		 1361 if (src_nents < 0) {
1362 dev_err(dev, "Invalid number of src SG.\n");
1363 err = ERR_PTR(-EINVAL);
1364 goto error_sg;
1365 }
62293a37 1366 src_nents = (src_nents == 1) ? 0 : src_nents;
6a1e8d14
LC		 1367 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1368 dst_nents = sg_nents_for_len(dst, dst_len);
8e409fe1
LC		 1369 if (dst_nents < 0) {
1370 dev_err(dev, "Invalid number of dst SG.\n");
1371 err = ERR_PTR(-EINVAL);
1372 goto error_sg;
1373 }
62293a37 1374 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
9c4a7965
KP		 1375 }
1376
1377 /*
1378 * allocate space for base edesc plus the link tables,
aeb4c132
HX		 1379 * allowing for two separate entries for AD and generated ICV (+ 2),
1380 * and space for two sets of ICVs (stashed and generated)
9c4a7965 1381 */
56af8cd4 1382 alloc_len = sizeof(struct talitos_edesc);
aeb4c132 1383 if (src_nents || dst_nents) {
6f65f6ac 1384 if (is_sec1)
6a1e8d14
LC		 1385 dma_len = (src_nents ? src_len : 0) +
1386 (dst_nents ? dst_len : 0);
6f65f6ac 1387 else
aeb4c132
HX		 1388 dma_len = (src_nents + dst_nents + 2) *
1389 sizeof(struct talitos_ptr) + authsize * 2;
9c4a7965
KP		 1390 alloc_len += dma_len;
1391 } else {
1392 dma_len = 0;
4de9d0b5 1393 alloc_len += icv_stashing ? authsize : 0;
9c4a7965
KP		 1394 }
1395
586725f8 1396 edesc = kmalloc(alloc_len, GFP_DMA | flags);
9c4a7965 1397 if (!edesc) {
4de9d0b5 1398 dev_err(dev, "could not allocate edescriptor\n");
8e409fe1
LC		 1399 err = ERR_PTR(-ENOMEM);
1400 goto error_sg;
9c4a7965 1401 }
e4a647c4 1402 memset(&edesc->desc, 0, sizeof(edesc->desc));
9c4a7965
KP		 1403
1404 edesc->src_nents = src_nents;
1405 edesc->dst_nents = dst_nents;
79fd31d3 1406 edesc->iv_dma = iv_dma;
9c4a7965 1407 edesc->dma_len = dma_len;
497f2e6b
LN		 1408 if (dma_len)
1409 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1410 edesc->dma_len,
1411 DMA_BIDIRECTIONAL);
9c4a7965
KP		 1412
1413 return edesc;
8e409fe1
LC		 1414error_sg:
1415 if (iv_dma)
1416 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1417 return err;
9c4a7965
KP		 1418}
1419
79fd31d3 1420static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
62293a37 1421 int icv_stashing, bool encrypt)
4de9d0b5
LN		 1422{
1423 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
aeb4c132 1424 unsigned int authsize = crypto_aead_authsize(authenc);
4de9d0b5 1425 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
79fd31d3 1426 unsigned int ivsize = crypto_aead_ivsize(authenc);
4de9d0b5 1427
aeb4c132 1428 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1429 iv, areq->assoclen, areq->cryptlen,
aeb4c132 1430 authsize, ivsize, icv_stashing,
62293a37 1431 areq->base.flags, encrypt);
4de9d0b5
LN		 1432}
1433
56af8cd4 1434static int aead_encrypt(struct aead_request *req)
9c4a7965
KP		 1435{
1436 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1437 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
56af8cd4 1438 struct talitos_edesc *edesc;
9c4a7965
KP		 1439
1440 /* allocate extended descriptor */
62293a37 1441 edesc = aead_edesc_alloc(req, req->iv, 0, true);
9c4a7965
KP		 1442 if (IS_ERR(edesc))
1443 return PTR_ERR(edesc);
1444
1445 /* set encrypt */
70bcaca7 1446 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965 1447
aeb4c132 1448 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
9c4a7965
KP		 1449}
1450
56af8cd4 1451static int aead_decrypt(struct aead_request *req)
9c4a7965
KP		 1452{
1453 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
aeb4c132 1454 unsigned int authsize = crypto_aead_authsize(authenc);
9c4a7965 1455 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
fe5720e2 1456 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
56af8cd4 1457 struct talitos_edesc *edesc;
9c4a7965
KP		 1458 struct scatterlist *sg;
1459 void *icvdata;
1460
1461 req->cryptlen -= authsize;
1462
1463 /* allocate extended descriptor */
62293a37 1464 edesc = aead_edesc_alloc(req, req->iv, 1, false);
9c4a7965
KP		 1465 if (IS_ERR(edesc))
1466 return PTR_ERR(edesc);
1467
fe5720e2 1468 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
e938e465
KP		 1469 ((!edesc->src_nents && !edesc->dst_nents) ||
1470 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
9c4a7965 1471
fe5720e2 1472 /* decrypt and check the ICV */
e938e465
KP		 1473 edesc->desc.hdr = ctx->desc_hdr_template |
1474 DESC_HDR_DIR_INBOUND |
fe5720e2 1475 DESC_HDR_MODE1_MDEU_CICV;
9c4a7965 1476
fe5720e2 1477 /* reset integrity check result bits */
9c4a7965 1478
aeb4c132 1479 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
e938e465 1480 }
fe5720e2 1481
e938e465
KP		 1482 /* Have to check the ICV with software */
1483 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
fe5720e2 1484
e938e465
KP		 1485 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1486 if (edesc->dma_len)
aeb4c132
HX		 1487 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1488 edesc->dst_nents + 2];
e938e465
KP		 1489 else
1490 icvdata = &edesc->link_tbl[0];
fe5720e2 1491
e938e465 1492 sg = sg_last(req->src, edesc->src_nents ? : 1);
fe5720e2 1493
aeb4c132 1494 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
9c4a7965 1495
aeb4c132 1496 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
9c4a7965
KP		 1497}
1498
4de9d0b5
LN		 1499static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1500 const u8 *key, unsigned int keylen)
1501{
1502 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
f384cdc4 1503 u32 tmp[DES_EXPKEY_WORDS];
4de9d0b5 1504
03d2c511
MH		 1505 if (keylen > TALITOS_MAX_KEY_SIZE) {
1506 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1507 return -EINVAL;
1508 }
1509
f384cdc4
LC		 1510 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1511 CRYPTO_TFM_REQ_WEAK_KEY) &&
1512 !des_ekey(tmp, key)) {
1513 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1514 return -EINVAL;
1515 }
1516
4de9d0b5
LN		 1517 memcpy(&ctx->key, key, keylen);
1518 ctx->keylen = keylen;
1519
1520 return 0;
4de9d0b5
LN		 1521}
1522
1523static void common_nonsnoop_unmap(struct device *dev,
1524 struct talitos_edesc *edesc,
1525 struct ablkcipher_request *areq)
1526{
1527 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
032d197e 1528
6a1e8d14 1529 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
4de9d0b5
LN		 1530 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1531 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1532
4de9d0b5
LN		 1533 if (edesc->dma_len)
1534 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1535 DMA_BIDIRECTIONAL);
1536}
1537
1538static void ablkcipher_done(struct device *dev,
1539 struct talitos_desc *desc, void *context,
1540 int err)
1541{
1542 struct ablkcipher_request *areq = context;
19bbbc63
KP		 1543 struct talitos_edesc *edesc;
1544
1545 edesc = container_of(desc, struct talitos_edesc, desc);
4de9d0b5
LN		 1546
1547 common_nonsnoop_unmap(dev, edesc, areq);
1548
1549 kfree(edesc);
1550
1551 areq->base.complete(&areq->base, err);
1552}
1553
1554static int common_nonsnoop(struct talitos_edesc *edesc,
1555 struct ablkcipher_request *areq,
4de9d0b5
LN		 1556 void (*callback) (struct device *dev,
1557 struct talitos_desc *desc,
1558 void *context, int error))
1559{
1560 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1561 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1562 struct device *dev = ctx->dev;
1563 struct talitos_desc *desc = &edesc->desc;
1564 unsigned int cryptlen = areq->nbytes;
79fd31d3 1565 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1566 int sg_count, ret;
6a1e8d14 1567 bool sync_needed = false;
922f9dc8
LC		 1568 struct talitos_private *priv = dev_get_drvdata(dev);
1569 bool is_sec1 = has_ftr_sec1(priv);
4de9d0b5
LN		 1570
1571 /* first DWORD empty */
4de9d0b5
LN		 1572
1573 /* cipher iv */
922f9dc8
LC		 1574 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1575 to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
4de9d0b5
LN		 1576
1577 /* cipher key */
1578 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
a2b35aa8 1579 (char *)&ctx->key, DMA_TO_DEVICE);
4de9d0b5 1580
6a1e8d14
LC		 1581 sg_count = edesc->src_nents ?: 1;
1582 if (is_sec1 && sg_count > 1)
1583 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1584 cryptlen);
1585 else
1586 sg_count = dma_map_sg(dev, areq->src, sg_count,
1587 (areq->src == areq->dst) ?
1588 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
4de9d0b5
LN		 1589 /*
1590 * cipher in
1591 */
6a1e8d14
LC		 1592 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1593 &desc->ptr[3], sg_count, 0, 0);
1594 if (sg_count > 1)
1595 sync_needed = true;
4de9d0b5
LN		 1596
1597 /* cipher out */
6a1e8d14
LC		 1598 if (areq->src != areq->dst) {
1599 sg_count = edesc->dst_nents ? : 1;
1600 if (!is_sec1 || sg_count == 1)
1601 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1602 }
1603
1604 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1605 sg_count, 0, (edesc->src_nents + 1));
1606 if (ret > 1)
1607 sync_needed = true;
4de9d0b5
LN		 1608
1609 /* iv out */
a2b35aa8 1610 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
4de9d0b5
LN		 1611 DMA_FROM_DEVICE);
1612
1613 /* last DWORD empty */
4de9d0b5 1614
6a1e8d14
LC		 1615 if (sync_needed)
1616 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1617 edesc->dma_len, DMA_BIDIRECTIONAL);
1618
5228f0f7 1619 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
4de9d0b5
LN		 1620 if (ret != -EINPROGRESS) {
1621 common_nonsnoop_unmap(dev, edesc, areq);
1622 kfree(edesc);
1623 }
1624 return ret;
1625}
1626
e938e465 1627static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
62293a37 1628 areq, bool encrypt)
4de9d0b5
LN		 1629{
1630 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1631 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
79fd31d3 1632 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
4de9d0b5 1633
aeb4c132 1634 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
79fd31d3 1635 areq->info, 0, areq->nbytes, 0, ivsize, 0,
62293a37 1636 areq->base.flags, encrypt);
4de9d0b5
LN		 1637}
1638
1639static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1640{
1641 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1642 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1643 struct talitos_edesc *edesc;
1644
1645 /* allocate extended descriptor */
62293a37 1646 edesc = ablkcipher_edesc_alloc(areq, true);
4de9d0b5
LN		 1647 if (IS_ERR(edesc))
1648 return PTR_ERR(edesc);
1649
1650 /* set encrypt */
1651 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1652
febec542 1653 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1654}
1655
1656static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1657{
1658 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1659 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1660 struct talitos_edesc *edesc;
1661
1662 /* allocate extended descriptor */
62293a37 1663 edesc = ablkcipher_edesc_alloc(areq, false);
4de9d0b5
LN		 1664 if (IS_ERR(edesc))
1665 return PTR_ERR(edesc);
1666
1667 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1668
febec542 1669 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1670}
1671
497f2e6b
LN		 1672static void common_nonsnoop_hash_unmap(struct device *dev,
1673 struct talitos_edesc *edesc,
1674 struct ahash_request *areq)
1675{
1676 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
922f9dc8
LC		 1677 struct talitos_private *priv = dev_get_drvdata(dev);
1678 bool is_sec1 = has_ftr_sec1(priv);
497f2e6b
LN		 1679
1680 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1681
6a1e8d14 1682 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
032d197e 1683
497f2e6b 1684 /* When using hashctx-in, must unmap it. */
922f9dc8 1685 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
497f2e6b
LN		 1686 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1687 DMA_TO_DEVICE);
1688
922f9dc8 1689 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
497f2e6b
LN		 1690 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1691 DMA_TO_DEVICE);
1692
497f2e6b
LN		 1693 if (edesc->dma_len)
1694 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1695 DMA_BIDIRECTIONAL);
1696
1697}
1698
1699static void ahash_done(struct device *dev,
1700 struct talitos_desc *desc, void *context,
1701 int err)
1702{
1703 struct ahash_request *areq = context;
1704 struct talitos_edesc *edesc =
1705 container_of(desc, struct talitos_edesc, desc);
1706 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1707
1708 if (!req_ctx->last && req_ctx->to_hash_later) {
1709 /* Position any partial block for next update/final/finup */
1710 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
5e833bc4 1711 req_ctx->nbuf = req_ctx->to_hash_later;
497f2e6b
LN		 1712 }
1713 common_nonsnoop_hash_unmap(dev, edesc, areq);
1714
1715 kfree(edesc);
1716
1717 areq->base.complete(&areq->base, err);
1718}
1719
2d02905e
LC		 1720/*
1721 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1722 * ourself and submit a padded block
1723 */
5b2cf268 1724static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
2d02905e
LC		 1725 struct talitos_edesc *edesc,
1726 struct talitos_ptr *ptr)
1727{
1728 static u8 padded_hash[64] = {
1729 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1730 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1731 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1732 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1733 };
1734
1735 pr_err_once("Bug in SEC1, padding ourself\n");
1736 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1737 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1738 (char *)padded_hash, DMA_TO_DEVICE);
1739}
1740
497f2e6b
LN		 1741static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1742 struct ahash_request *areq, unsigned int length,
1743 void (*callback) (struct device *dev,
1744 struct talitos_desc *desc,
1745 void *context, int error))
1746{
1747 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1748 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1749 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1750 struct device *dev = ctx->dev;
1751 struct talitos_desc *desc = &edesc->desc;
032d197e 1752 int ret;
6a1e8d14 1753 bool sync_needed = false;
922f9dc8
LC		 1754 struct talitos_private *priv = dev_get_drvdata(dev);
1755 bool is_sec1 = has_ftr_sec1(priv);
6a1e8d14 1756 int sg_count;
497f2e6b
LN		 1757
1758 /* first DWORD empty */
497f2e6b 1759
60f208d7
KP		 1760 /* hash context in */
1761 if (!req_ctx->first || req_ctx->swinit) {
497f2e6b
LN		 1762 map_single_talitos_ptr(dev, &desc->ptr[1],
1763 req_ctx->hw_context_size,
a2b35aa8 1764 (char *)req_ctx->hw_context,
497f2e6b 1765 DMA_TO_DEVICE);
60f208d7 1766 req_ctx->swinit = 0;
497f2e6b 1767 }
afd62fa2
LC		 1768 /* Indicate next op is not the first. */
1769 req_ctx->first = 0;
497f2e6b
LN		 1770
1771 /* HMAC key */
1772 if (ctx->keylen)
1773 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
a2b35aa8 1774 (char *)&ctx->key, DMA_TO_DEVICE);
497f2e6b 1775
6a1e8d14
LC		 1776 sg_count = edesc->src_nents ?: 1;
1777 if (is_sec1 && sg_count > 1)
886a27c0 1778 sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
6a1e8d14
LC		 1779 else
1780 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1781 DMA_TO_DEVICE);
497f2e6b
LN		 1782 /*
1783 * data in
1784 */
6a1e8d14
LC		 1785 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1786 &desc->ptr[3], sg_count, 0, 0);
1787 if (sg_count > 1)
1788 sync_needed = true;
497f2e6b
LN		 1789
1790 /* fifth DWORD empty */
497f2e6b
LN		 1791
1792 /* hash/HMAC out -or- hash context out */
1793 if (req_ctx->last)
1794 map_single_talitos_ptr(dev, &desc->ptr[5],
1795 crypto_ahash_digestsize(tfm),
a2b35aa8 1796 areq->result, DMA_FROM_DEVICE);
497f2e6b
LN		 1797 else
1798 map_single_talitos_ptr(dev, &desc->ptr[5],
1799 req_ctx->hw_context_size,
a2b35aa8 1800 req_ctx->hw_context, DMA_FROM_DEVICE);
497f2e6b
LN		 1801
1802 /* last DWORD empty */
497f2e6b 1803
2d02905e
LC		 1804 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1805 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1806
6a1e8d14
LC		 1807 if (sync_needed)
1808 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1809 edesc->dma_len, DMA_BIDIRECTIONAL);
1810
5228f0f7 1811 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
497f2e6b
LN		 1812 if (ret != -EINPROGRESS) {
1813 common_nonsnoop_hash_unmap(dev, edesc, areq);
1814 kfree(edesc);
1815 }
1816 return ret;
1817}
1818
1819static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1820 unsigned int nbytes)
1821{
1822 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1823 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1824 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1825
aeb4c132 1826 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
62293a37 1827 nbytes, 0, 0, 0, areq->base.flags, false);
497f2e6b
LN		 1828}
1829
1830static int ahash_init(struct ahash_request *areq)
1831{
1832 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1833 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1834
1835 /* Initialize the context */
5e833bc4 1836 req_ctx->nbuf = 0;
60f208d7
KP		 1837 req_ctx->first = 1; /* first indicates h/w must init its context */
1838 req_ctx->swinit = 0; /* assume h/w init of context */
497f2e6b
LN		 1839 req_ctx->hw_context_size =
1840 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1841 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1842 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1843
1844 return 0;
1845}
1846
60f208d7
KP		 1847/*
1848 * on h/w without explicit sha224 support, we initialize h/w context
1849 * manually with sha224 constants, and tell it to run sha256.
1850 */
1851static int ahash_init_sha224_swinit(struct ahash_request *areq)
1852{
1853 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1854
1855 ahash_init(areq);
1856 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1857
a752447a
KP		 1858 req_ctx->hw_context[0] = SHA224_H0;
1859 req_ctx->hw_context[1] = SHA224_H1;
1860 req_ctx->hw_context[2] = SHA224_H2;
1861 req_ctx->hw_context[3] = SHA224_H3;
1862 req_ctx->hw_context[4] = SHA224_H4;
1863 req_ctx->hw_context[5] = SHA224_H5;
1864 req_ctx->hw_context[6] = SHA224_H6;
1865 req_ctx->hw_context[7] = SHA224_H7;
60f208d7
KP		 1866
1867 /* init 64-bit count */
1868 req_ctx->hw_context[8] = 0;
1869 req_ctx->hw_context[9] = 0;
1870
1871 return 0;
1872}
1873
497f2e6b
LN		 1874static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1875{
1876 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1877 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1878 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1879 struct talitos_edesc *edesc;
1880 unsigned int blocksize =
1881 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1882 unsigned int nbytes_to_hash;
1883 unsigned int to_hash_later;
5e833bc4 1884 unsigned int nsg;
8e409fe1 1885 int nents;
497f2e6b 1886
5e833bc4
LN		 1887 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1888 /* Buffer up to one whole block */
8e409fe1
LC		 1889 nents = sg_nents_for_len(areq->src, nbytes);
1890 if (nents < 0) {
1891 dev_err(ctx->dev, "Invalid number of src SG.\n");
1892 return nents;
1893 }
1894 sg_copy_to_buffer(areq->src, nents,
5e833bc4
LN		 1895 req_ctx->buf + req_ctx->nbuf, nbytes);
1896 req_ctx->nbuf += nbytes;
497f2e6b
LN		 1897 return 0;
1898 }
1899
5e833bc4
LN		 1900 /* At least (blocksize + 1) bytes are available to hash */
1901 nbytes_to_hash = nbytes + req_ctx->nbuf;
1902 to_hash_later = nbytes_to_hash & (blocksize - 1);
1903
1904 if (req_ctx->last)
1905 to_hash_later = 0;
1906 else if (to_hash_later)
1907 /* There is a partial block. Hash the full block(s) now */
1908 nbytes_to_hash -= to_hash_later;
1909 else {
1910 /* Keep one block buffered */
1911 nbytes_to_hash -= blocksize;
1912 to_hash_later = blocksize;
1913 }
1914
1915 /* Chain in any previously buffered data */
1916 if (req_ctx->nbuf) {
1917 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1918 sg_init_table(req_ctx->bufsl, nsg);
1919 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1920 if (nsg > 1)
c56f6d12 1921 sg_chain(req_ctx->bufsl, 2, areq->src);
497f2e6b 1922 req_ctx->psrc = req_ctx->bufsl;
5e833bc4 1923 } else
497f2e6b 1924 req_ctx->psrc = areq->src;
5e833bc4
LN		 1925
1926 if (to_hash_later) {
8e409fe1
LC		 1927 nents = sg_nents_for_len(areq->src, nbytes);
1928 if (nents < 0) {
1929 dev_err(ctx->dev, "Invalid number of src SG.\n");
1930 return nents;
1931 }
d0525723 1932 sg_pcopy_to_buffer(areq->src, nents,
5e833bc4
LN		 1933 req_ctx->bufnext,
1934 to_hash_later,
1935 nbytes - to_hash_later);
497f2e6b 1936 }
5e833bc4 1937 req_ctx->to_hash_later = to_hash_later;
497f2e6b 1938
5e833bc4 1939 /* Allocate extended descriptor */
497f2e6b
LN		 1940 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1941 if (IS_ERR(edesc))
1942 return PTR_ERR(edesc);
1943
1944 edesc->desc.hdr = ctx->desc_hdr_template;
1945
1946 /* On last one, request SEC to pad; otherwise continue */
1947 if (req_ctx->last)
1948 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1949 else
1950 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1951
60f208d7
KP		 1952 /* request SEC to INIT hash. */
1953 if (req_ctx->first && !req_ctx->swinit)
497f2e6b
LN		 1954 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1955
1956 /* When the tfm context has a keylen, it's an HMAC.
1957 * A first or last (ie. not middle) descriptor must request HMAC.
1958 */
1959 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1960 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1961
1962 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1963 ahash_done);
1964}
1965
1966static int ahash_update(struct ahash_request *areq)
1967{
1968 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1969
1970 req_ctx->last = 0;
1971
1972 return ahash_process_req(areq, areq->nbytes);
1973}
1974
1975static int ahash_final(struct ahash_request *areq)
1976{
1977 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1978
1979 req_ctx->last = 1;
1980
1981 return ahash_process_req(areq, 0);
1982}
1983
1984static int ahash_finup(struct ahash_request *areq)
1985{
1986 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1987
1988 req_ctx->last = 1;
1989
1990 return ahash_process_req(areq, areq->nbytes);
1991}
1992
1993static int ahash_digest(struct ahash_request *areq)
1994{
1995 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
60f208d7 1996 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
497f2e6b 1997
60f208d7 1998 ahash->init(areq);
497f2e6b
LN		 1999 req_ctx->last = 1;
2000
2001 return ahash_process_req(areq, areq->nbytes);
2002}
2003
3639ca84
HG		 2004static int ahash_export(struct ahash_request *areq, void *out)
2005{
2006 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2007 struct talitos_export_state *export = out;
2008
2009 memcpy(export->hw_context, req_ctx->hw_context,
2010 req_ctx->hw_context_size);
2011 memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2012 export->swinit = req_ctx->swinit;
2013 export->first = req_ctx->first;
2014 export->last = req_ctx->last;
2015 export->to_hash_later = req_ctx->to_hash_later;
2016 export->nbuf = req_ctx->nbuf;
2017
2018 return 0;
2019}
2020
2021static int ahash_import(struct ahash_request *areq, const void *in)
2022{
2023 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2024 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2025 const struct talitos_export_state *export = in;
2026
2027 memset(req_ctx, 0, sizeof(*req_ctx));
2028 req_ctx->hw_context_size =
2029 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2030 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2031 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2032 memcpy(req_ctx->hw_context, export->hw_context,
2033 req_ctx->hw_context_size);
2034 memcpy(req_ctx->buf, export->buf, export->nbuf);
2035 req_ctx->swinit = export->swinit;
2036 req_ctx->first = export->first;
2037 req_ctx->last = export->last;
2038 req_ctx->to_hash_later = export->to_hash_later;
2039 req_ctx->nbuf = export->nbuf;
2040
2041 return 0;
2042}
2043
79b3a418
LN		 2044struct keyhash_result {
2045 struct completion completion;
2046 int err;
2047};
2048
2049static void keyhash_complete(struct crypto_async_request *req, int err)
2050{
2051 struct keyhash_result *res = req->data;
2052
2053 if (err == -EINPROGRESS)
2054 return;
2055
2056 res->err = err;
2057 complete(&res->completion);
2058}
2059
2060static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2061 u8 *hash)
2062{
2063 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2064
2065 struct scatterlist sg[1];
2066 struct ahash_request *req;
2067 struct keyhash_result hresult;
2068 int ret;
2069
2070 init_completion(&hresult.completion);
2071
2072 req = ahash_request_alloc(tfm, GFP_KERNEL);
2073 if (!req)
2074 return -ENOMEM;
2075
2076 /* Keep tfm keylen == 0 during hash of the long key */
2077 ctx->keylen = 0;
2078 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2079 keyhash_complete, &hresult);
2080
2081 sg_init_one(&sg[0], key, keylen);
2082
2083 ahash_request_set_crypt(req, sg, hash, keylen);
2084 ret = crypto_ahash_digest(req);
2085 switch (ret) {
2086 case 0:
2087 break;
2088 case -EINPROGRESS:
2089 case -EBUSY:
2090 ret = wait_for_completion_interruptible(
2091 &hresult.completion);
2092 if (!ret)
2093 ret = hresult.err;
2094 break;
2095 default:
2096 break;
2097 }
2098 ahash_request_free(req);
2099
2100 return ret;
2101}
2102
2103static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2104 unsigned int keylen)
2105{
2106 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2107 unsigned int blocksize =
2108 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2109 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2110 unsigned int keysize = keylen;
2111 u8 hash[SHA512_DIGEST_SIZE];
2112 int ret;
2113
2114 if (keylen <= blocksize)
2115 memcpy(ctx->key, key, keysize);
2116 else {
2117 /* Must get the hash of the long key */
2118 ret = keyhash(tfm, key, keylen, hash);
2119
2120 if (ret) {
2121 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2122 return -EINVAL;
2123 }
2124
2125 keysize = digestsize;
2126 memcpy(ctx->key, hash, digestsize);
2127 }
2128
2129 ctx->keylen = keysize;
2130
2131 return 0;
2132}
2133
2134
9c4a7965 2135struct talitos_alg_template {
d5e4aaef 2136 u32 type;
b0057763 2137 u32 priority;
d5e4aaef
LN		 2138 union {
2139 struct crypto_alg crypto;
acbf7c62 2140 struct ahash_alg hash;
aeb4c132 2141 struct aead_alg aead;
d5e4aaef 2142 } alg;
9c4a7965
KP		 2143 __be32 desc_hdr_template;
2144};
2145
2146static struct talitos_alg_template driver_algs[] = {
991155ba 2147 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
d5e4aaef 2148 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2149 .alg.aead = {
2150 .base = {
2151 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2152 .cra_driver_name = "authenc-hmac-sha1-"
2153 "cbc-aes-talitos",
2154 .cra_blocksize = AES_BLOCK_SIZE,
2155 .cra_flags = CRYPTO_ALG_ASYNC,
2156 },
2157 .ivsize = AES_BLOCK_SIZE,
2158 .maxauthsize = SHA1_DIGEST_SIZE,
56af8cd4 2159 },
9c4a7965
KP		 2160 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2161 DESC_HDR_SEL0_AESU |
2162 DESC_HDR_MODE0_AESU_CBC |
2163 DESC_HDR_SEL1_MDEUA |
2164 DESC_HDR_MODE1_MDEU_INIT |
2165 DESC_HDR_MODE1_MDEU_PAD |
2166 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
70bcaca7 2167 },
7405c8d7
LC		 2168 { .type = CRYPTO_ALG_TYPE_AEAD,
2169 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2170 .alg.aead = {
2171 .base = {
2172 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2173 .cra_driver_name = "authenc-hmac-sha1-"
2174 "cbc-aes-talitos",
2175 .cra_blocksize = AES_BLOCK_SIZE,
2176 .cra_flags = CRYPTO_ALG_ASYNC,
2177 },
2178 .ivsize = AES_BLOCK_SIZE,
2179 .maxauthsize = SHA1_DIGEST_SIZE,
2180 },
2181 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2182 DESC_HDR_SEL0_AESU |
2183 DESC_HDR_MODE0_AESU_CBC |
2184 DESC_HDR_SEL1_MDEUA |
2185 DESC_HDR_MODE1_MDEU_INIT |
2186 DESC_HDR_MODE1_MDEU_PAD |
2187 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2188 },
d5e4aaef 2189 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2190 .alg.aead = {
2191 .base = {
2192 .cra_name = "authenc(hmac(sha1),"
2193 "cbc(des3_ede))",
2194 .cra_driver_name = "authenc-hmac-sha1-"
2195 "cbc-3des-talitos",
2196 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2197 .cra_flags = CRYPTO_ALG_ASYNC,
2198 },
2199 .ivsize = DES3_EDE_BLOCK_SIZE,
2200 .maxauthsize = SHA1_DIGEST_SIZE,
56af8cd4 2201 },
70bcaca7
LN		 2202 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2203 DESC_HDR_SEL0_DEU |
2204 DESC_HDR_MODE0_DEU_CBC |
2205 DESC_HDR_MODE0_DEU_3DES |
2206 DESC_HDR_SEL1_MDEUA |
2207 DESC_HDR_MODE1_MDEU_INIT |
2208 DESC_HDR_MODE1_MDEU_PAD |
2209 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
3952f17e 2210 },
7405c8d7
LC		 2211 { .type = CRYPTO_ALG_TYPE_AEAD,
2212 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2213 .alg.aead = {
2214 .base = {
2215 .cra_name = "authenc(hmac(sha1),"
2216 "cbc(des3_ede))",
2217 .cra_driver_name = "authenc-hmac-sha1-"
2218 "cbc-3des-talitos",
2219 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2220 .cra_flags = CRYPTO_ALG_ASYNC,
2221 },
2222 .ivsize = DES3_EDE_BLOCK_SIZE,
2223 .maxauthsize = SHA1_DIGEST_SIZE,
2224 },
2225 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2226 DESC_HDR_SEL0_DEU |
2227 DESC_HDR_MODE0_DEU_CBC |
2228 DESC_HDR_MODE0_DEU_3DES |
2229 DESC_HDR_SEL1_MDEUA |
2230 DESC_HDR_MODE1_MDEU_INIT |
2231 DESC_HDR_MODE1_MDEU_PAD |
2232 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2233 },
357fb605 2234 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2235 .alg.aead = {
2236 .base = {
2237 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2238 .cra_driver_name = "authenc-hmac-sha224-"
2239 "cbc-aes-talitos",
2240 .cra_blocksize = AES_BLOCK_SIZE,
2241 .cra_flags = CRYPTO_ALG_ASYNC,
2242 },
2243 .ivsize = AES_BLOCK_SIZE,
2244 .maxauthsize = SHA224_DIGEST_SIZE,
357fb605
HG		 2245 },
2246 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2247 DESC_HDR_SEL0_AESU |
2248 DESC_HDR_MODE0_AESU_CBC |
2249 DESC_HDR_SEL1_MDEUA |
2250 DESC_HDR_MODE1_MDEU_INIT |
2251 DESC_HDR_MODE1_MDEU_PAD |
2252 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2253 },
7405c8d7
LC		 2254 { .type = CRYPTO_ALG_TYPE_AEAD,
2255 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2256 .alg.aead = {
2257 .base = {
2258 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2259 .cra_driver_name = "authenc-hmac-sha224-"
2260 "cbc-aes-talitos",
2261 .cra_blocksize = AES_BLOCK_SIZE,
2262 .cra_flags = CRYPTO_ALG_ASYNC,
2263 },
2264 .ivsize = AES_BLOCK_SIZE,
2265 .maxauthsize = SHA224_DIGEST_SIZE,
2266 },
2267 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2268 DESC_HDR_SEL0_AESU |
2269 DESC_HDR_MODE0_AESU_CBC |
2270 DESC_HDR_SEL1_MDEUA |
2271 DESC_HDR_MODE1_MDEU_INIT |
2272 DESC_HDR_MODE1_MDEU_PAD |
2273 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2274 },
357fb605 2275 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2276 .alg.aead = {
2277 .base = {
2278 .cra_name = "authenc(hmac(sha224),"
2279 "cbc(des3_ede))",
2280 .cra_driver_name = "authenc-hmac-sha224-"
2281 "cbc-3des-talitos",
2282 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283 .cra_flags = CRYPTO_ALG_ASYNC,
2284 },
2285 .ivsize = DES3_EDE_BLOCK_SIZE,
2286 .maxauthsize = SHA224_DIGEST_SIZE,
357fb605
HG		 2287 },
2288 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2289 DESC_HDR_SEL0_DEU |
2290 DESC_HDR_MODE0_DEU_CBC |
2291 DESC_HDR_MODE0_DEU_3DES |
2292 DESC_HDR_SEL1_MDEUA |
2293 DESC_HDR_MODE1_MDEU_INIT |
2294 DESC_HDR_MODE1_MDEU_PAD |
2295 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2296 },
7405c8d7
LC		 2297 { .type = CRYPTO_ALG_TYPE_AEAD,
2298 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2299 .alg.aead = {
2300 .base = {
2301 .cra_name = "authenc(hmac(sha224),"
2302 "cbc(des3_ede))",
2303 .cra_driver_name = "authenc-hmac-sha224-"
2304 "cbc-3des-talitos",
2305 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2306 .cra_flags = CRYPTO_ALG_ASYNC,
2307 },
2308 .ivsize = DES3_EDE_BLOCK_SIZE,
2309 .maxauthsize = SHA224_DIGEST_SIZE,
2310 },
2311 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2312 DESC_HDR_SEL0_DEU |
2313 DESC_HDR_MODE0_DEU_CBC |
2314 DESC_HDR_MODE0_DEU_3DES |
2315 DESC_HDR_SEL1_MDEUA |
2316 DESC_HDR_MODE1_MDEU_INIT |
2317 DESC_HDR_MODE1_MDEU_PAD |
2318 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2319 },
d5e4aaef 2320 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2321 .alg.aead = {
2322 .base = {
2323 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2324 .cra_driver_name = "authenc-hmac-sha256-"
2325 "cbc-aes-talitos",
2326 .cra_blocksize = AES_BLOCK_SIZE,
2327 .cra_flags = CRYPTO_ALG_ASYNC,
2328 },
2329 .ivsize = AES_BLOCK_SIZE,
2330 .maxauthsize = SHA256_DIGEST_SIZE,
56af8cd4 2331 },
3952f17e
LN		 2332 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2333 DESC_HDR_SEL0_AESU |
2334 DESC_HDR_MODE0_AESU_CBC |
2335 DESC_HDR_SEL1_MDEUA |
2336 DESC_HDR_MODE1_MDEU_INIT |
2337 DESC_HDR_MODE1_MDEU_PAD |
2338 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2339 },
7405c8d7
LC		 2340 { .type = CRYPTO_ALG_TYPE_AEAD,
2341 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2342 .alg.aead = {
2343 .base = {
2344 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2345 .cra_driver_name = "authenc-hmac-sha256-"
2346 "cbc-aes-talitos",
2347 .cra_blocksize = AES_BLOCK_SIZE,
2348 .cra_flags = CRYPTO_ALG_ASYNC,
2349 },
2350 .ivsize = AES_BLOCK_SIZE,
2351 .maxauthsize = SHA256_DIGEST_SIZE,
2352 },
2353 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2354 DESC_HDR_SEL0_AESU |
2355 DESC_HDR_MODE0_AESU_CBC |
2356 DESC_HDR_SEL1_MDEUA |
2357 DESC_HDR_MODE1_MDEU_INIT |
2358 DESC_HDR_MODE1_MDEU_PAD |
2359 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2360 },
d5e4aaef 2361 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2362 .alg.aead = {
2363 .base = {
2364 .cra_name = "authenc(hmac(sha256),"
2365 "cbc(des3_ede))",
2366 .cra_driver_name = "authenc-hmac-sha256-"
2367 "cbc-3des-talitos",
2368 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2369 .cra_flags = CRYPTO_ALG_ASYNC,
2370 },
2371 .ivsize = DES3_EDE_BLOCK_SIZE,
2372 .maxauthsize = SHA256_DIGEST_SIZE,
56af8cd4 2373 },
3952f17e
LN		 2374 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2375 DESC_HDR_SEL0_DEU |
2376 DESC_HDR_MODE0_DEU_CBC |
2377 DESC_HDR_MODE0_DEU_3DES |
2378 DESC_HDR_SEL1_MDEUA |
2379 DESC_HDR_MODE1_MDEU_INIT |
2380 DESC_HDR_MODE1_MDEU_PAD |
2381 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2382 },
7405c8d7
LC		 2383 { .type = CRYPTO_ALG_TYPE_AEAD,
2384 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2385 .alg.aead = {
2386 .base = {
2387 .cra_name = "authenc(hmac(sha256),"
2388 "cbc(des3_ede))",
2389 .cra_driver_name = "authenc-hmac-sha256-"
2390 "cbc-3des-talitos",
2391 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2392 .cra_flags = CRYPTO_ALG_ASYNC,
2393 },
2394 .ivsize = DES3_EDE_BLOCK_SIZE,
2395 .maxauthsize = SHA256_DIGEST_SIZE,
2396 },
2397 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2398 DESC_HDR_SEL0_DEU |
2399 DESC_HDR_MODE0_DEU_CBC |
2400 DESC_HDR_MODE0_DEU_3DES |
2401 DESC_HDR_SEL1_MDEUA |
2402 DESC_HDR_MODE1_MDEU_INIT |
2403 DESC_HDR_MODE1_MDEU_PAD |
2404 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2405 },
d5e4aaef 2406 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2407 .alg.aead = {
2408 .base = {
2409 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2410 .cra_driver_name = "authenc-hmac-sha384-"
2411 "cbc-aes-talitos",
2412 .cra_blocksize = AES_BLOCK_SIZE,
2413 .cra_flags = CRYPTO_ALG_ASYNC,
2414 },
2415 .ivsize = AES_BLOCK_SIZE,
2416 .maxauthsize = SHA384_DIGEST_SIZE,
357fb605
HG		 2417 },
2418 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2419 DESC_HDR_SEL0_AESU |
2420 DESC_HDR_MODE0_AESU_CBC |
2421 DESC_HDR_SEL1_MDEUB |
2422 DESC_HDR_MODE1_MDEU_INIT |
2423 DESC_HDR_MODE1_MDEU_PAD |
2424 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2425 },
2426 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2427 .alg.aead = {
2428 .base = {
2429 .cra_name = "authenc(hmac(sha384),"
2430 "cbc(des3_ede))",
2431 .cra_driver_name = "authenc-hmac-sha384-"
2432 "cbc-3des-talitos",
2433 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2434 .cra_flags = CRYPTO_ALG_ASYNC,
2435 },
2436 .ivsize = DES3_EDE_BLOCK_SIZE,
2437 .maxauthsize = SHA384_DIGEST_SIZE,
357fb605
HG		 2438 },
2439 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2440 DESC_HDR_SEL0_DEU |
2441 DESC_HDR_MODE0_DEU_CBC |
2442 DESC_HDR_MODE0_DEU_3DES |
2443 DESC_HDR_SEL1_MDEUB |
2444 DESC_HDR_MODE1_MDEU_INIT |
2445 DESC_HDR_MODE1_MDEU_PAD |
2446 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2447 },
2448 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2449 .alg.aead = {
2450 .base = {
2451 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2452 .cra_driver_name = "authenc-hmac-sha512-"
2453 "cbc-aes-talitos",
2454 .cra_blocksize = AES_BLOCK_SIZE,
2455 .cra_flags = CRYPTO_ALG_ASYNC,
2456 },
2457 .ivsize = AES_BLOCK_SIZE,
2458 .maxauthsize = SHA512_DIGEST_SIZE,
357fb605
HG		 2459 },
2460 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2461 DESC_HDR_SEL0_AESU |
2462 DESC_HDR_MODE0_AESU_CBC |
2463 DESC_HDR_SEL1_MDEUB |
2464 DESC_HDR_MODE1_MDEU_INIT |
2465 DESC_HDR_MODE1_MDEU_PAD |
2466 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2467 },
2468 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2469 .alg.aead = {
2470 .base = {
2471 .cra_name = "authenc(hmac(sha512),"
2472 "cbc(des3_ede))",
2473 .cra_driver_name = "authenc-hmac-sha512-"
2474 "cbc-3des-talitos",
2475 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2476 .cra_flags = CRYPTO_ALG_ASYNC,
2477 },
2478 .ivsize = DES3_EDE_BLOCK_SIZE,
2479 .maxauthsize = SHA512_DIGEST_SIZE,
357fb605
HG		 2480 },
2481 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2482 DESC_HDR_SEL0_DEU |
2483 DESC_HDR_MODE0_DEU_CBC |
2484 DESC_HDR_MODE0_DEU_3DES |
2485 DESC_HDR_SEL1_MDEUB |
2486 DESC_HDR_MODE1_MDEU_INIT |
2487 DESC_HDR_MODE1_MDEU_PAD |
2488 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2489 },
2490 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2491 .alg.aead = {
2492 .base = {
2493 .cra_name = "authenc(hmac(md5),cbc(aes))",
2494 .cra_driver_name = "authenc-hmac-md5-"
2495 "cbc-aes-talitos",
2496 .cra_blocksize = AES_BLOCK_SIZE,
2497 .cra_flags = CRYPTO_ALG_ASYNC,
2498 },
2499 .ivsize = AES_BLOCK_SIZE,
2500 .maxauthsize = MD5_DIGEST_SIZE,
56af8cd4 2501 },
3952f17e
LN		 2502 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2503 DESC_HDR_SEL0_AESU |
2504 DESC_HDR_MODE0_AESU_CBC |
2505 DESC_HDR_SEL1_MDEUA |
2506 DESC_HDR_MODE1_MDEU_INIT |
2507 DESC_HDR_MODE1_MDEU_PAD |
2508 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2509 },
7405c8d7
LC		 2510 { .type = CRYPTO_ALG_TYPE_AEAD,
2511 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2512 .alg.aead = {
2513 .base = {
2514 .cra_name = "authenc(hmac(md5),cbc(aes))",
2515 .cra_driver_name = "authenc-hmac-md5-"
2516 "cbc-aes-talitos",
2517 .cra_blocksize = AES_BLOCK_SIZE,
2518 .cra_flags = CRYPTO_ALG_ASYNC,
2519 },
2520 .ivsize = AES_BLOCK_SIZE,
2521 .maxauthsize = MD5_DIGEST_SIZE,
2522 },
2523 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2524 DESC_HDR_SEL0_AESU |
2525 DESC_HDR_MODE0_AESU_CBC |
2526 DESC_HDR_SEL1_MDEUA |
2527 DESC_HDR_MODE1_MDEU_INIT |
2528 DESC_HDR_MODE1_MDEU_PAD |
2529 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2530 },
d5e4aaef 2531 { .type = CRYPTO_ALG_TYPE_AEAD,
aeb4c132
HX		 2532 .alg.aead = {
2533 .base = {
2534 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2535 .cra_driver_name = "authenc-hmac-md5-"
2536 "cbc-3des-talitos",
2537 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2538 .cra_flags = CRYPTO_ALG_ASYNC,
2539 },
2540 .ivsize = DES3_EDE_BLOCK_SIZE,
2541 .maxauthsize = MD5_DIGEST_SIZE,
56af8cd4 2542 },
3952f17e
LN		 2543 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2544 DESC_HDR_SEL0_DEU |
2545 DESC_HDR_MODE0_DEU_CBC |
2546 DESC_HDR_MODE0_DEU_3DES |
2547 DESC_HDR_SEL1_MDEUA |
2548 DESC_HDR_MODE1_MDEU_INIT |
2549 DESC_HDR_MODE1_MDEU_PAD |
2550 DESC_HDR_MODE1_MDEU_MD5_HMAC,
4de9d0b5 2551 },
7405c8d7
LC		 2552 { .type = CRYPTO_ALG_TYPE_AEAD,
2553 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2554 .alg.aead = {
2555 .base = {
2556 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2557 .cra_driver_name = "authenc-hmac-md5-"
2558 "cbc-3des-talitos",
2559 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2560 .cra_flags = CRYPTO_ALG_ASYNC,
2561 },
2562 .ivsize = DES3_EDE_BLOCK_SIZE,
2563 .maxauthsize = MD5_DIGEST_SIZE,
2564 },
2565 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2566 DESC_HDR_SEL0_DEU |
2567 DESC_HDR_MODE0_DEU_CBC |
2568 DESC_HDR_MODE0_DEU_3DES |
2569 DESC_HDR_SEL1_MDEUA |
2570 DESC_HDR_MODE1_MDEU_INIT |
2571 DESC_HDR_MODE1_MDEU_PAD |
2572 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2573 },
4de9d0b5 2574 /* ABLKCIPHER algorithms. */
5e75ae1b
LC		 2575 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2576 .alg.crypto = {
2577 .cra_name = "ecb(aes)",
2578 .cra_driver_name = "ecb-aes-talitos",
2579 .cra_blocksize = AES_BLOCK_SIZE,
2580 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2581 CRYPTO_ALG_ASYNC,
2582 .cra_ablkcipher = {
2583 .min_keysize = AES_MIN_KEY_SIZE,
2584 .max_keysize = AES_MAX_KEY_SIZE,
2585 .ivsize = AES_BLOCK_SIZE,
2586 }
2587 },
2588 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2589 DESC_HDR_SEL0_AESU,
2590 },
d5e4aaef
LN		 2591 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2592 .alg.crypto = {
4de9d0b5
LN		 2593 .cra_name = "cbc(aes)",
2594 .cra_driver_name = "cbc-aes-talitos",
2595 .cra_blocksize = AES_BLOCK_SIZE,
2596 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2597 CRYPTO_ALG_ASYNC,
4de9d0b5 2598 .cra_ablkcipher = {
4de9d0b5
LN		 2599 .min_keysize = AES_MIN_KEY_SIZE,
2600 .max_keysize = AES_MAX_KEY_SIZE,
2601 .ivsize = AES_BLOCK_SIZE,
2602 }
2603 },
2604 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2605 DESC_HDR_SEL0_AESU |
2606 DESC_HDR_MODE0_AESU_CBC,
2607 },
5e75ae1b
LC		 2608 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2609 .alg.crypto = {
2610 .cra_name = "ctr(aes)",
2611 .cra_driver_name = "ctr-aes-talitos",
2612 .cra_blocksize = AES_BLOCK_SIZE,
2613 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2614 CRYPTO_ALG_ASYNC,
2615 .cra_ablkcipher = {
2616 .min_keysize = AES_MIN_KEY_SIZE,
2617 .max_keysize = AES_MAX_KEY_SIZE,
2618 .ivsize = AES_BLOCK_SIZE,
2619 }
2620 },
70d355cc 2621 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
5e75ae1b
LC		 2622 DESC_HDR_SEL0_AESU |
2623 DESC_HDR_MODE0_AESU_CTR,
2624 },
2625 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2626 .alg.crypto = {
2627 .cra_name = "ecb(des)",
2628 .cra_driver_name = "ecb-des-talitos",
2629 .cra_blocksize = DES_BLOCK_SIZE,
2630 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2631 CRYPTO_ALG_ASYNC,
2632 .cra_ablkcipher = {
2633 .min_keysize = DES_KEY_SIZE,
2634 .max_keysize = DES_KEY_SIZE,
2635 .ivsize = DES_BLOCK_SIZE,
2636 }
2637 },
2638 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2639 DESC_HDR_SEL0_DEU,
2640 },
2641 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2642 .alg.crypto = {
2643 .cra_name = "cbc(des)",
2644 .cra_driver_name = "cbc-des-talitos",
2645 .cra_blocksize = DES_BLOCK_SIZE,
2646 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2647 CRYPTO_ALG_ASYNC,
2648 .cra_ablkcipher = {
2649 .min_keysize = DES_KEY_SIZE,
2650 .max_keysize = DES_KEY_SIZE,
2651 .ivsize = DES_BLOCK_SIZE,
2652 }
2653 },
2654 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2655 DESC_HDR_SEL0_DEU |
2656 DESC_HDR_MODE0_DEU_CBC,
2657 },
2658 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2659 .alg.crypto = {
2660 .cra_name = "ecb(des3_ede)",
2661 .cra_driver_name = "ecb-3des-talitos",
2662 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2663 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2664 CRYPTO_ALG_ASYNC,
2665 .cra_ablkcipher = {
2666 .min_keysize = DES3_EDE_KEY_SIZE,
2667 .max_keysize = DES3_EDE_KEY_SIZE,
2668 .ivsize = DES3_EDE_BLOCK_SIZE,
2669 }
2670 },
2671 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2672 DESC_HDR_SEL0_DEU |
2673 DESC_HDR_MODE0_DEU_3DES,
2674 },
d5e4aaef
LN		 2675 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2676 .alg.crypto = {
4de9d0b5
LN		 2677 .cra_name = "cbc(des3_ede)",
2678 .cra_driver_name = "cbc-3des-talitos",
2679 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2680 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2681 CRYPTO_ALG_ASYNC,
4de9d0b5 2682 .cra_ablkcipher = {
4de9d0b5
LN		 2683 .min_keysize = DES3_EDE_KEY_SIZE,
2684 .max_keysize = DES3_EDE_KEY_SIZE,
2685 .ivsize = DES3_EDE_BLOCK_SIZE,
2686 }
2687 },
2688 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2689 DESC_HDR_SEL0_DEU |
2690 DESC_HDR_MODE0_DEU_CBC |
2691 DESC_HDR_MODE0_DEU_3DES,
497f2e6b
LN		 2692 },
2693 /* AHASH algorithms. */
2694 { .type = CRYPTO_ALG_TYPE_AHASH,
2695 .alg.hash = {
497f2e6b 2696 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 2697 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2698 .halg.base = {
2699 .cra_name = "md5",
2700 .cra_driver_name = "md5-talitos",
b3988618 2701 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
497f2e6b
LN		 2702 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2703 CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2704 }
2705 },
2706 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2707 DESC_HDR_SEL0_MDEUA |
2708 DESC_HDR_MODE0_MDEU_MD5,
2709 },
2710 { .type = CRYPTO_ALG_TYPE_AHASH,
2711 .alg.hash = {
497f2e6b 2712 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 2713 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2714 .halg.base = {
2715 .cra_name = "sha1",
2716 .cra_driver_name = "sha1-talitos",
2717 .cra_blocksize = SHA1_BLOCK_SIZE,
2718 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2719 CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2720 }
2721 },
2722 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2723 DESC_HDR_SEL0_MDEUA |
2724 DESC_HDR_MODE0_MDEU_SHA1,
2725 },
60f208d7
KP		 2726 { .type = CRYPTO_ALG_TYPE_AHASH,
2727 .alg.hash = {
60f208d7 2728 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 2729 .halg.statesize = sizeof(struct talitos_export_state),
60f208d7
KP		 2730 .halg.base = {
2731 .cra_name = "sha224",
2732 .cra_driver_name = "sha224-talitos",
2733 .cra_blocksize = SHA224_BLOCK_SIZE,
2734 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2735 CRYPTO_ALG_ASYNC,
60f208d7
KP		 2736 }
2737 },
2738 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2739 DESC_HDR_SEL0_MDEUA |
2740 DESC_HDR_MODE0_MDEU_SHA224,
2741 },
497f2e6b
LN		 2742 { .type = CRYPTO_ALG_TYPE_AHASH,
2743 .alg.hash = {
497f2e6b 2744 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 2745 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2746 .halg.base = {
2747 .cra_name = "sha256",
2748 .cra_driver_name = "sha256-talitos",
2749 .cra_blocksize = SHA256_BLOCK_SIZE,
2750 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2751 CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2752 }
2753 },
2754 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2755 DESC_HDR_SEL0_MDEUA |
2756 DESC_HDR_MODE0_MDEU_SHA256,
2757 },
2758 { .type = CRYPTO_ALG_TYPE_AHASH,
2759 .alg.hash = {
497f2e6b 2760 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 2761 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2762 .halg.base = {
2763 .cra_name = "sha384",
2764 .cra_driver_name = "sha384-talitos",
2765 .cra_blocksize = SHA384_BLOCK_SIZE,
2766 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2767 CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2768 }
2769 },
2770 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2771 DESC_HDR_SEL0_MDEUB |
2772 DESC_HDR_MODE0_MDEUB_SHA384,
2773 },
2774 { .type = CRYPTO_ALG_TYPE_AHASH,
2775 .alg.hash = {
497f2e6b 2776 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 2777 .halg.statesize = sizeof(struct talitos_export_state),
497f2e6b
LN		 2778 .halg.base = {
2779 .cra_name = "sha512",
2780 .cra_driver_name = "sha512-talitos",
2781 .cra_blocksize = SHA512_BLOCK_SIZE,
2782 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2783 CRYPTO_ALG_ASYNC,
497f2e6b
LN		 2784 }
2785 },
2786 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2787 DESC_HDR_SEL0_MDEUB |
2788 DESC_HDR_MODE0_MDEUB_SHA512,
2789 },
79b3a418
LN		 2790 { .type = CRYPTO_ALG_TYPE_AHASH,
2791 .alg.hash = {
79b3a418 2792 .halg.digestsize = MD5_DIGEST_SIZE,
3639ca84 2793 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2794 .halg.base = {
2795 .cra_name = "hmac(md5)",
2796 .cra_driver_name = "hmac-md5-talitos",
b3988618 2797 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
79b3a418
LN		 2798 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2799 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2800 }
2801 },
2802 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2803 DESC_HDR_SEL0_MDEUA |
2804 DESC_HDR_MODE0_MDEU_MD5,
2805 },
2806 { .type = CRYPTO_ALG_TYPE_AHASH,
2807 .alg.hash = {
79b3a418 2808 .halg.digestsize = SHA1_DIGEST_SIZE,
3639ca84 2809 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2810 .halg.base = {
2811 .cra_name = "hmac(sha1)",
2812 .cra_driver_name = "hmac-sha1-talitos",
2813 .cra_blocksize = SHA1_BLOCK_SIZE,
2814 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2815 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2816 }
2817 },
2818 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2819 DESC_HDR_SEL0_MDEUA |
2820 DESC_HDR_MODE0_MDEU_SHA1,
2821 },
2822 { .type = CRYPTO_ALG_TYPE_AHASH,
2823 .alg.hash = {
79b3a418 2824 .halg.digestsize = SHA224_DIGEST_SIZE,
3639ca84 2825 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2826 .halg.base = {
2827 .cra_name = "hmac(sha224)",
2828 .cra_driver_name = "hmac-sha224-talitos",
2829 .cra_blocksize = SHA224_BLOCK_SIZE,
2830 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2831 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2832 }
2833 },
2834 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2835 DESC_HDR_SEL0_MDEUA |
2836 DESC_HDR_MODE0_MDEU_SHA224,
2837 },
2838 { .type = CRYPTO_ALG_TYPE_AHASH,
2839 .alg.hash = {
79b3a418 2840 .halg.digestsize = SHA256_DIGEST_SIZE,
3639ca84 2841 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2842 .halg.base = {
2843 .cra_name = "hmac(sha256)",
2844 .cra_driver_name = "hmac-sha256-talitos",
2845 .cra_blocksize = SHA256_BLOCK_SIZE,
2846 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2847 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2848 }
2849 },
2850 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2851 DESC_HDR_SEL0_MDEUA |
2852 DESC_HDR_MODE0_MDEU_SHA256,
2853 },
2854 { .type = CRYPTO_ALG_TYPE_AHASH,
2855 .alg.hash = {
79b3a418 2856 .halg.digestsize = SHA384_DIGEST_SIZE,
3639ca84 2857 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2858 .halg.base = {
2859 .cra_name = "hmac(sha384)",
2860 .cra_driver_name = "hmac-sha384-talitos",
2861 .cra_blocksize = SHA384_BLOCK_SIZE,
2862 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2863 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2864 }
2865 },
2866 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2867 DESC_HDR_SEL0_MDEUB |
2868 DESC_HDR_MODE0_MDEUB_SHA384,
2869 },
2870 { .type = CRYPTO_ALG_TYPE_AHASH,
2871 .alg.hash = {
79b3a418 2872 .halg.digestsize = SHA512_DIGEST_SIZE,
3639ca84 2873 .halg.statesize = sizeof(struct talitos_export_state),
79b3a418
LN		 2874 .halg.base = {
2875 .cra_name = "hmac(sha512)",
2876 .cra_driver_name = "hmac-sha512-talitos",
2877 .cra_blocksize = SHA512_BLOCK_SIZE,
2878 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2879 CRYPTO_ALG_ASYNC,
79b3a418
LN		 2880 }
2881 },
2882 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2883 DESC_HDR_SEL0_MDEUB |
2884 DESC_HDR_MODE0_MDEUB_SHA512,
2885 }
9c4a7965
KP		 2886};
2887
2888struct talitos_crypto_alg {
2889 struct list_head entry;
2890 struct device *dev;
acbf7c62 2891 struct talitos_alg_template algt;
9c4a7965
KP		 2892};
2893
89d124cb
JE		 2894static int talitos_init_common(struct talitos_ctx *ctx,
2895 struct talitos_crypto_alg *talitos_alg)
9c4a7965 2896{
5228f0f7 2897 struct talitos_private *priv;
9c4a7965
KP		 2898
2899 /* update context with ptr to dev */
2900 ctx->dev = talitos_alg->dev;
19bbbc63 2901
5228f0f7
KP		 2902 /* assign SEC channel to tfm in round-robin fashion */
2903 priv = dev_get_drvdata(ctx->dev);
2904 ctx->ch = atomic_inc_return(&priv->last_chan) &
2905 (priv->num_channels - 1);
2906
9c4a7965 2907 /* copy descriptor header template value */
acbf7c62 2908 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
9c4a7965 2909
602dba5a
KP		 2910 /* select done notification */
2911 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2912
497f2e6b
LN		 2913 return 0;
2914}
2915
89d124cb
JE		 2916static int talitos_cra_init(struct crypto_tfm *tfm)
2917{
2918 struct crypto_alg *alg = tfm->__crt_alg;
2919 struct talitos_crypto_alg *talitos_alg;
2920 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2921
2922 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2923 talitos_alg = container_of(__crypto_ahash_alg(alg),
2924 struct talitos_crypto_alg,
2925 algt.alg.hash);
2926 else
2927 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2928 algt.alg.crypto);
2929
2930 return talitos_init_common(ctx, talitos_alg);
2931}
2932
aeb4c132 2933static int talitos_cra_init_aead(struct crypto_aead *tfm)
497f2e6b 2934{
89d124cb
JE		 2935 struct aead_alg *alg = crypto_aead_alg(tfm);
2936 struct talitos_crypto_alg *talitos_alg;
2937 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2938
2939 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2940 algt.alg.aead);
2941
2942 return talitos_init_common(ctx, talitos_alg);
9c4a7965
KP		 2943}
2944
497f2e6b
LN		 2945static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2946{
2947 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2948
2949 talitos_cra_init(tfm);
2950
2951 ctx->keylen = 0;
2952 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2953 sizeof(struct talitos_ahash_req_ctx));
2954
2955 return 0;
2956}
2957
9c4a7965
KP		 2958/*
2959 * given the alg's descriptor header template, determine whether descriptor
2960 * type and primary/secondary execution units required match the hw
2961 * capabilities description provided in the device tree node.
2962 */
2963static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2964{
2965 struct talitos_private *priv = dev_get_drvdata(dev);
2966 int ret;
2967
2968 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2969 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2970
2971 if (SECONDARY_EU(desc_hdr_template))
2972 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2973 & priv->exec_units);
2974
2975 return ret;
2976}
2977
2dc11581 2978static int talitos_remove(struct platform_device *ofdev)
9c4a7965
KP		 2979{
2980 struct device *dev = &ofdev->dev;
2981 struct talitos_private *priv = dev_get_drvdata(dev);
2982 struct talitos_crypto_alg *t_alg, *n;
2983 int i;
2984
2985 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
acbf7c62
LN		 2986 switch (t_alg->algt.type) {
2987 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62 2988 break;
aeb4c132
HX		 2989 case CRYPTO_ALG_TYPE_AEAD:
2990 crypto_unregister_aead(&t_alg->algt.alg.aead);
acbf7c62
LN		 2991 case CRYPTO_ALG_TYPE_AHASH:
2992 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2993 break;
2994 }
9c4a7965 2995 list_del(&t_alg->entry);
9c4a7965
KP		 2996 }
2997
2998 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2999 talitos_unregister_rng(dev);
3000
c3e337f8 3001 for (i = 0; i < 2; i++)
2cdba3cf 3002 if (priv->irq[i]) {
c3e337f8
KP		 3003 free_irq(priv->irq[i], dev);
3004 irq_dispose_mapping(priv->irq[i]);
3005 }
9c4a7965 3006
c3e337f8 3007 tasklet_kill(&priv->done_task[0]);
2cdba3cf 3008 if (priv->irq[1])
c3e337f8 3009 tasklet_kill(&priv->done_task[1]);
9c4a7965
KP		 3010
3011 iounmap(priv->reg);
3012
9c4a7965
KP		 3013 return 0;
3014}
3015
3016static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3017 struct talitos_alg_template
3018 *template)
3019{
60f208d7 3020 struct talitos_private *priv = dev_get_drvdata(dev);
9c4a7965
KP		 3021 struct talitos_crypto_alg *t_alg;
3022 struct crypto_alg *alg;
3023
24b92ff2
LC		 3024 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3025 GFP_KERNEL);
9c4a7965
KP		 3026 if (!t_alg)
3027 return ERR_PTR(-ENOMEM);
3028
acbf7c62
LN		 3029 t_alg->algt = *template;
3030
3031 switch (t_alg->algt.type) {
3032 case CRYPTO_ALG_TYPE_ABLKCIPHER:
497f2e6b
LN		 3033 alg = &t_alg->algt.alg.crypto;
3034 alg->cra_init = talitos_cra_init;
d4cd3283 3035 alg->cra_type = &crypto_ablkcipher_type;
b286e003
KP		 3036 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3037 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3038 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3039 alg->cra_ablkcipher.geniv = "eseqiv";
497f2e6b 3040 break;
acbf7c62 3041 case CRYPTO_ALG_TYPE_AEAD:
aeb4c132 3042 alg = &t_alg->algt.alg.aead.base;
aeb4c132
HX		 3043 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3044 t_alg->algt.alg.aead.setkey = aead_setkey;
3045 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3046 t_alg->algt.alg.aead.decrypt = aead_decrypt;
6cda075a
LC		 3047 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3048 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
24b92ff2 3049 devm_kfree(dev, t_alg);
6cda075a
LC		 3050 return ERR_PTR(-ENOTSUPP);
3051 }
acbf7c62
LN		 3052 break;
3053 case CRYPTO_ALG_TYPE_AHASH:
3054 alg = &t_alg->algt.alg.hash.halg.base;
497f2e6b 3055 alg->cra_init = talitos_cra_init_ahash;
d4cd3283 3056 alg->cra_type = &crypto_ahash_type;
b286e003
KP		 3057 t_alg->algt.alg.hash.init = ahash_init;
3058 t_alg->algt.alg.hash.update = ahash_update;
3059 t_alg->algt.alg.hash.final = ahash_final;
3060 t_alg->algt.alg.hash.finup = ahash_finup;
3061 t_alg->algt.alg.hash.digest = ahash_digest;
56136631
LC		 3062 if (!strncmp(alg->cra_name, "hmac", 4))
3063 t_alg->algt.alg.hash.setkey = ahash_setkey;
3639ca84
HG		 3064 t_alg->algt.alg.hash.import = ahash_import;
3065 t_alg->algt.alg.hash.export = ahash_export;
b286e003 3066
79b3a418 3067 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
0b2730d8 3068 !strncmp(alg->cra_name, "hmac", 4)) {
24b92ff2 3069 devm_kfree(dev, t_alg);
79b3a418 3070 return ERR_PTR(-ENOTSUPP);
0b2730d8 3071 }
60f208d7 3072 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
79b3a418
LN		 3073 (!strcmp(alg->cra_name, "sha224") ||
3074 !strcmp(alg->cra_name, "hmac(sha224)"))) {
60f208d7
KP		 3075 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3076 t_alg->algt.desc_hdr_template =
3077 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3078 DESC_HDR_SEL0_MDEUA |
3079 DESC_HDR_MODE0_MDEU_SHA256;
3080 }
497f2e6b 3081 break;
1d11911a
KP		 3082 default:
3083 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
24b92ff2 3084 devm_kfree(dev, t_alg);
1d11911a 3085 return ERR_PTR(-EINVAL);
acbf7c62 3086 }
9c4a7965 3087
9c4a7965 3088 alg->cra_module = THIS_MODULE;
b0057763
LC		 3089 if (t_alg->algt.priority)
3090 alg->cra_priority = t_alg->algt.priority;
3091 else
3092 alg->cra_priority = TALITOS_CRA_PRIORITY;
9c4a7965 3093 alg->cra_alignmask = 0;
9c4a7965 3094 alg->cra_ctxsize = sizeof(struct talitos_ctx);
d912bb76 3095 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
9c4a7965 3096
9c4a7965
KP		 3097 t_alg->dev = dev;
3098
3099 return t_alg;
3100}
3101
c3e337f8
KP		 3102static int talitos_probe_irq(struct platform_device *ofdev)
3103{
3104 struct device *dev = &ofdev->dev;
3105 struct device_node *np = ofdev->dev.of_node;
3106 struct talitos_private *priv = dev_get_drvdata(dev);
3107 int err;
dd3c0987 3108 bool is_sec1 = has_ftr_sec1(priv);
c3e337f8
KP		 3109
3110 priv->irq[0] = irq_of_parse_and_map(np, 0);
2cdba3cf 3111 if (!priv->irq[0]) {
c3e337f8
KP		 3112 dev_err(dev, "failed to map irq\n");
3113 return -EINVAL;
3114 }
dd3c0987
LC		 3115 if (is_sec1) {
3116 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3117 dev_driver_string(dev), dev);
3118 goto primary_out;
3119 }
c3e337f8
KP		 3120
3121 priv->irq[1] = irq_of_parse_and_map(np, 1);
3122
3123 /* get the primary irq line */
2cdba3cf 3124 if (!priv->irq[1]) {
dd3c0987 3125 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
c3e337f8
KP		 3126 dev_driver_string(dev), dev);
3127 goto primary_out;
3128 }
3129
dd3c0987 3130 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
c3e337f8
KP		 3131 dev_driver_string(dev), dev);
3132 if (err)
3133 goto primary_out;
3134
3135 /* get the secondary irq line */
dd3c0987 3136 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
c3e337f8
KP		 3137 dev_driver_string(dev), dev);
3138 if (err) {
3139 dev_err(dev, "failed to request secondary irq\n");
3140 irq_dispose_mapping(priv->irq[1]);
2cdba3cf 3141 priv->irq[1] = 0;
c3e337f8
KP		 3142 }
3143
3144 return err;
3145
3146primary_out:
3147 if (err) {
3148 dev_err(dev, "failed to request primary irq\n");
3149 irq_dispose_mapping(priv->irq[0]);
2cdba3cf 3150 priv->irq[0] = 0;
c3e337f8
KP		 3151 }
3152
3153 return err;
3154}
3155
1c48a5c9 3156static int talitos_probe(struct platform_device *ofdev)
9c4a7965
KP		 3157{
3158 struct device *dev = &ofdev->dev;
61c7a080 3159 struct device_node *np = ofdev->dev.of_node;
9c4a7965
KP		 3160 struct talitos_private *priv;
3161 const unsigned int *prop;
3162 int i, err;
5fa7fa14 3163 int stride;
9c4a7965 3164
24b92ff2 3165 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
9c4a7965
KP		 3166 if (!priv)
3167 return -ENOMEM;
3168
f3de9cb1
KH		 3169 INIT_LIST_HEAD(&priv->alg_list);
3170
9c4a7965
KP		 3171 dev_set_drvdata(dev, priv);
3172
3173 priv->ofdev = ofdev;
3174
511d63cb
HG		 3175 spin_lock_init(&priv->reg_lock);
3176
9c4a7965
KP		 3177 priv->reg = of_iomap(np, 0);
3178 if (!priv->reg) {
3179 dev_err(dev, "failed to of_iomap\n");
3180 err = -ENOMEM;
3181 goto err_out;
3182 }
3183
3184 /* get SEC version capabilities from device tree */
3185 prop = of_get_property(np, "fsl,num-channels", NULL);
3186 if (prop)
3187 priv->num_channels = *prop;
3188
3189 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3190 if (prop)
3191 priv->chfifo_len = *prop;
3192
3193 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3194 if (prop)
3195 priv->exec_units = *prop;
3196
3197 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3198 if (prop)
3199 priv->desc_types = *prop;
3200
3201 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3202 !priv->exec_units || !priv->desc_types) {
3203 dev_err(dev, "invalid property data in device tree node\n");
3204 err = -EINVAL;
3205 goto err_out;
3206 }
3207
f3c85bc1
LN		 3208 if (of_device_is_compatible(np, "fsl,sec3.0"))
3209 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3210
fe5720e2 3211 if (of_device_is_compatible(np, "fsl,sec2.1"))
60f208d7 3212 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
79b3a418
LN		 3213 TALITOS_FTR_SHA224_HWINIT |
3214 TALITOS_FTR_HMAC_OK;
fe5720e2 3215
21590888
LC		 3216 if (of_device_is_compatible(np, "fsl,sec1.0"))
3217 priv->features |= TALITOS_FTR_SEC1;
3218
5fa7fa14
LC		 3219 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3220 priv->reg_deu = priv->reg + TALITOS12_DEU;
3221 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3222 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3223 stride = TALITOS1_CH_STRIDE;
3224 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3225 priv->reg_deu = priv->reg + TALITOS10_DEU;
3226 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3227 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3228 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3229 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3230 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3231 stride = TALITOS1_CH_STRIDE;
3232 } else {
3233 priv->reg_deu = priv->reg + TALITOS2_DEU;
3234 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3235 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3236 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3237 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3238 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3239 priv->reg_keu = priv->reg + TALITOS2_KEU;
3240 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3241 stride = TALITOS2_CH_STRIDE;
3242 }
3243
dd3c0987
LC		 3244 err = talitos_probe_irq(ofdev);
3245 if (err)
3246 goto err_out;
3247
3248 if (of_device_is_compatible(np, "fsl,sec1.0")) {
3249 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3250 (unsigned long)dev);
3251 } else {
3252 if (!priv->irq[1]) {
3253 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3254 (unsigned long)dev);
3255 } else {
3256 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3257 (unsigned long)dev);
3258 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3259 (unsigned long)dev);
3260 }
3261 }
3262
24b92ff2
LC		 3263 priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) *
3264 priv->num_channels, GFP_KERNEL);
4b992628
KP		 3265 if (!priv->chan) {
3266 dev_err(dev, "failed to allocate channel management space\n");
9c4a7965
KP		 3267 err = -ENOMEM;
3268 goto err_out;
3269 }
3270
f641dddd
MH		 3271 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3272
c3e337f8 3273 for (i = 0; i < priv->num_channels; i++) {
5fa7fa14 3274 priv->chan[i].reg = priv->reg + stride * (i + 1);
2cdba3cf 3275 if (!priv->irq[1] || !(i & 1))
c3e337f8 3276 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
ad42d5fc 3277
4b992628
KP		 3278 spin_lock_init(&priv->chan[i].head_lock);
3279 spin_lock_init(&priv->chan[i].tail_lock);
9c4a7965 3280
24b92ff2
LC		 3281 priv->chan[i].fifo = devm_kzalloc(dev,
3282 sizeof(struct talitos_request) *
3283 priv->fifo_len, GFP_KERNEL);
4b992628 3284 if (!priv->chan[i].fifo) {
9c4a7965
KP		 3285 dev_err(dev, "failed to allocate request fifo %d\n", i);
3286 err = -ENOMEM;
3287 goto err_out;
3288 }
9c4a7965 3289
4b992628
KP		 3290 atomic_set(&priv->chan[i].submit_count,
3291 -(priv->chfifo_len - 1));
f641dddd 3292 }
9c4a7965 3293
81eb024c
KP		 3294 dma_set_mask(dev, DMA_BIT_MASK(36));
3295
9c4a7965
KP		 3296 /* reset and initialize the h/w */
3297 err = init_device(dev);
3298 if (err) {
3299 dev_err(dev, "failed to initialize device\n");
3300 goto err_out;
3301 }
3302
3303 /* register the RNG, if available */
3304 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3305 err = talitos_register_rng(dev);
3306 if (err) {
3307 dev_err(dev, "failed to register hwrng: %d\n", err);
3308 goto err_out;
3309 } else
3310 dev_info(dev, "hwrng\n");
3311 }
3312
3313 /* register crypto algorithms the device supports */
9c4a7965
KP		 3314 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3315 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3316 struct talitos_crypto_alg *t_alg;
aeb4c132 3317 struct crypto_alg *alg = NULL;
9c4a7965
KP		 3318
3319 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3320 if (IS_ERR(t_alg)) {
3321 err = PTR_ERR(t_alg);
0b2730d8 3322 if (err == -ENOTSUPP)
79b3a418 3323 continue;
9c4a7965
KP		 3324 goto err_out;
3325 }
3326
acbf7c62
LN		 3327 switch (t_alg->algt.type) {
3328 case CRYPTO_ALG_TYPE_ABLKCIPHER:
acbf7c62
LN		 3329 err = crypto_register_alg(
3330 &t_alg->algt.alg.crypto);
aeb4c132 3331 alg = &t_alg->algt.alg.crypto;
acbf7c62 3332 break;
aeb4c132
HX		 3333
3334 case CRYPTO_ALG_TYPE_AEAD:
3335 err = crypto_register_aead(
3336 &t_alg->algt.alg.aead);
3337 alg = &t_alg->algt.alg.aead.base;
3338 break;
3339
acbf7c62
LN		 3340 case CRYPTO_ALG_TYPE_AHASH:
3341 err = crypto_register_ahash(
3342 &t_alg->algt.alg.hash);
aeb4c132 3343 alg = &t_alg->algt.alg.hash.halg.base;
acbf7c62
LN		 3344 break;
3345 }
9c4a7965
KP		 3346 if (err) {
3347 dev_err(dev, "%s alg registration failed\n",
aeb4c132 3348 alg->cra_driver_name);
24b92ff2 3349 devm_kfree(dev, t_alg);
991155ba 3350 } else
9c4a7965 3351 list_add_tail(&t_alg->entry, &priv->alg_list);
9c4a7965
KP		 3352 }
3353 }
5b859b6e
KP		 3354 if (!list_empty(&priv->alg_list))
3355 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3356 (char *)of_get_property(np, "compatible", NULL));
9c4a7965
KP		 3357
3358 return 0;
3359
3360err_out:
3361 talitos_remove(ofdev);
9c4a7965
KP		 3362
3363 return err;
3364}
3365
6c3f975a 3366static const struct of_device_id talitos_match[] = {
0635b7db
LC		 3367#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3368 {
3369 .compatible = "fsl,sec1.0",
3370 },
3371#endif
3372#ifdef CONFIG_CRYPTO_DEV_TALITOS2
9c4a7965
KP		 3373 {
3374 .compatible = "fsl,sec2.0",
3375 },
0635b7db 3376#endif
9c4a7965
KP		 3377 {},
3378};
3379MODULE_DEVICE_TABLE(of, talitos_match);
3380
1c48a5c9 3381static struct platform_driver talitos_driver = {
4018294b
GL		 3382 .driver = {
3383 .name = "talitos",
4018294b
GL		 3384 .of_match_table = talitos_match,
3385 },
9c4a7965 3386 .probe = talitos_probe,
596f1034 3387 .remove = talitos_remove,
9c4a7965
KP		 3388};
3389
741e8c2d 3390module_platform_driver(talitos_driver);
9c4a7965
KP		 3391
3392MODULE_LICENSE("GPL");
3393MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3394MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
Linux 6.x block layer and io_uring tree(s)