projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'pci-v6.16-fixes-3' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci
[linux-2.6-block.git] / drivers / crypto / caam / caamalg.c
CommitLineData
618b5dc4 1// SPDX-License-Identifier: GPL-2.0+
8e8ec596
KP		 2/*
3 * caam - Freescale FSL CAAM support for crypto API
4 *
5 * Copyright 2008-2011 Freescale Semiconductor, Inc.
ae1dd17d 6 * Copyright 2016-2019, 2023 NXP
8e8ec596
KP		 7 *
8 * Based on talitos crypto API driver.
9 *
10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
11 *
12 * --------------- ---------------
13 * | JobDesc #1 |-------------------->| ShareDesc |
14 * | *(packet 1) | | (PDB) |
15 * --------------- |------------->| (hashKey) |
16 * . | | (cipherKey) |
17 * . | |-------->| (operation) |
18 * --------------- | | ---------------
19 * | JobDesc #2 |------| |
20 * | *(packet 2) | |
21 * --------------- |
22 * . |
23 * . |
24 * --------------- |
25 * | JobDesc #3 |------------
26 * | *(packet 3) |
27 * ---------------
28 *
29 * The SharedDesc never changes for a connection unless rekeyed, but
30 * each packet will likely be in a different place. So all we need
31 * to know to process the packet is where the input is, where the
32 * output goes, and what context we want to process with. Context is
33 * in the SharedDesc, packet references in the JobDesc.
34 *
35 * So, a job desc looks like:
36 *
37 * ---------------------
38 * | Header |
39 * | ShareDesc Pointer |
40 * | SEQ_OUT_PTR |
41 * | (output buffer) |
6ec47334 42 * | (output length) |
8e8ec596
KP		 43 * | SEQ_IN_PTR |
44 * | (input buffer) |
6ec47334 45 * | (input length) |
8e8ec596
KP		 46 * ---------------------
47 */
48
49#include "compat.h"
50
51#include "regs.h"
52#include "intern.h"
53#include "desc_constr.h"
54#include "jr.h"
55#include "error.h"
a299c837 56#include "sg_sw_sec4.h"
4c1ec1f9 57#include "key_gen.h"
8cea7b66 58#include "caamalg_desc.h"
5f60d5f6 59#include <linux/unaligned.h>
623814c0 60#include <crypto/internal/aead.h>
4ac1a2d8 61#include <crypto/internal/engine.h>
623814c0 62#include <crypto/internal/skcipher.h>
4ac1a2d8 63#include <crypto/xts.h>
199354d7 64#include <linux/dma-mapping.h>
660ca947
HX		 65#include <linux/device.h>
66#include <linux/err.h>
623814c0 67#include <linux/module.h>
199354d7 68#include <linux/kernel.h>
660ca947
HX		 69#include <linux/slab.h>
70#include <linux/string.h>
8e8ec596
KP		 71
72/*
73 * crypto alg
74 */
75#define CAAM_CRA_PRIORITY 3000
76/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
77#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
daebc465 78 CTR_RFC3686_NONCE_SIZE + \
8e8ec596 79 SHA512_DIGEST_SIZE * 2)
8e8ec596 80
f2147b88
HX		 81#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
82#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
83 CAAM_CMD_SZ * 4)
479bcc7c
HX		 84#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
85 CAAM_CMD_SZ * 5)
f2147b88 86
d6bbd4ee
HG		 87#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
88
1a3daadc 89#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
87e51b07 90#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
4427b1b4 91
479bcc7c
HX		 92struct caam_alg_entry {
93 int class1_alg_type;
94 int class2_alg_type;
479bcc7c
HX		 95 bool rfc3686;
96 bool geniv;
24586b5f 97 bool nodkp;
479bcc7c
HX		 98};
99
100struct caam_aead_alg {
623814c0 101 struct aead_engine_alg aead;
479bcc7c
HX		 102 struct caam_alg_entry caam;
103 bool registered;
104};
105
5ca7badb 106struct caam_skcipher_alg {
623814c0 107 struct skcipher_engine_alg skcipher;
5ca7badb
HG		 108 struct caam_alg_entry caam;
109 bool registered;
110};
111
8e8ec596
KP		 112/*
113 * per-session context
114 */
115struct caam_ctx {
1acebad3
YK		 116 u32 sh_desc_enc[DESC_MAX_USED_LEN];
117 u32 sh_desc_dec[DESC_MAX_USED_LEN];
bbf22344 118 u8 key[CAAM_MAX_KEY_SIZE];
1acebad3
YK		 119 dma_addr_t sh_desc_enc_dma;
120 dma_addr_t sh_desc_dec_dma;
885e9e2f 121 dma_addr_t key_dma;
7e0880b9 122 enum dma_data_direction dir;
bbf22344 123 struct device *jrdev;
db57656b
HG		 124 struct alginfo adata;
125 struct alginfo cdata;
8e8ec596 126 unsigned int authsize;
c91f7348 127 bool xts_key_fallback;
9d9b14db 128 struct crypto_skcipher *fallback;
8e8ec596
KP		 129};
130
ee38767f
IP		 131struct caam_skcipher_req_ctx {
132 struct skcipher_edesc *edesc;
9d9b14db 133 struct skcipher_request fallback_req;
ee38767f
IP		 134};
135
1c240226
IP		 136struct caam_aead_req_ctx {
137 struct aead_edesc *edesc;
138};
139
ae4a825f
HG		 140static int aead_null_set_sh_desc(struct crypto_aead *aead)
141{
4cb4f7c1 142 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
ae4a825f 143 struct device *jrdev = ctx->jrdev;
7e0880b9 144 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
ae4a825f 145 u32 *desc;
4cbe79cc
HG		 146 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
147 ctx->adata.keylen_pad;
ae4a825f
HG		 148
149 /*
150 * Job Descriptor and Shared Descriptors
151 * must all fit into the 64-word Descriptor h/w Buffer
152 */
4cbe79cc 153 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
db57656b 154 ctx->adata.key_inline = true;
9c0bc511 155 ctx->adata.key_virt = ctx->key;
db57656b
HG		 156 } else {
157 ctx->adata.key_inline = false;
9c0bc511 158 ctx->adata.key_dma = ctx->key_dma;
db57656b 159 }
ae4a825f 160
479bcc7c 161 /* aead_encrypt shared descriptor */
ae4a825f 162 desc = ctx->sh_desc_enc;
7e0880b9
HG		 163 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
164 ctrlpriv->era);
bbf22344 165 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 166 desc_bytes(desc), ctx->dir);
ae4a825f
HG		 167
168 /*
169 * Job Descriptor and Shared Descriptors
170 * must all fit into the 64-word Descriptor h/w Buffer
171 */
4cbe79cc 172 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
db57656b 173 ctx->adata.key_inline = true;
9c0bc511 174 ctx->adata.key_virt = ctx->key;
db57656b
HG		 175 } else {
176 ctx->adata.key_inline = false;
9c0bc511 177 ctx->adata.key_dma = ctx->key_dma;
db57656b 178 }
ae4a825f 179
479bcc7c 180 /* aead_decrypt shared descriptor */
8cea7b66 181 desc = ctx->sh_desc_dec;
7e0880b9
HG		 182 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
183 ctrlpriv->era);
bbf22344 184 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 185 desc_bytes(desc), ctx->dir);
ae4a825f
HG		 186
187 return 0;
188}
189
1acebad3
YK		 190static int aead_set_sh_desc(struct crypto_aead *aead)
191{
479bcc7c 192 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
623814c0
HX		 193 struct caam_aead_alg,
194 aead.base);
add86d55 195 unsigned int ivsize = crypto_aead_ivsize(aead);
4cb4f7c1 196 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1acebad3 197 struct device *jrdev = ctx->jrdev;
7e0880b9 198 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
daebc465 199 u32 ctx1_iv_off = 0;
8cea7b66 200 u32 *desc, *nonce = NULL;
4cbe79cc
HG		 201 u32 inl_mask;
202 unsigned int data_len[2];
db57656b 203 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
daebc465 204 OP_ALG_AAI_CTR_MOD128);
479bcc7c 205 const bool is_rfc3686 = alg->caam.rfc3686;
1acebad3 206
2fdea258
HG		 207 if (!ctx->authsize)
208 return 0;
209
ae4a825f 210 /* NULL encryption / decryption */
db57656b 211 if (!ctx->cdata.keylen)
ae4a825f
HG		 212 return aead_null_set_sh_desc(aead);
213
daebc465
CV		 214 /*
215 * AES-CTR needs to load IV in CONTEXT1 reg
216 * at an offset of 128bits (16bytes)
217 * CONTEXT1[255:128] = IV
218 */
219 if (ctr_mode)
220 ctx1_iv_off = 16;
221
222 /*
223 * RFC3686 specific:
224 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
225 */
8cea7b66 226 if (is_rfc3686) {
daebc465 227 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
8cea7b66
HG		 228 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
229 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
230 }
daebc465 231
e9b4913a
HG		 232 /*
233 * In case |user key| > |derived key|, using DKP<imm,imm>
234 * would result in invalid opcodes (last bytes of user key) in
235 * the resulting descriptor. Use DKP<ptr,imm> instead => both
236 * virtual and dma key addresses are needed.
237 */
238 ctx->adata.key_virt = ctx->key;
239 ctx->adata.key_dma = ctx->key_dma;
240
241 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
242 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
243
4cbe79cc
HG		 244 data_len[0] = ctx->adata.keylen_pad;
245 data_len[1] = ctx->cdata.keylen;
246
479bcc7c
HX		 247 if (alg->caam.geniv)
248 goto skip_enc;
249
1acebad3
YK		 250 /*
251 * Job Descriptor and Shared Descriptors
252 * must all fit into the 64-word Descriptor h/w Buffer
253 */
4cbe79cc
HG		 254 if (desc_inline_query(DESC_AEAD_ENC_LEN +
255 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
256 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
257 ARRAY_SIZE(data_len)) < 0)
258 return -EINVAL;
259
4cbe79cc
HG		 260 ctx->adata.key_inline = !!(inl_mask & 1);
261 ctx->cdata.key_inline = !!(inl_mask & 2);
1acebad3 262
479bcc7c 263 /* aead_encrypt shared descriptor */
1acebad3 264 desc = ctx->sh_desc_enc;
b189817c
HG		 265 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
266 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
7e0880b9 267 false, ctrlpriv->era);
bbf22344 268 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 269 desc_bytes(desc), ctx->dir);
1acebad3 270
479bcc7c 271skip_enc:
1acebad3
YK		 272 /*
273 * Job Descriptor and Shared Descriptors
274 * must all fit into the 64-word Descriptor h/w Buffer
275 */
4cbe79cc
HG		 276 if (desc_inline_query(DESC_AEAD_DEC_LEN +
277 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
278 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
279 ARRAY_SIZE(data_len)) < 0)
280 return -EINVAL;
281
4cbe79cc
HG		 282 ctx->adata.key_inline = !!(inl_mask & 1);
283 ctx->cdata.key_inline = !!(inl_mask & 2);
1acebad3 284
479bcc7c 285 /* aead_decrypt shared descriptor */
4464a7d4 286 desc = ctx->sh_desc_dec;
8cea7b66
HG		 287 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
288 ctx->authsize, alg->caam.geniv, is_rfc3686,
7e0880b9 289 nonce, ctx1_iv_off, false, ctrlpriv->era);
bbf22344 290 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 291 desc_bytes(desc), ctx->dir);
1acebad3 292
479bcc7c
HX		 293 if (!alg->caam.geniv)
294 goto skip_givenc;
295
1acebad3
YK		 296 /*
297 * Job Descriptor and Shared Descriptors
298 * must all fit into the 64-word Descriptor h/w Buffer
299 */
4cbe79cc
HG		 300 if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
301 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
302 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
303 ARRAY_SIZE(data_len)) < 0)
304 return -EINVAL;
305
4cbe79cc
HG		 306 ctx->adata.key_inline = !!(inl_mask & 1);
307 ctx->cdata.key_inline = !!(inl_mask & 2);
1acebad3
YK		 308
309 /* aead_givencrypt shared descriptor */
1d2d87e8 310 desc = ctx->sh_desc_enc;
8cea7b66
HG		 311 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
312 ctx->authsize, is_rfc3686, nonce,
7e0880b9 313 ctx1_iv_off, false, ctrlpriv->era);
bbf22344 314 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 315 desc_bytes(desc), ctx->dir);
1acebad3 316
479bcc7c 317skip_givenc:
1acebad3
YK		 318 return 0;
319}
320
0e479300 321static int aead_setauthsize(struct crypto_aead *authenc,
8e8ec596
KP		 322 unsigned int authsize)
323{
4cb4f7c1 324 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
8e8ec596
KP		 325
326 ctx->authsize = authsize;
1acebad3 327 aead_set_sh_desc(authenc);
8e8ec596
KP		 328
329 return 0;
330}
331
3ef8d945
TA		 332static int gcm_set_sh_desc(struct crypto_aead *aead)
333{
4cb4f7c1 334 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
3ef8d945 335 struct device *jrdev = ctx->jrdev;
87ec3a0b 336 unsigned int ivsize = crypto_aead_ivsize(aead);
3ef8d945 337 u32 *desc;
4cbe79cc
HG		 338 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
339 ctx->cdata.keylen;
3ef8d945 340
db57656b 341 if (!ctx->cdata.keylen || !ctx->authsize)
3ef8d945
TA		 342 return 0;
343
344 /*
345 * AES GCM encrypt shared descriptor
346 * Job Descriptor and Shared Descriptor
347 * must fit into the 64-word Descriptor h/w Buffer
348 */
4cbe79cc 349 if (rem_bytes >= DESC_GCM_ENC_LEN) {
db57656b 350 ctx->cdata.key_inline = true;
9c0bc511 351 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 352 } else {
353 ctx->cdata.key_inline = false;
9c0bc511 354 ctx->cdata.key_dma = ctx->key_dma;
db57656b 355 }
3ef8d945
TA		 356
357 desc = ctx->sh_desc_enc;
87ec3a0b 358 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
bbf22344 359 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 360 desc_bytes(desc), ctx->dir);
3ef8d945
TA		 361
362 /*
363 * Job Descriptor and Shared Descriptors
364 * must all fit into the 64-word Descriptor h/w Buffer
365 */
4cbe79cc 366 if (rem_bytes >= DESC_GCM_DEC_LEN) {
db57656b 367 ctx->cdata.key_inline = true;
9c0bc511 368 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 369 } else {
370 ctx->cdata.key_inline = false;
9c0bc511 371 ctx->cdata.key_dma = ctx->key_dma;
db57656b 372 }
3ef8d945
TA		 373
374 desc = ctx->sh_desc_dec;
87ec3a0b 375 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
bbf22344 376 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 377 desc_bytes(desc), ctx->dir);
3ef8d945
TA		 378
379 return 0;
380}
381
382static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
383{
4cb4f7c1 384 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
68a51394
IP		 385 int err;
386
387 err = crypto_gcm_check_authsize(authsize);
388 if (err)
389 return err;
3ef8d945
TA		 390
391 ctx->authsize = authsize;
392 gcm_set_sh_desc(authenc);
393
394 return 0;
395}
396
bac68f2c
TA		 397static int rfc4106_set_sh_desc(struct crypto_aead *aead)
398{
4cb4f7c1 399 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
bac68f2c 400 struct device *jrdev = ctx->jrdev;
87ec3a0b 401 unsigned int ivsize = crypto_aead_ivsize(aead);
bac68f2c 402 u32 *desc;
4cbe79cc
HG		 403 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
404 ctx->cdata.keylen;
bac68f2c 405
db57656b 406 if (!ctx->cdata.keylen || !ctx->authsize)
bac68f2c
TA		 407 return 0;
408
409 /*
410 * RFC4106 encrypt shared descriptor
411 * Job Descriptor and Shared Descriptor
412 * must fit into the 64-word Descriptor h/w Buffer
413 */
4cbe79cc 414 if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
db57656b 415 ctx->cdata.key_inline = true;
9c0bc511 416 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 417 } else {
418 ctx->cdata.key_inline = false;
9c0bc511 419 ctx->cdata.key_dma = ctx->key_dma;
db57656b 420 }
bac68f2c
TA		 421
422 desc = ctx->sh_desc_enc;
87ec3a0b
HG		 423 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
424 false);
bbf22344 425 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 426 desc_bytes(desc), ctx->dir);
bac68f2c
TA		 427
428 /*
429 * Job Descriptor and Shared Descriptors
430 * must all fit into the 64-word Descriptor h/w Buffer
431 */
4cbe79cc 432 if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
db57656b 433 ctx->cdata.key_inline = true;
9c0bc511 434 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 435 } else {
436 ctx->cdata.key_inline = false;
9c0bc511 437 ctx->cdata.key_dma = ctx->key_dma;
db57656b 438 }
bac68f2c
TA		 439
440 desc = ctx->sh_desc_dec;
87ec3a0b
HG		 441 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
442 false);
bbf22344 443 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 444 desc_bytes(desc), ctx->dir);
bac68f2c 445
bac68f2c
TA		 446 return 0;
447}
448
449static int rfc4106_setauthsize(struct crypto_aead *authenc,
450 unsigned int authsize)
451{
4cb4f7c1 452 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
68a51394
IP		 453 int err;
454
455 err = crypto_rfc4106_check_authsize(authsize);
456 if (err)
457 return err;
bac68f2c
TA		 458
459 ctx->authsize = authsize;
460 rfc4106_set_sh_desc(authenc);
461
462 return 0;
463}
464
5d0429a3
TA		 465static int rfc4543_set_sh_desc(struct crypto_aead *aead)
466{
4cb4f7c1 467 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
5d0429a3 468 struct device *jrdev = ctx->jrdev;
87ec3a0b 469 unsigned int ivsize = crypto_aead_ivsize(aead);
5d0429a3 470 u32 *desc;
4cbe79cc
HG		 471 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
472 ctx->cdata.keylen;
5d0429a3 473
db57656b 474 if (!ctx->cdata.keylen || !ctx->authsize)
5d0429a3
TA		 475 return 0;
476
477 /*
478 * RFC4543 encrypt shared descriptor
479 * Job Descriptor and Shared Descriptor
480 * must fit into the 64-word Descriptor h/w Buffer
481 */
4cbe79cc 482 if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
db57656b 483 ctx->cdata.key_inline = true;
9c0bc511 484 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 485 } else {
486 ctx->cdata.key_inline = false;
9c0bc511 487 ctx->cdata.key_dma = ctx->key_dma;
db57656b 488 }
5d0429a3
TA		 489
490 desc = ctx->sh_desc_enc;
87ec3a0b
HG		 491 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
492 false);
bbf22344 493 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 494 desc_bytes(desc), ctx->dir);
5d0429a3
TA		 495
496 /*
497 * Job Descriptor and Shared Descriptors
498 * must all fit into the 64-word Descriptor h/w Buffer
499 */
4cbe79cc 500 if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
db57656b 501 ctx->cdata.key_inline = true;
9c0bc511 502 ctx->cdata.key_virt = ctx->key;
db57656b
HG		 503 } else {
504 ctx->cdata.key_inline = false;
9c0bc511 505 ctx->cdata.key_dma = ctx->key_dma;
db57656b 506 }
5d0429a3
TA		 507
508 desc = ctx->sh_desc_dec;
87ec3a0b
HG		 509 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
510 false);
bbf22344 511 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 512 desc_bytes(desc), ctx->dir);
5d0429a3 513
f2147b88
HX		 514 return 0;
515}
5d0429a3 516
f2147b88
HX		 517static int rfc4543_setauthsize(struct crypto_aead *authenc,
518 unsigned int authsize)
519{
4cb4f7c1 520 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
5d0429a3 521
68a51394
IP		 522 if (authsize != 16)
523 return -EINVAL;
524
f2147b88
HX		 525 ctx->authsize = authsize;
526 rfc4543_set_sh_desc(authenc);
5d0429a3 527
f2147b88
HX		 528 return 0;
529}
5d0429a3 530
d6bbd4ee
HG		 531static int chachapoly_set_sh_desc(struct crypto_aead *aead)
532{
4cb4f7c1 533 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
d6bbd4ee
HG		 534 struct device *jrdev = ctx->jrdev;
535 unsigned int ivsize = crypto_aead_ivsize(aead);
536 u32 *desc;
537
538 if (!ctx->cdata.keylen || !ctx->authsize)
539 return 0;
540
541 desc = ctx->sh_desc_enc;
542 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
c10a5336 543 ctx->authsize, true, false);
d6bbd4ee
HG		 544 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
545 desc_bytes(desc), ctx->dir);
546
547 desc = ctx->sh_desc_dec;
548 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
c10a5336 549 ctx->authsize, false, false);
d6bbd4ee
HG		 550 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
551 desc_bytes(desc), ctx->dir);
552
553 return 0;
554}
555
556static int chachapoly_setauthsize(struct crypto_aead *aead,
557 unsigned int authsize)
558{
4cb4f7c1 559 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
d6bbd4ee
HG		 560
561 if (authsize != POLY1305_DIGEST_SIZE)
562 return -EINVAL;
563
564 ctx->authsize = authsize;
565 return chachapoly_set_sh_desc(aead);
566}
567
568static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
569 unsigned int keylen)
570{
4cb4f7c1 571 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
d6bbd4ee
HG		 572 unsigned int ivsize = crypto_aead_ivsize(aead);
573 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
574
674f368a 575 if (keylen != CHACHA_KEY_SIZE + saltlen)
d6bbd4ee 576 return -EINVAL;
d6bbd4ee 577
a8d3cdcc
GJ		 578 memcpy(ctx->key, key, keylen);
579 ctx->cdata.key_virt = ctx->key;
d6bbd4ee
HG		 580 ctx->cdata.keylen = keylen - saltlen;
581
582 return chachapoly_set_sh_desc(aead);
583}
584
0e479300 585static int aead_setkey(struct crypto_aead *aead,
8e8ec596
KP		 586 const u8 *key, unsigned int keylen)
587{
4cb4f7c1 588 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
8e8ec596 589 struct device *jrdev = ctx->jrdev;
7e0880b9 590 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
4e6e0b27 591 struct crypto_authenc_keys keys;
8e8ec596
KP		 592 int ret = 0;
593
4e6e0b27 594 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
8e8ec596
KP		 595 goto badkey;
596
6e005503 597 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
4e6e0b27
HG		 598 keys.authkeylen + keys.enckeylen, keys.enckeylen,
599 keys.authkeylen);
6e005503
SH		 600 print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
601 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
8e8ec596 602
7e0880b9
HG		 603 /*
604 * If DKP is supported, use it in the shared descriptor to generate
605 * the split key.
606 */
607 if (ctrlpriv->era >= 6) {
608 ctx->adata.keylen = keys.authkeylen;
609 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
610 OP_ALG_ALGSEL_MASK);
611
612 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
613 goto badkey;
614
615 memcpy(ctx->key, keys.authkey, keys.authkeylen);
616 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
617 keys.enckeylen);
618 dma_sync_single_for_device(jrdev, ctx->key_dma,
619 ctx->adata.keylen_pad +
620 keys.enckeylen, ctx->dir);
621 goto skip_split_key;
622 }
623
6655cb8e
HG		 624 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
625 keys.authkeylen, CAAM_MAX_KEY_SIZE -
626 keys.enckeylen);
8e8ec596 627 if (ret) {
8e8ec596
KP		 628 goto badkey;
629 }
630
631 /* postpend encryption key to auth split key */
db57656b 632 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
bbf22344 633 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
7e0880b9 634 keys.enckeylen, ctx->dir);
6e005503
SH		 635
636 print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
637 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
638 ctx->adata.keylen_pad + keys.enckeylen, 1);
7e0880b9
HG		 639
640skip_split_key:
db57656b 641 ctx->cdata.keylen = keys.enckeylen;
61dab972 642 memzero_explicit(&keys, sizeof(keys));
bbf22344 643 return aead_set_sh_desc(aead);
8e8ec596 644badkey:
61dab972 645 memzero_explicit(&keys, sizeof(keys));
8e8ec596
KP		 646 return -EINVAL;
647}
648
1b52c409
HX		 649static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
650 unsigned int keylen)
651{
652 struct crypto_authenc_keys keys;
1b52c409
HX		 653 int err;
654
655 err = crypto_authenc_extractkeys(&keys, key, keylen);
656 if (unlikely(err))
a628c5a1 657 return err;
1b52c409 658
a628c5a1
AB		 659 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
660 aead_setkey(aead, key, keylen);
1b52c409 661
1b52c409
HX		 662 memzero_explicit(&keys, sizeof(keys));
663 return err;
1b52c409
HX		 664}
665
3ef8d945
TA		 666static int gcm_setkey(struct crypto_aead *aead,
667 const u8 *key, unsigned int keylen)
668{
4cb4f7c1 669 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
3ef8d945 670 struct device *jrdev = ctx->jrdev;
836d8f43
IP		 671 int err;
672
673 err = aes_check_keylen(keylen);
674f368a 674 if (err)
836d8f43 675 return err;
3ef8d945 676
6e005503
SH		 677 print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
678 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
3ef8d945
TA		 679
680 memcpy(ctx->key, key, keylen);
7e0880b9 681 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
db57656b 682 ctx->cdata.keylen = keylen;
3ef8d945 683
bbf22344 684 return gcm_set_sh_desc(aead);
3ef8d945
TA		 685}
686
bac68f2c
TA		 687static int rfc4106_setkey(struct crypto_aead *aead,
688 const u8 *key, unsigned int keylen)
689{
4cb4f7c1 690 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
bac68f2c 691 struct device *jrdev = ctx->jrdev;
836d8f43 692 int err;
bac68f2c 693
836d8f43 694 err = aes_check_keylen(keylen - 4);
674f368a 695 if (err)
836d8f43 696 return err;
bac68f2c 697
6e005503
SH		 698 print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
699 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
bac68f2c
TA		 700
701 memcpy(ctx->key, key, keylen);
702
703 /*
704 * The last four bytes of the key material are used as the salt value
705 * in the nonce. Update the AES key length.
706 */
db57656b 707 ctx->cdata.keylen = keylen - 4;
bbf22344 708 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
7e0880b9 709 ctx->dir);
bbf22344 710 return rfc4106_set_sh_desc(aead);
bac68f2c
TA		 711}
712
5d0429a3
TA		 713static int rfc4543_setkey(struct crypto_aead *aead,
714 const u8 *key, unsigned int keylen)
715{
4cb4f7c1 716 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
5d0429a3 717 struct device *jrdev = ctx->jrdev;
836d8f43 718 int err;
5d0429a3 719
836d8f43 720 err = aes_check_keylen(keylen - 4);
674f368a 721 if (err)
836d8f43 722 return err;
5d0429a3 723
6e005503
SH		 724 print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
725 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
5d0429a3
TA		 726
727 memcpy(ctx->key, key, keylen);
728
729 /*
730 * The last four bytes of the key material are used as the salt value
731 * in the nonce. Update the AES key length.
732 */
db57656b 733 ctx->cdata.keylen = keylen - 4;
bbf22344 734 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
7e0880b9 735 ctx->dir);
bbf22344 736 return rfc4543_set_sh_desc(aead);
5d0429a3
TA		 737}
738
5ca7badb 739static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
836d8f43 740 unsigned int keylen, const u32 ctx1_iv_off)
acdca31d 741{
4cb4f7c1 742 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
5ca7badb
HG		 743 struct caam_skcipher_alg *alg =
744 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
623814c0 745 skcipher.base);
acdca31d 746 struct device *jrdev = ctx->jrdev;
5ca7badb 747 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
acdca31d 748 u32 *desc;
5ca7badb 749 const bool is_rfc3686 = alg->caam.rfc3686;
acdca31d 750
6e005503
SH		 751 print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
752 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
a5f57cff 753
db57656b 754 ctx->cdata.keylen = keylen;
662f70ed 755 ctx->cdata.key_virt = key;
db57656b 756 ctx->cdata.key_inline = true;
acdca31d 757
5ca7badb 758 /* skcipher_encrypt shared descriptor */
acdca31d 759 desc = ctx->sh_desc_enc;
9dbe3072
HG		 760 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
761 ctx1_iv_off);
bbf22344 762 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 763 desc_bytes(desc), ctx->dir);
8cea7b66 764
5ca7badb 765 /* skcipher_decrypt shared descriptor */
acdca31d 766 desc = ctx->sh_desc_dec;
9dbe3072
HG		 767 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
768 ctx1_iv_off);
bbf22344 769 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 770 desc_bytes(desc), ctx->dir);
acdca31d 771
8cea7b66 772 return 0;
acdca31d
YK		 773}
774
836d8f43
IP		 775static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
776 const u8 *key, unsigned int keylen)
777{
778 int err;
779
780 err = aes_check_keylen(keylen);
674f368a 781 if (err)
836d8f43 782 return err;
836d8f43
IP		 783
784 return skcipher_setkey(skcipher, key, keylen, 0);
785}
786
787static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
788 const u8 *key, unsigned int keylen)
789{
790 u32 ctx1_iv_off;
791 int err;
792
793 /*
794 * RFC3686 specific:
795 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
796 * | *key = {KEY, NONCE}
797 */
798 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
799 keylen -= CTR_RFC3686_NONCE_SIZE;
800
801 err = aes_check_keylen(keylen);
674f368a 802 if (err)
836d8f43 803 return err;
836d8f43
IP		 804
805 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
806}
807
808static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
809 const u8 *key, unsigned int keylen)
810{
811 u32 ctx1_iv_off;
812 int err;
813
814 /*
815 * AES-CTR needs to load IV in CONTEXT1 reg
816 * at an offset of 128bits (16bytes)
817 * CONTEXT1[255:128] = IV
818 */
819 ctx1_iv_off = 16;
820
821 err = aes_check_keylen(keylen);
674f368a 822 if (err)
836d8f43 823 return err;
836d8f43
IP		 824
825 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
826}
827
eaed71a4
IP		 828static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
829 const u8 *key, unsigned int keylen)
830{
a628c5a1
AB		 831 return verify_skcipher_des_key(skcipher, key) ?:
832 skcipher_setkey(skcipher, key, keylen, 0);
833}
eaed71a4 834
a628c5a1
AB		 835static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
836 const u8 *key, unsigned int keylen)
837{
838 return verify_skcipher_des3_key(skcipher, key) ?:
839 skcipher_setkey(skcipher, key, keylen, 0);
eaed71a4
IP		 840}
841
5ca7badb
HG		 842static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
843 unsigned int keylen)
c6415a60 844{
4cb4f7c1 845 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
c6415a60 846 struct device *jrdev = ctx->jrdev;
78eebbfa 847 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
8cea7b66 848 u32 *desc;
9d9b14db 849 int err;
c6415a60 850
c91f7348
AB		 851 err = xts_verify_key(skcipher, key, keylen);
852 if (err) {
da6a6685 853 dev_dbg(jrdev, "key size mismatch\n");
c91f7348 854 return err;
c6415a60
CV		 855 }
856
c91f7348
AB		 857 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
858 ctx->xts_key_fallback = true;
859
78eebbfa
AB		 860 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
861 err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
862 if (err)
863 return err;
864 }
9d9b14db 865
db57656b 866 ctx->cdata.keylen = keylen;
662f70ed 867 ctx->cdata.key_virt = key;
db57656b 868 ctx->cdata.key_inline = true;
c6415a60 869
5ca7badb 870 /* xts_skcipher_encrypt shared descriptor */
c6415a60 871 desc = ctx->sh_desc_enc;
9dbe3072 872 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
bbf22344 873 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
7e0880b9 874 desc_bytes(desc), ctx->dir);
c6415a60 875
5ca7badb 876 /* xts_skcipher_decrypt shared descriptor */
c6415a60 877 desc = ctx->sh_desc_dec;
9dbe3072 878 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
bbf22344 879 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
7e0880b9 880 desc_bytes(desc), ctx->dir);
c6415a60
CV		 881
882 return 0;
883}
884
8e8ec596 885/*
1acebad3 886 * aead_edesc - s/w-extended aead descriptor
fa0c92db
HG		 887 * @src_nents: number of segments in input s/w scatterlist
888 * @dst_nents: number of segments in output s/w scatterlist
ba4cf71b
IP		 889 * @mapped_src_nents: number of segments in input h/w link table
890 * @mapped_dst_nents: number of segments in output h/w link table
a299c837 891 * @sec4_sg_bytes: length of dma mapped sec4_sg space
1c240226 892 * @bklog: stored to determine if the request needs backlog
a299c837 893 * @sec4_sg_dma: bus physical mapped address of h/w link table
4ca7c7d8 894 * @sec4_sg: pointer to h/w link table
8e8ec596
KP		 895 * @hw_desc: the h/w job descriptor followed by any referenced link tables
896 */
0e479300 897struct aead_edesc {
8e8ec596
KP		 898 int src_nents;
899 int dst_nents;
ba4cf71b
IP		 900 int mapped_src_nents;
901 int mapped_dst_nents;
a299c837 902 int sec4_sg_bytes;
1c240226 903 bool bklog;
a299c837
YK		 904 dma_addr_t sec4_sg_dma;
905 struct sec4_sg_entry *sec4_sg;
f2147b88 906 u32 hw_desc[];
8e8ec596
KP		 907};
908
acdca31d 909/*
5ca7badb 910 * skcipher_edesc - s/w-extended skcipher descriptor
fa0c92db
HG		 911 * @src_nents: number of segments in input s/w scatterlist
912 * @dst_nents: number of segments in output s/w scatterlist
ba4cf71b
IP		 913 * @mapped_src_nents: number of segments in input h/w link table
914 * @mapped_dst_nents: number of segments in output h/w link table
acdca31d 915 * @iv_dma: dma address of iv for checking continuity and link table
a299c837 916 * @sec4_sg_bytes: length of dma mapped sec4_sg space
ee38767f 917 * @bklog: stored to determine if the request needs backlog
a299c837 918 * @sec4_sg_dma: bus physical mapped address of h/w link table
4ca7c7d8 919 * @sec4_sg: pointer to h/w link table
acdca31d 920 * @hw_desc: the h/w job descriptor followed by any referenced link tables
115957bb 921 * and IV
acdca31d 922 */
5ca7badb 923struct skcipher_edesc {
acdca31d
YK		 924 int src_nents;
925 int dst_nents;
ba4cf71b
IP		 926 int mapped_src_nents;
927 int mapped_dst_nents;
acdca31d 928 dma_addr_t iv_dma;
a299c837 929 int sec4_sg_bytes;
ee38767f 930 bool bklog;
a299c837
YK		 931 dma_addr_t sec4_sg_dma;
932 struct sec4_sg_entry *sec4_sg;
5a8a0765 933 u32 hw_desc[];
acdca31d
YK		 934};
935
1acebad3 936static void caam_unmap(struct device *dev, struct scatterlist *src,
643b39b0 937 struct scatterlist *dst, int src_nents,
13fb8fd7 938 int dst_nents,
cf5448b5 939 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
a299c837 940 int sec4_sg_bytes)
8e8ec596 941{
643b39b0 942 if (dst != src) {
fa0c92db
HG		 943 if (src_nents)
944 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
763069ba
HG		 945 if (dst_nents)
946 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
8e8ec596 947 } else {
fa0c92db 948 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
8e8ec596
KP		 949 }
950
1acebad3 951 if (iv_dma)
334d37c9 952 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
a299c837
YK		 953 if (sec4_sg_bytes)
954 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
8e8ec596
KP		 955 DMA_TO_DEVICE);
956}
957
1acebad3
YK		 958static void aead_unmap(struct device *dev,
959 struct aead_edesc *edesc,
960 struct aead_request *req)
f2147b88
HX		 961{
962 caam_unmap(dev, req->src, req->dst,
cf5448b5 963 edesc->src_nents, edesc->dst_nents, 0, 0,
f2147b88
HX		 964 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
965}
966
5ca7badb
HG		 967static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
968 struct skcipher_request *req)
acdca31d 969{
5ca7badb
HG		 970 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
971 int ivsize = crypto_skcipher_ivsize(skcipher);
acdca31d
YK		 972
973 caam_unmap(dev, req->src, req->dst,
13fb8fd7 974 edesc->src_nents, edesc->dst_nents,
cf5448b5 975 edesc->iv_dma, ivsize,
643b39b0 976 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
acdca31d
YK		 977}
978
b7f17fe2
IP		 979static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
980 void *context)
8e8ec596 981{
0e479300 982 struct aead_request *req = context;
1c240226
IP		 983 struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
984 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
0e479300 985 struct aead_edesc *edesc;
1984aaee 986 int ecode = 0;
5ed1e8b8 987 bool has_bklog;
f2147b88 988
6e005503 989 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
f2147b88 990
1c240226 991 edesc = rctx->edesc;
5ed1e8b8 992 has_bklog = edesc->bklog;
f2147b88
HX		 993
994 if (err)
1984aaee 995 ecode = caam_jr_strstatus(jrdev, err);
f2147b88
HX		 996
997 aead_unmap(jrdev, edesc, req);
998
999 kfree(edesc);
1000
1c240226
IP		 1001 /*
1002 * If no backlog flag, the completion of the request is done
1003 * by CAAM, not crypto engine.
1004 */
5ed1e8b8 1005 if (!has_bklog)
1c240226
IP		 1006 aead_request_complete(req, ecode);
1007 else
1008 crypto_finalize_aead_request(jrp->engine, req, ecode);
f2147b88
HX		 1009}
1010
660ca947
HX		 1011static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc)
1012{
1013
1014 return PTR_ALIGN((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
1015 dma_get_cache_alignment());
1016}
1017
b7f17fe2
IP		 1018static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
1019 void *context)
acdca31d 1020{
5ca7badb
HG		 1021 struct skcipher_request *req = context;
1022 struct skcipher_edesc *edesc;
ee38767f 1023 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
5ca7badb 1024 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
ee38767f 1025 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
5ca7badb 1026 int ivsize = crypto_skcipher_ivsize(skcipher);
1984aaee 1027 int ecode = 0;
5af4e8d4 1028 bool has_bklog;
acdca31d 1029
6e005503 1030 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
acdca31d 1031
ee38767f 1032 edesc = rctx->edesc;
5af4e8d4 1033 has_bklog = edesc->bklog;
fa9659cd 1034 if (err)
1984aaee 1035 ecode = caam_jr_strstatus(jrdev, err);
acdca31d 1036
bb992bc4
SH		 1037 skcipher_unmap(jrdev, edesc, req);
1038
334d37c9
HG		 1039 /*
1040 * The crypto API expects us to set the IV (req->iv) to the last
1041 * ciphertext block (CBC mode) or last counter (CTR mode).
1042 * This is used e.g. by the CTS mode.
1043 */
1ccb39eb 1044 if (ivsize && !ecode) {
660ca947 1045 memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
334d37c9
HG		 1046
1047 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
1048 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1049 ivsize, 1);
1050 }
1051
8a82451b 1052 caam_dump_sg("dst @" __stringify(__LINE__)": ",
972b812b 1053 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
5ca7badb 1054 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
acdca31d 1055
acdca31d
YK		 1056 kfree(edesc);
1057
ee38767f
IP		 1058 /*
1059 * If no backlog flag, the completion of the request is done
1060 * by CAAM, not crypto engine.
1061 */
5af4e8d4 1062 if (!has_bklog)
ee38767f
IP		 1063 skcipher_request_complete(req, ecode);
1064 else
1065 crypto_finalize_skcipher_request(jrp->engine, req, ecode);
acdca31d
YK		 1066}
1067
f2147b88
HX		 1068/*
1069 * Fill in aead job descriptor
1070 */
1071static void init_aead_job(struct aead_request *req,
1072 struct aead_edesc *edesc,
1073 bool all_contig, bool encrypt)
1074{
1075 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1076 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
f2147b88
HX		 1077 int authsize = ctx->authsize;
1078 u32 *desc = edesc->hw_desc;
1079 u32 out_options, in_options;
1080 dma_addr_t dst_dma, src_dma;
1081 int len, sec4_sg_index = 0;
1082 dma_addr_t ptr;
1083 u32 *sh_desc;
1084
1085 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1086 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1087
1088 len = desc_len(sh_desc);
1089 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1090
1091 if (all_contig) {
ba4cf71b
IP		 1092 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
1093 0;
f2147b88
HX		 1094 in_options = 0;
1095 } else {
1096 src_dma = edesc->sec4_sg_dma;
ba4cf71b 1097 sec4_sg_index += edesc->mapped_src_nents;
f2147b88
HX		 1098 in_options = LDST_SGF;
1099 }
1100
1101 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1102 in_options);
1103
1104 dst_dma = src_dma;
1105 out_options = in_options;
1106
1107 if (unlikely(req->src != req->dst)) {
ba4cf71b 1108 if (!edesc->mapped_dst_nents) {
763069ba 1109 dst_dma = 0;
dcd9c76e 1110 out_options = 0;
ba4cf71b 1111 } else if (edesc->mapped_dst_nents == 1) {
f2147b88 1112 dst_dma = sg_dma_address(req->dst);
42e95d1f 1113 out_options = 0;
f2147b88
HX		 1114 } else {
1115 dst_dma = edesc->sec4_sg_dma +
1116 sec4_sg_index *
1117 sizeof(struct sec4_sg_entry);
1118 out_options = LDST_SGF;
1119 }
1120 }
1121
1122 if (encrypt)
1123 append_seq_out_ptr(desc, dst_dma,
1124 req->assoclen + req->cryptlen + authsize,
1125 out_options);
1126 else
1127 append_seq_out_ptr(desc, dst_dma,
1128 req->assoclen + req->cryptlen - authsize,
1129 out_options);
f2147b88
HX		 1130}
1131
1132static void init_gcm_job(struct aead_request *req,
1133 struct aead_edesc *edesc,
1134 bool all_contig, bool encrypt)
1135{
1136 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1137 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
f2147b88
HX		 1138 unsigned int ivsize = crypto_aead_ivsize(aead);
1139 u32 *desc = edesc->hw_desc;
7545e166 1140 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
f2147b88
HX		 1141 unsigned int last;
1142
1143 init_aead_job(req, edesc, all_contig, encrypt);
7e0880b9 1144 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
f2147b88
HX		 1145
1146 /* BUG This should not be specific to generic GCM. */
1147 last = 0;
1148 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1149 last = FIFOLD_TYPE_LAST1;
1150
1151 /* Read GCM IV */
1152 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
7545e166 1153 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
f2147b88
HX		 1154 /* Append Salt */
1155 if (!generic_gcm)
db57656b 1156 append_data(desc, ctx->key + ctx->cdata.keylen, 4);
f2147b88
HX		 1157 /* Append IV */
1158 append_data(desc, req->iv, ivsize);
1159 /* End of blank commands */
1160}
1161
d6bbd4ee
HG		 1162static void init_chachapoly_job(struct aead_request *req,
1163 struct aead_edesc *edesc, bool all_contig,
1164 bool encrypt)
1165{
1166 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1167 unsigned int ivsize = crypto_aead_ivsize(aead);
1168 unsigned int assoclen = req->assoclen;
1169 u32 *desc = edesc->hw_desc;
1170 u32 ctx_iv_off = 4;
1171
1172 init_aead_job(req, edesc, all_contig, encrypt);
1173
1174 if (ivsize != CHACHAPOLY_IV_SIZE) {
1175 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1176 ctx_iv_off += 4;
1177
1178 /*
1179 * The associated data comes already with the IV but we need
1180 * to skip it when we authenticate or encrypt...
1181 */
1182 assoclen -= ivsize;
1183 }
1184
1185 append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1186
1187 /*
1188 * For IPsec load the IV further in the same register.
1189 * For RFC7539 simply load the 12 bytes nonce in a single operation
1190 */
1191 append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
1192 LDST_SRCDST_BYTE_CONTEXT |
1193 ctx_iv_off << LDST_OFFSET_SHIFT);
1194}
1195
479bcc7c
HX		 1196static void init_authenc_job(struct aead_request *req,
1197 struct aead_edesc *edesc,
1198 bool all_contig, bool encrypt)
1acebad3
YK		 1199{
1200 struct crypto_aead *aead = crypto_aead_reqtfm(req);
479bcc7c 1201 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
623814c0
HX		 1202 struct caam_aead_alg,
1203 aead.base);
479bcc7c 1204 unsigned int ivsize = crypto_aead_ivsize(aead);
4cb4f7c1 1205 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
7e0880b9 1206 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
db57656b 1207 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
479bcc7c
HX		 1208 OP_ALG_AAI_CTR_MOD128);
1209 const bool is_rfc3686 = alg->caam.rfc3686;
1acebad3 1210 u32 *desc = edesc->hw_desc;
479bcc7c 1211 u32 ivoffset = 0;
8e8ec596 1212
479bcc7c
HX		 1213 /*
1214 * AES-CTR needs to load IV in CONTEXT1 reg
1215 * at an offset of 128bits (16bytes)
1216 * CONTEXT1[255:128] = IV
1217 */
1218 if (ctr_mode)
1219 ivoffset = 16;
1acebad3 1220
479bcc7c
HX		 1221 /*
1222 * RFC3686 specific:
1223 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1224 */
1225 if (is_rfc3686)
1226 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
8e8ec596 1227
479bcc7c 1228 init_aead_job(req, edesc, all_contig, encrypt);
1acebad3 1229
7e0880b9
HG		 1230 /*
1231 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1232 * having DPOVRD as destination.
1233 */
1234 if (ctrlpriv->era < 3)
1235 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1236 else
1237 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1238
8b18e235 1239 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
479bcc7c
HX		 1240 append_load_as_imm(desc, req->iv, ivsize,
1241 LDST_CLASS_1_CCB |
1242 LDST_SRCDST_BYTE_CONTEXT |
1243 (ivoffset << LDST_OFFSET_SHIFT));
8e8ec596
KP		 1244}
1245
acdca31d 1246/*
5ca7badb 1247 * Fill in skcipher job descriptor
acdca31d 1248 */
5ca7badb
HG		 1249static void init_skcipher_job(struct skcipher_request *req,
1250 struct skcipher_edesc *edesc,
1251 const bool encrypt)
acdca31d 1252{
5ca7badb 1253 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
4cb4f7c1 1254 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
6e005503 1255 struct device *jrdev = ctx->jrdev;
5ca7badb 1256 int ivsize = crypto_skcipher_ivsize(skcipher);
acdca31d 1257 u32 *desc = edesc->hw_desc;
5ca7badb 1258 u32 *sh_desc;
eaed71a4
IP		 1259 u32 in_options = 0, out_options = 0;
1260 dma_addr_t src_dma, dst_dma, ptr;
1261 int len, sec4_sg_index = 0;
acdca31d 1262
6e005503
SH		 1263 print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
1264 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1265 dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
5ca7badb 1266 (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
6e005503 1267
8a82451b 1268 caam_dump_sg("src @" __stringify(__LINE__)": ",
972b812b 1269 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
5ca7badb
HG		 1270 edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1271
1272 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1273 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
acdca31d
YK		 1274
1275 len = desc_len(sh_desc);
1276 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1277
eaed71a4
IP		 1278 if (ivsize || edesc->mapped_src_nents > 1) {
1279 src_dma = edesc->sec4_sg_dma;
1280 sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
1281 in_options = LDST_SGF;
1282 } else {
1283 src_dma = sg_dma_address(req->src);
1284 }
1285
1286 append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
acdca31d
YK		 1287
1288 if (likely(req->src == req->dst)) {
eaed71a4
IP		 1289 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
1290 out_options = in_options;
334d37c9 1291 } else if (!ivsize && edesc->mapped_dst_nents == 1) {
eaed71a4 1292 dst_dma = sg_dma_address(req->dst);
acdca31d 1293 } else {
eaed71a4
IP		 1294 dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
1295 sizeof(struct sec4_sg_entry);
1296 out_options = LDST_SGF;
acdca31d 1297 }
eaed71a4 1298
334d37c9 1299 append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
acdca31d
YK		 1300}
1301
8e8ec596 1302/*
1acebad3 1303 * allocate and map the aead extended descriptor
8e8ec596 1304 */
479bcc7c
HX		 1305static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1306 int desc_bytes, bool *all_contig_ptr,
1307 bool encrypt)
8e8ec596 1308{
0e479300 1309 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1310 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
8e8ec596 1311 struct device *jrdev = ctx->jrdev;
1c240226 1312 struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
019d62db
HG		 1313 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1314 GFP_KERNEL : GFP_ATOMIC;
838e0a89 1315 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
059d73ee 1316 int src_len, dst_len = 0;
0e479300 1317 struct aead_edesc *edesc;
fa0c92db 1318 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
bbf9c893 1319 unsigned int authsize = ctx->authsize;
1acebad3 1320
bbf9c893 1321 if (unlikely(req->dst != req->src)) {
059d73ee
HG		 1322 src_len = req->assoclen + req->cryptlen;
1323 dst_len = src_len + (encrypt ? authsize : (-authsize));
1324
1325 src_nents = sg_nents_for_len(req->src, src_len);
fd144d83
HG		 1326 if (unlikely(src_nents < 0)) {
1327 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
059d73ee 1328 src_len);
fd144d83
HG		 1329 return ERR_PTR(src_nents);
1330 }
1331
059d73ee 1332 dst_nents = sg_nents_for_len(req->dst, dst_len);
fd144d83
HG		 1333 if (unlikely(dst_nents < 0)) {
1334 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
059d73ee 1335 dst_len);
fd144d83
HG		 1336 return ERR_PTR(dst_nents);
1337 }
bbf9c893 1338 } else {
059d73ee
HG		 1339 src_len = req->assoclen + req->cryptlen +
1340 (encrypt ? authsize : 0);
1341
1342 src_nents = sg_nents_for_len(req->src, src_len);
fd144d83
HG		 1343 if (unlikely(src_nents < 0)) {
1344 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
059d73ee 1345 src_len);
fd144d83
HG		 1346 return ERR_PTR(src_nents);
1347 }
f2147b88 1348 }
3ef8d945 1349
f2147b88 1350 if (likely(req->src == req->dst)) {
838e0a89
HG		 1351 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1352 DMA_BIDIRECTIONAL);
1353 if (unlikely(!mapped_src_nents)) {
f2147b88 1354 dev_err(jrdev, "unable to map source\n");
f2147b88
HX		 1355 return ERR_PTR(-ENOMEM);
1356 }
1357 } else {
fa0c92db
HG		 1358 /* Cover also the case of null (zero length) input data */
1359 if (src_nents) {
838e0a89
HG		 1360 mapped_src_nents = dma_map_sg(jrdev, req->src,
1361 src_nents, DMA_TO_DEVICE);
1362 if (unlikely(!mapped_src_nents)) {
fa0c92db 1363 dev_err(jrdev, "unable to map source\n");
fa0c92db
HG		 1364 return ERR_PTR(-ENOMEM);
1365 }
838e0a89
HG		 1366 } else {
1367 mapped_src_nents = 0;
f2147b88
HX		 1368 }
1369
763069ba
HG		 1370 /* Cover also the case of null (zero length) output data */
1371 if (dst_nents) {
1372 mapped_dst_nents = dma_map_sg(jrdev, req->dst,
1373 dst_nents,
1374 DMA_FROM_DEVICE);
1375 if (unlikely(!mapped_dst_nents)) {
1376 dev_err(jrdev, "unable to map destination\n");
1377 dma_unmap_sg(jrdev, req->src, src_nents,
1378 DMA_TO_DEVICE);
1379 return ERR_PTR(-ENOMEM);
1380 }
1381 } else {
1382 mapped_dst_nents = 0;
f2147b88
HX		 1383 }
1384 }
1385
a5e5c133
HG		 1386 /*
1387 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1388 * the end of the table by allocating more S/G entries.
1389 */
838e0a89 1390 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
a5e5c133
HG		 1391 if (mapped_dst_nents > 1)
1392 sec4_sg_len += pad_sg_nents(mapped_dst_nents);
1393 else
1394 sec4_sg_len = pad_sg_nents(sec4_sg_len);
1395
838e0a89
HG		 1396 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1397
1398 /* allocate space for base edesc and hw desc commands, link tables */
199354d7 1399 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags);
838e0a89
HG		 1400 if (!edesc) {
1401 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
cf5448b5 1402 0, 0, 0);
838e0a89
HG		 1403 return ERR_PTR(-ENOMEM);
1404 }
1405
8e8ec596
KP		 1406 edesc->src_nents = src_nents;
1407 edesc->dst_nents = dst_nents;
ba4cf71b
IP		 1408 edesc->mapped_src_nents = mapped_src_nents;
1409 edesc->mapped_dst_nents = mapped_dst_nents;
a299c837
YK		 1410 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1411 desc_bytes;
1c240226
IP		 1412
1413 rctx->edesc = edesc;
1414
838e0a89 1415 *all_contig_ptr = !(mapped_src_nents > 1);
1acebad3 1416
a299c837 1417 sec4_sg_index = 0;
838e0a89 1418 if (mapped_src_nents > 1) {
059d73ee 1419 sg_to_sec4_sg_last(req->src, src_len,
838e0a89
HG		 1420 edesc->sec4_sg + sec4_sg_index, 0);
1421 sec4_sg_index += mapped_src_nents;
1acebad3 1422 }
838e0a89 1423 if (mapped_dst_nents > 1) {
059d73ee 1424 sg_to_sec4_sg_last(req->dst, dst_len,
a299c837 1425 edesc->sec4_sg + sec4_sg_index, 0);
1acebad3 1426 }
f2147b88
HX		 1427
1428 if (!sec4_sg_bytes)
1429 return edesc;
1430
1da2be33
RG		 1431 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1432 sec4_sg_bytes, DMA_TO_DEVICE);
ce572085
HG		 1433 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1434 dev_err(jrdev, "unable to map S/G table\n");
f2147b88
HX		 1435 aead_unmap(jrdev, edesc, req);
1436 kfree(edesc);
ce572085
HG		 1437 return ERR_PTR(-ENOMEM);
1438 }
8e8ec596 1439
f2147b88
HX		 1440 edesc->sec4_sg_bytes = sec4_sg_bytes;
1441
8e8ec596
KP		 1442 return edesc;
1443}
1444
1c240226
IP		 1445static int aead_enqueue_req(struct device *jrdev, struct aead_request *req)
1446{
1447 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
1448 struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1449 struct aead_edesc *edesc = rctx->edesc;
1450 u32 *desc = edesc->hw_desc;
1451 int ret;
1452
1453 /*
1454 * Only the backlog request are sent to crypto-engine since the others
1455 * can be handled by CAAM, if free, especially since JR has up to 1024
1456 * entries (more than the 10 entries from crypto-engine).
1457 */
1458 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1459 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine,
1460 req);
1461 else
1462 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req);
1463
1464 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1465 aead_unmap(jrdev, edesc, req);
1466 kfree(rctx->edesc);
1467 }
1468
1469 return ret;
1470}
1471
b7f17fe2 1472static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
d6bbd4ee
HG		 1473{
1474 struct aead_edesc *edesc;
1475 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1476 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
d6bbd4ee
HG		 1477 struct device *jrdev = ctx->jrdev;
1478 bool all_contig;
1479 u32 *desc;
d6bbd4ee
HG		 1480
1481 edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
b7f17fe2 1482 encrypt);
d6bbd4ee
HG		 1483 if (IS_ERR(edesc))
1484 return PTR_ERR(edesc);
1485
1486 desc = edesc->hw_desc;
1487
b7f17fe2 1488 init_chachapoly_job(req, edesc, all_contig, encrypt);
d6bbd4ee
HG		 1489 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1490 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1491 1);
1492
1c240226 1493 return aead_enqueue_req(jrdev, req);
d6bbd4ee
HG		 1494}
1495
b7f17fe2 1496static int chachapoly_encrypt(struct aead_request *req)
d6bbd4ee 1497{
b7f17fe2 1498 return chachapoly_crypt(req, true);
d6bbd4ee
HG		 1499}
1500
b7f17fe2 1501static int chachapoly_decrypt(struct aead_request *req)
46218750 1502{
b7f17fe2 1503 return chachapoly_crypt(req, false);
46218750
HX		 1504}
1505
b7f17fe2 1506static inline int aead_crypt(struct aead_request *req, bool encrypt)
f2147b88
HX		 1507{
1508 struct aead_edesc *edesc;
1509 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1510 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
f2147b88
HX		 1511 struct device *jrdev = ctx->jrdev;
1512 bool all_contig;
f2147b88
HX		 1513
1514 /* allocate extended descriptor */
479bcc7c 1515 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
b7f17fe2 1516 &all_contig, encrypt);
f2147b88
HX		 1517 if (IS_ERR(edesc))
1518 return PTR_ERR(edesc);
1519
1520 /* Create and submit job descriptor */
b7f17fe2 1521 init_authenc_job(req, edesc, all_contig, encrypt);
6e005503
SH		 1522
1523 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1524 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1525 desc_bytes(edesc->hw_desc), 1);
f2147b88 1526
1c240226 1527 return aead_enqueue_req(jrdev, req);
f2147b88
HX		 1528}
1529
b7f17fe2
IP		 1530static int aead_encrypt(struct aead_request *req)
1531{
1532 return aead_crypt(req, true);
1533}
1534
1535static int aead_decrypt(struct aead_request *req)
1536{
1537 return aead_crypt(req, false);
1538}
1539
1c240226
IP		 1540static int aead_do_one_req(struct crypto_engine *engine, void *areq)
1541{
1542 struct aead_request *req = aead_request_cast(areq);
4cb4f7c1 1543 struct caam_ctx *ctx = crypto_aead_ctx_dma(crypto_aead_reqtfm(req));
1c240226
IP		 1544 struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1545 u32 *desc = rctx->edesc->hw_desc;
1546 int ret;
1547
1548 rctx->edesc->bklog = true;
1549
1550 ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
1551
087e1d71
GJ		 1552 if (ret == -ENOSPC && engine->retry_support)
1553 return ret;
1554
1c240226
IP		 1555 if (ret != -EINPROGRESS) {
1556 aead_unmap(ctx->jrdev, rctx->edesc, req);
1557 kfree(rctx->edesc);
1558 } else {
1559 ret = 0;
1560 }
1561
1562 return ret;
1563}
1564
b7f17fe2 1565static inline int gcm_crypt(struct aead_request *req, bool encrypt)
f2147b88
HX		 1566{
1567 struct aead_edesc *edesc;
1568 struct crypto_aead *aead = crypto_aead_reqtfm(req);
4cb4f7c1 1569 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
f2147b88
HX		 1570 struct device *jrdev = ctx->jrdev;
1571 bool all_contig;
f2147b88
HX		 1572
1573 /* allocate extended descriptor */
b7f17fe2
IP		 1574 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig,
1575 encrypt);
f2147b88
HX		 1576 if (IS_ERR(edesc))
1577 return PTR_ERR(edesc);
1578
b7f17fe2
IP		 1579 /* Create and submit job descriptor */
1580 init_gcm_job(req, edesc, all_contig, encrypt);
6e005503
SH		 1581
1582 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1583 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1584 desc_bytes(edesc->hw_desc), 1);
f2147b88 1585
1c240226 1586 return aead_enqueue_req(jrdev, req);
f2147b88
HX		 1587}
1588
b7f17fe2 1589static int gcm_encrypt(struct aead_request *req)
46218750 1590{
b7f17fe2 1591 return gcm_crypt(req, true);
46218750
HX		 1592}
1593
b7f17fe2 1594static int gcm_decrypt(struct aead_request *req)
8e8ec596 1595{
b7f17fe2
IP		 1596 return gcm_crypt(req, false);
1597}
1acebad3 1598
b7f17fe2
IP		 1599static int ipsec_gcm_encrypt(struct aead_request *req)
1600{
1601 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
1602}
8e8ec596 1603
b7f17fe2
IP		 1604static int ipsec_gcm_decrypt(struct aead_request *req)
1605{
1606 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
1acebad3 1607}
8e8ec596 1608
acdca31d 1609/*
5ca7badb 1610 * allocate and map the skcipher extended descriptor for skcipher
acdca31d 1611 */
5ca7badb
HG		 1612static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1613 int desc_bytes)
acdca31d 1614{
5ca7badb 1615 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
4cb4f7c1 1616 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
ee38767f 1617 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
acdca31d 1618 struct device *jrdev = ctx->jrdev;
42cfcafb 1619 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
acdca31d 1620 GFP_KERNEL : GFP_ATOMIC;
838e0a89 1621 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
5ca7badb 1622 struct skcipher_edesc *edesc;
eaed71a4 1623 dma_addr_t iv_dma = 0;
115957bb 1624 u8 *iv;
5ca7badb 1625 int ivsize = crypto_skcipher_ivsize(skcipher);
838e0a89 1626 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
199354d7 1627 unsigned int aligned_size;
acdca31d 1628
5ca7badb 1629 src_nents = sg_nents_for_len(req->src, req->cryptlen);
fd144d83
HG		 1630 if (unlikely(src_nents < 0)) {
1631 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
5ca7badb 1632 req->cryptlen);
fd144d83
HG		 1633 return ERR_PTR(src_nents);
1634 }
acdca31d 1635
fd144d83 1636 if (req->dst != req->src) {
5ca7badb 1637 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
fd144d83
HG		 1638 if (unlikely(dst_nents < 0)) {
1639 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
5ca7badb 1640 req->cryptlen);
fd144d83
HG		 1641 return ERR_PTR(dst_nents);
1642 }
1643 }
acdca31d
YK		 1644
1645 if (likely(req->src == req->dst)) {
838e0a89
HG		 1646 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1647 DMA_BIDIRECTIONAL);
1648 if (unlikely(!mapped_src_nents)) {
c73e36e8
HG		 1649 dev_err(jrdev, "unable to map source\n");
1650 return ERR_PTR(-ENOMEM);
1651 }
acdca31d 1652 } else {
838e0a89
HG		 1653 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1654 DMA_TO_DEVICE);
1655 if (unlikely(!mapped_src_nents)) {
c73e36e8
HG		 1656 dev_err(jrdev, "unable to map source\n");
1657 return ERR_PTR(-ENOMEM);
1658 }
838e0a89
HG		 1659 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1660 DMA_FROM_DEVICE);
1661 if (unlikely(!mapped_dst_nents)) {
c73e36e8 1662 dev_err(jrdev, "unable to map destination\n");
fa0c92db 1663 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
c73e36e8
HG		 1664 return ERR_PTR(-ENOMEM);
1665 }
acdca31d
YK		 1666 }
1667
eaed71a4
IP		 1668 if (!ivsize && mapped_src_nents == 1)
1669 sec4_sg_ents = 0; // no need for an input hw s/g table
1670 else
1671 sec4_sg_ents = mapped_src_nents + !!ivsize;
fa0c92db 1672 dst_sg_idx = sec4_sg_ents;
a5e5c133
HG		 1673
1674 /*
334d37c9
HG		 1675 * Input, output HW S/G tables: [IV, src][dst, IV]
1676 * IV entries point to the same buffer
1677 * If src == dst, S/G entries are reused (S/G tables overlap)
1678 *
a5e5c133
HG		 1679 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1680 * the end of the table by allocating more S/G entries. Logic:
334d37c9 1681 * if (output S/G)
a5e5c133 1682 * pad output S/G, if needed
a5e5c133
HG		 1683 * else if (input S/G) ...
1684 * pad input S/G, if needed
1685 */
334d37c9
HG		 1686 if (ivsize || mapped_dst_nents > 1) {
1687 if (req->src == req->dst)
1688 sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
1689 else
1690 sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
1691 !!ivsize);
1692 } else {
a5e5c133 1693 sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
334d37c9 1694 }
a5e5c133 1695
fa0c92db 1696 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
acdca31d 1697
115957bb
HG		 1698 /*
1699 * allocate space for base edesc and hw desc commands, link tables, IV
1700 */
660ca947 1701 aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
199354d7 1702 aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
660ca947
HX		 1703 aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
1704 (dma_get_cache_alignment() - 1);
1705 aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
1706 edesc = kzalloc(aligned_size, flags);
1707 if (!edesc) {
acdca31d 1708 dev_err(jrdev, "could not allocate extended descriptor\n");
115957bb 1709 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
cf5448b5 1710 0, 0, 0);
acdca31d
YK		 1711 return ERR_PTR(-ENOMEM);
1712 }
1713
1714 edesc->src_nents = src_nents;
1715 edesc->dst_nents = dst_nents;
ba4cf71b
IP		 1716 edesc->mapped_src_nents = mapped_src_nents;
1717 edesc->mapped_dst_nents = mapped_dst_nents;
a299c837 1718 edesc->sec4_sg_bytes = sec4_sg_bytes;
13cc6f48
HG		 1719 edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
1720 desc_bytes);
ee38767f 1721 rctx->edesc = edesc;
acdca31d 1722
115957bb 1723 /* Make sure IV is located in a DMAable area */
eaed71a4 1724 if (ivsize) {
660ca947 1725 iv = skcipher_edesc_iv(edesc);
eaed71a4
IP		 1726 memcpy(iv, req->iv, ivsize);
1727
334d37c9 1728 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
eaed71a4
IP		 1729 if (dma_mapping_error(jrdev, iv_dma)) {
1730 dev_err(jrdev, "unable to map IV\n");
1731 caam_unmap(jrdev, req->src, req->dst, src_nents,
1732 dst_nents, 0, 0, 0, 0);
1733 kfree(edesc);
1734 return ERR_PTR(-ENOMEM);
1735 }
115957bb 1736
eaed71a4 1737 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
acdca31d 1738 }
eaed71a4 1739 if (dst_sg_idx)
334d37c9
HG		 1740 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
1741 !!ivsize, 0);
115957bb 1742
334d37c9
HG		 1743 if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
1744 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
1745 dst_sg_idx, 0);
1746
1747 if (ivsize)
1748 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
1749 mapped_dst_nents, iv_dma, ivsize, 0);
1750
1751 if (ivsize || mapped_dst_nents > 1)
1752 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
55b3209a 1753 mapped_dst_nents - 1 + !!ivsize);
acdca31d 1754
eaed71a4
IP		 1755 if (sec4_sg_bytes) {
1756 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1757 sec4_sg_bytes,
1758 DMA_TO_DEVICE);
1759 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1760 dev_err(jrdev, "unable to map S/G table\n");
1761 caam_unmap(jrdev, req->src, req->dst, src_nents,
1762 dst_nents, iv_dma, ivsize, 0, 0);
1763 kfree(edesc);
1764 return ERR_PTR(-ENOMEM);
1765 }
ce572085
HG		 1766 }
1767
acdca31d
YK		 1768 edesc->iv_dma = iv_dma;
1769
6e005503
SH		 1770 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
1771 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1772 sec4_sg_bytes, 1);
acdca31d 1773
acdca31d
YK		 1774 return edesc;
1775}
1776
ee38767f
IP		 1777static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
1778{
1779 struct skcipher_request *req = skcipher_request_cast(areq);
4cb4f7c1 1780 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(crypto_skcipher_reqtfm(req));
ee38767f
IP		 1781 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1782 u32 *desc = rctx->edesc->hw_desc;
1783 int ret;
1784
1785 rctx->edesc->bklog = true;
1786
1787 ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
1788
087e1d71
GJ		 1789 if (ret == -ENOSPC && engine->retry_support)
1790 return ret;
1791
ee38767f
IP		 1792 if (ret != -EINPROGRESS) {
1793 skcipher_unmap(ctx->jrdev, rctx->edesc, req);
1794 kfree(rctx->edesc);
1795 } else {
1796 ret = 0;
1797 }
1798
1799 return ret;
1800}
1801
9d9b14db
AB		 1802static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1803{
1804 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1805 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1806
1807 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1808}
1809
b7f17fe2 1810static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
acdca31d 1811{
5ca7badb
HG		 1812 struct skcipher_edesc *edesc;
1813 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
4cb4f7c1 1814 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
acdca31d 1815 struct device *jrdev = ctx->jrdev;
ee38767f 1816 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
78eebbfa 1817 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
acdca31d
YK		 1818 u32 *desc;
1819 int ret = 0;
1820
297b931c
AB		 1821 /*
1822 * XTS is expected to return an error even for input length = 0
1823 * Note that the case input length < block size will be caught during
1824 * HW offloading and return an error.
1825 */
1826 if (!req->cryptlen && !ctx->fallback)
31bb2f0d
IP		 1827 return 0;
1828
78eebbfa 1829 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
c91f7348 1830 ctx->xts_key_fallback)) {
9d9b14db
AB		 1831 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1832
1833 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
1834 skcipher_request_set_callback(&rctx->fallback_req,
1835 req->base.flags,
1836 req->base.complete,
1837 req->base.data);
1838 skcipher_request_set_crypt(&rctx->fallback_req, req->src,
1839 req->dst, req->cryptlen, req->iv);
1840
1841 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
1842 crypto_skcipher_decrypt(&rctx->fallback_req);
1843 }
1844
acdca31d 1845 /* allocate extended descriptor */
5ca7badb 1846 edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
acdca31d
YK		 1847 if (IS_ERR(edesc))
1848 return PTR_ERR(edesc);
1849
1850 /* Create and submit job descriptor*/
b7f17fe2 1851 init_skcipher_job(req, edesc, encrypt);
6e005503
SH		 1852
1853 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
1854 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1855 desc_bytes(edesc->hw_desc), 1);
1856
acdca31d 1857 desc = edesc->hw_desc;
ee38767f
IP		 1858 /*
1859 * Only the backlog request are sent to crypto-engine since the others
1860 * can be handled by CAAM, if free, especially since JR has up to 1024
1861 * entries (more than the 10 entries from crypto-engine).
1862 */
1863 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1864 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine,
1865 req);
1866 else
1867 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req);
acdca31d 1868
ee38767f 1869 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
5ca7badb 1870 skcipher_unmap(jrdev, edesc, req);
acdca31d
YK		 1871 kfree(edesc);
1872 }
1873
1874 return ret;
1875}
1876
b7f17fe2 1877static int skcipher_encrypt(struct skcipher_request *req)
acdca31d 1878{
b7f17fe2
IP		 1879 return skcipher_crypt(req, true);
1880}
acdca31d 1881
b7f17fe2
IP		 1882static int skcipher_decrypt(struct skcipher_request *req)
1883{
1884 return skcipher_crypt(req, false);
acdca31d
YK		 1885}
1886
5ca7badb 1887static struct caam_skcipher_alg driver_algs[] = {
ae4a825f 1888 {
623814c0 1889 .skcipher.base = {
5ca7badb
HG		 1890 .base = {
1891 .cra_name = "cbc(aes)",
1892 .cra_driver_name = "cbc-aes-caam",
1893 .cra_blocksize = AES_BLOCK_SIZE,
1894 },
836d8f43 1895 .setkey = aes_skcipher_setkey,
5ca7badb
HG		 1896 .encrypt = skcipher_encrypt,
1897 .decrypt = skcipher_decrypt,
479bcc7c
HX		 1898 .min_keysize = AES_MIN_KEY_SIZE,
1899 .max_keysize = AES_MAX_KEY_SIZE,
1900 .ivsize = AES_BLOCK_SIZE,
5ca7badb 1901 },
623814c0
HX		 1902 .skcipher.op = {
1903 .do_one_request = skcipher_do_one_req,
1904 },
5ca7badb 1905 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
479bcc7c
HX		 1906 },
1907 {
623814c0 1908 .skcipher.base = {
5ca7badb
HG		 1909 .base = {
1910 .cra_name = "cbc(des3_ede)",
1911 .cra_driver_name = "cbc-3des-caam",
1912 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1913 },
a628c5a1 1914 .setkey = des3_skcipher_setkey,
5ca7badb
HG		 1915 .encrypt = skcipher_encrypt,
1916 .decrypt = skcipher_decrypt,
479bcc7c
HX		 1917 .min_keysize = DES3_EDE_KEY_SIZE,
1918 .max_keysize = DES3_EDE_KEY_SIZE,
1919 .ivsize = DES3_EDE_BLOCK_SIZE,
5ca7badb 1920 },
623814c0
HX		 1921 .skcipher.op = {
1922 .do_one_request = skcipher_do_one_req,
1923 },
5ca7badb 1924 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
479bcc7c
HX		 1925 },
1926 {
623814c0 1927 .skcipher.base = {
5ca7badb
HG		 1928 .base = {
1929 .cra_name = "cbc(des)",
1930 .cra_driver_name = "cbc-des-caam",
1931 .cra_blocksize = DES_BLOCK_SIZE,
1932 },
cf64e495 1933 .setkey = des_skcipher_setkey,
5ca7badb
HG		 1934 .encrypt = skcipher_encrypt,
1935 .decrypt = skcipher_decrypt,
479bcc7c
HX		 1936 .min_keysize = DES_KEY_SIZE,
1937 .max_keysize = DES_KEY_SIZE,
1938 .ivsize = DES_BLOCK_SIZE,
5ca7badb 1939 },
623814c0
HX		 1940 .skcipher.op = {
1941 .do_one_request = skcipher_do_one_req,
1942 },
5ca7badb 1943 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
479bcc7c
HX		 1944 },
1945 {
623814c0 1946 .skcipher.base = {
5ca7badb
HG		 1947 .base = {
1948 .cra_name = "ctr(aes)",
1949 .cra_driver_name = "ctr-aes-caam",
1950 .cra_blocksize = 1,
1951 },
836d8f43 1952 .setkey = ctr_skcipher_setkey,
5ca7badb
HG		 1953 .encrypt = skcipher_encrypt,
1954 .decrypt = skcipher_decrypt,
479bcc7c
HX		 1955 .min_keysize = AES_MIN_KEY_SIZE,
1956 .max_keysize = AES_MAX_KEY_SIZE,
1957 .ivsize = AES_BLOCK_SIZE,
5ca7badb
HG		 1958 .chunksize = AES_BLOCK_SIZE,
1959 },
623814c0
HX		 1960 .skcipher.op = {
1961 .do_one_request = skcipher_do_one_req,
1962 },
5ca7badb
HG		 1963 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1964 OP_ALG_AAI_CTR_MOD128,
479bcc7c
HX		 1965 },
1966 {
623814c0 1967 .skcipher.base = {
5ca7badb
HG		 1968 .base = {
1969 .cra_name = "rfc3686(ctr(aes))",
1970 .cra_driver_name = "rfc3686-ctr-aes-caam",
1971 .cra_blocksize = 1,
1972 },
836d8f43 1973 .setkey = rfc3686_skcipher_setkey,
5ca7badb
HG		 1974 .encrypt = skcipher_encrypt,
1975 .decrypt = skcipher_decrypt,
479bcc7c
HX		 1976 .min_keysize = AES_MIN_KEY_SIZE +
1977 CTR_RFC3686_NONCE_SIZE,
1978 .max_keysize = AES_MAX_KEY_SIZE +
1979 CTR_RFC3686_NONCE_SIZE,
1980 .ivsize = CTR_RFC3686_IV_SIZE,
5ca7badb
HG		 1981 .chunksize = AES_BLOCK_SIZE,
1982 },
623814c0
HX		 1983 .skcipher.op = {
1984 .do_one_request = skcipher_do_one_req,
1985 },
5ca7badb
HG		 1986 .caam = {
1987 .class1_alg_type = OP_ALG_ALGSEL_AES |
1988 OP_ALG_AAI_CTR_MOD128,
1989 .rfc3686 = true,
1990 },
c6415a60
CV		 1991 },
1992 {
623814c0 1993 .skcipher.base = {
5ca7badb
HG		 1994 .base = {
1995 .cra_name = "xts(aes)",
1996 .cra_driver_name = "xts-aes-caam",
9d9b14db 1997 .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
5ca7badb
HG		 1998 .cra_blocksize = AES_BLOCK_SIZE,
1999 },
2000 .setkey = xts_skcipher_setkey,
2001 .encrypt = skcipher_encrypt,
2002 .decrypt = skcipher_decrypt,
c6415a60
CV		 2003 .min_keysize = 2 * AES_MIN_KEY_SIZE,
2004 .max_keysize = 2 * AES_MAX_KEY_SIZE,
2005 .ivsize = AES_BLOCK_SIZE,
5ca7badb 2006 },
623814c0
HX		 2007 .skcipher.op = {
2008 .do_one_request = skcipher_do_one_req,
2009 },
5ca7badb 2010 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
c6415a60 2011 },
eaed71a4 2012 {
623814c0 2013 .skcipher.base = {
eaed71a4
IP		 2014 .base = {
2015 .cra_name = "ecb(des)",
2016 .cra_driver_name = "ecb-des-caam",
2017 .cra_blocksize = DES_BLOCK_SIZE,
2018 },
2019 .setkey = des_skcipher_setkey,
2020 .encrypt = skcipher_encrypt,
2021 .decrypt = skcipher_decrypt,
2022 .min_keysize = DES_KEY_SIZE,
2023 .max_keysize = DES_KEY_SIZE,
2024 },
623814c0
HX		 2025 .skcipher.op = {
2026 .do_one_request = skcipher_do_one_req,
2027 },
eaed71a4
IP		 2028 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
2029 },
2030 {
623814c0 2031 .skcipher.base = {
eaed71a4
IP		 2032 .base = {
2033 .cra_name = "ecb(aes)",
2034 .cra_driver_name = "ecb-aes-caam",
2035 .cra_blocksize = AES_BLOCK_SIZE,
2036 },
836d8f43 2037 .setkey = aes_skcipher_setkey,
eaed71a4
IP		 2038 .encrypt = skcipher_encrypt,
2039 .decrypt = skcipher_decrypt,
2040 .min_keysize = AES_MIN_KEY_SIZE,
2041 .max_keysize = AES_MAX_KEY_SIZE,
2042 },
623814c0
HX		 2043 .skcipher.op = {
2044 .do_one_request = skcipher_do_one_req,
2045 },
eaed71a4
IP		 2046 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
2047 },
2048 {
623814c0 2049 .skcipher.base = {
eaed71a4
IP		 2050 .base = {
2051 .cra_name = "ecb(des3_ede)",
2052 .cra_driver_name = "ecb-des3-caam",
2053 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2054 },
a628c5a1 2055 .setkey = des3_skcipher_setkey,
eaed71a4
IP		 2056 .encrypt = skcipher_encrypt,
2057 .decrypt = skcipher_decrypt,
2058 .min_keysize = DES3_EDE_KEY_SIZE,
2059 .max_keysize = DES3_EDE_KEY_SIZE,
2060 },
623814c0
HX		 2061 .skcipher.op = {
2062 .do_one_request = skcipher_do_one_req,
2063 },
eaed71a4
IP		 2064 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
2065 },
479bcc7c
HX		 2066};
2067
2068static struct caam_aead_alg driver_aeads[] = {
2069 {
623814c0 2070 .aead.base = {
479bcc7c
HX		 2071 .base = {
2072 .cra_name = "rfc4106(gcm(aes))",
2073 .cra_driver_name = "rfc4106-gcm-aes-caam",
2074 .cra_blocksize = 1,
2075 },
2076 .setkey = rfc4106_setkey,
2077 .setauthsize = rfc4106_setauthsize,
2078 .encrypt = ipsec_gcm_encrypt,
2079 .decrypt = ipsec_gcm_decrypt,
7545e166 2080 .ivsize = GCM_RFC4106_IV_SIZE,
479bcc7c
HX		 2081 .maxauthsize = AES_BLOCK_SIZE,
2082 },
623814c0
HX		 2083 .aead.op = {
2084 .do_one_request = aead_do_one_req,
2085 },
479bcc7c
HX		 2086 .caam = {
2087 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
24586b5f 2088 .nodkp = true,
479bcc7c
HX		 2089 },
2090 },
2091 {
623814c0 2092 .aead.base = {
479bcc7c
HX		 2093 .base = {
2094 .cra_name = "rfc4543(gcm(aes))",
2095 .cra_driver_name = "rfc4543-gcm-aes-caam",
2096 .cra_blocksize = 1,
2097 },
2098 .setkey = rfc4543_setkey,
2099 .setauthsize = rfc4543_setauthsize,
2100 .encrypt = ipsec_gcm_encrypt,
2101 .decrypt = ipsec_gcm_decrypt,
7545e166 2102 .ivsize = GCM_RFC4543_IV_SIZE,
479bcc7c
HX		 2103 .maxauthsize = AES_BLOCK_SIZE,
2104 },
623814c0
HX		 2105 .aead.op = {
2106 .do_one_request = aead_do_one_req,
2107 },
479bcc7c
HX		 2108 .caam = {
2109 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
24586b5f 2110 .nodkp = true,
479bcc7c
HX		 2111 },
2112 },
2113 /* Galois Counter Mode */
2114 {
623814c0 2115 .aead.base = {
479bcc7c
HX		 2116 .base = {
2117 .cra_name = "gcm(aes)",
2118 .cra_driver_name = "gcm-aes-caam",
2119 .cra_blocksize = 1,
2120 },
2121 .setkey = gcm_setkey,
2122 .setauthsize = gcm_setauthsize,
2123 .encrypt = gcm_encrypt,
2124 .decrypt = gcm_decrypt,
7545e166 2125 .ivsize = GCM_AES_IV_SIZE,
479bcc7c
HX		 2126 .maxauthsize = AES_BLOCK_SIZE,
2127 },
623814c0
HX		 2128 .aead.op = {
2129 .do_one_request = aead_do_one_req,
2130 },
479bcc7c
HX		 2131 .caam = {
2132 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
24586b5f 2133 .nodkp = true,
479bcc7c
HX		 2134 },
2135 },
2136 /* single-pass ipsec_esp descriptor */
2137 {
623814c0 2138 .aead.base = {
479bcc7c
HX		 2139 .base = {
2140 .cra_name = "authenc(hmac(md5),"
2141 "ecb(cipher_null))",
2142 .cra_driver_name = "authenc-hmac-md5-"
2143 "ecb-cipher_null-caam",
2144 .cra_blocksize = NULL_BLOCK_SIZE,
2145 },
2146 .setkey = aead_setkey,
2147 .setauthsize = aead_setauthsize,
2148 .encrypt = aead_encrypt,
2149 .decrypt = aead_decrypt,
ae4a825f 2150 .ivsize = NULL_IV_SIZE,
479bcc7c
HX		 2151 .maxauthsize = MD5_DIGEST_SIZE,
2152 },
623814c0
HX		 2153 .aead.op = {
2154 .do_one_request = aead_do_one_req,
2155 },
479bcc7c
HX		 2156 .caam = {
2157 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2158 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2159 },
2160 },
2161 {
623814c0 2162 .aead.base = {
479bcc7c
HX		 2163 .base = {
2164 .cra_name = "authenc(hmac(sha1),"
2165 "ecb(cipher_null))",
2166 .cra_driver_name = "authenc-hmac-sha1-"
2167 "ecb-cipher_null-caam",
2168 .cra_blocksize = NULL_BLOCK_SIZE,
ae4a825f 2169 },
479bcc7c
HX		 2170 .setkey = aead_setkey,
2171 .setauthsize = aead_setauthsize,
2172 .encrypt = aead_encrypt,
2173 .decrypt = aead_decrypt,
2174 .ivsize = NULL_IV_SIZE,
2175 .maxauthsize = SHA1_DIGEST_SIZE,
2176 },
623814c0
HX		 2177 .aead.op = {
2178 .do_one_request = aead_do_one_req,
2179 },
479bcc7c
HX		 2180 .caam = {
2181 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2182 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2183 },
ae4a825f
HG		 2184 },
2185 {
623814c0 2186 .aead.base = {
479bcc7c
HX		 2187 .base = {
2188 .cra_name = "authenc(hmac(sha224),"
2189 "ecb(cipher_null))",
2190 .cra_driver_name = "authenc-hmac-sha224-"
2191 "ecb-cipher_null-caam",
2192 .cra_blocksize = NULL_BLOCK_SIZE,
2193 },
ae4a825f
HG		 2194 .setkey = aead_setkey,
2195 .setauthsize = aead_setauthsize,
479bcc7c
HX		 2196 .encrypt = aead_encrypt,
2197 .decrypt = aead_decrypt,
ae4a825f
HG		 2198 .ivsize = NULL_IV_SIZE,
2199 .maxauthsize = SHA224_DIGEST_SIZE,
479bcc7c 2200 },
623814c0
HX		 2201 .aead.op = {
2202 .do_one_request = aead_do_one_req,
2203 },
479bcc7c
HX		 2204 .caam = {
2205 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2206 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2207 },
ae4a825f
HG		 2208 },
2209 {
623814c0 2210 .aead.base = {
479bcc7c
HX		 2211 .base = {
2212 .cra_name = "authenc(hmac(sha256),"
2213 "ecb(cipher_null))",
2214 .cra_driver_name = "authenc-hmac-sha256-"
2215 "ecb-cipher_null-caam",
2216 .cra_blocksize = NULL_BLOCK_SIZE,
2217 },
ae4a825f
HG		 2218 .setkey = aead_setkey,
2219 .setauthsize = aead_setauthsize,
479bcc7c
HX		 2220 .encrypt = aead_encrypt,
2221 .decrypt = aead_decrypt,
ae4a825f
HG		 2222 .ivsize = NULL_IV_SIZE,
2223 .maxauthsize = SHA256_DIGEST_SIZE,
479bcc7c 2224 },
623814c0
HX		 2225 .aead.op = {
2226 .do_one_request = aead_do_one_req,
2227 },
479bcc7c
HX		 2228 .caam = {
2229 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2230 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2231 },
ae4a825f
HG		 2232 },
2233 {
623814c0 2234 .aead.base = {
479bcc7c
HX		 2235 .base = {
2236 .cra_name = "authenc(hmac(sha384),"
2237 "ecb(cipher_null))",
2238 .cra_driver_name = "authenc-hmac-sha384-"
2239 "ecb-cipher_null-caam",
2240 .cra_blocksize = NULL_BLOCK_SIZE,
2241 },
ae4a825f
HG		 2242 .setkey = aead_setkey,
2243 .setauthsize = aead_setauthsize,
479bcc7c
HX		 2244 .encrypt = aead_encrypt,
2245 .decrypt = aead_decrypt,
ae4a825f
HG		 2246 .ivsize = NULL_IV_SIZE,
2247 .maxauthsize = SHA384_DIGEST_SIZE,
479bcc7c 2248 },
623814c0
HX		 2249 .aead.op = {
2250 .do_one_request = aead_do_one_req,
2251 },
479bcc7c
HX		 2252 .caam = {
2253 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2254 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2255 },
ae4a825f
HG		 2256 },
2257 {
623814c0 2258 .aead.base = {
479bcc7c
HX		 2259 .base = {
2260 .cra_name = "authenc(hmac(sha512),"
2261 "ecb(cipher_null))",
2262 .cra_driver_name = "authenc-hmac-sha512-"
2263 "ecb-cipher_null-caam",
2264 .cra_blocksize = NULL_BLOCK_SIZE,
2265 },
ae4a825f
HG		 2266 .setkey = aead_setkey,
2267 .setauthsize = aead_setauthsize,
479bcc7c
HX		 2268 .encrypt = aead_encrypt,
2269 .decrypt = aead_decrypt,
ae4a825f
HG		 2270 .ivsize = NULL_IV_SIZE,
2271 .maxauthsize = SHA512_DIGEST_SIZE,
479bcc7c 2272 },
623814c0
HX		 2273 .aead.op = {
2274 .do_one_request = aead_do_one_req,
2275 },
479bcc7c
HX		 2276 .caam = {
2277 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2278 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2279 },
2280 },
2281 {
623814c0 2282 .aead.base = {
479bcc7c
HX		 2283 .base = {
2284 .cra_name = "authenc(hmac(md5),cbc(aes))",
2285 .cra_driver_name = "authenc-hmac-md5-"
2286 "cbc-aes-caam",
2287 .cra_blocksize = AES_BLOCK_SIZE,
ae4a825f 2288 },
479bcc7c
HX		 2289 .setkey = aead_setkey,
2290 .setauthsize = aead_setauthsize,
2291 .encrypt = aead_encrypt,
2292 .decrypt = aead_decrypt,
2293 .ivsize = AES_BLOCK_SIZE,
2294 .maxauthsize = MD5_DIGEST_SIZE,
2295 },
623814c0
HX		 2296 .aead.op = {
2297 .do_one_request = aead_do_one_req,
2298 },
479bcc7c
HX		 2299 .caam = {
2300 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2301 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2302 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2303 },
ae4a825f 2304 },
8b4d43a4 2305 {
623814c0 2306 .aead.base = {
479bcc7c
HX		 2307 .base = {
2308 .cra_name = "echainiv(authenc(hmac(md5),"
2309 "cbc(aes)))",
2310 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2311 "cbc-aes-caam",
2312 .cra_blocksize = AES_BLOCK_SIZE,
2313 },
8b4d43a4
KP		 2314 .setkey = aead_setkey,
2315 .setauthsize = aead_setauthsize,
479bcc7c 2316 .encrypt = aead_encrypt,
8b18e235 2317 .decrypt = aead_decrypt,
8b4d43a4
KP		 2318 .ivsize = AES_BLOCK_SIZE,
2319 .maxauthsize = MD5_DIGEST_SIZE,
479bcc7c 2320 },
623814c0
HX		 2321 .aead.op = {
2322 .do_one_request = aead_do_one_req,
2323 },
479bcc7c
HX		 2324 .caam = {
2325 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2326 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2327 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2328 .geniv = true,
2329 },
2330 },
2331 {
623814c0 2332 .aead.base = {
479bcc7c
HX		 2333 .base = {
2334 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2335 .cra_driver_name = "authenc-hmac-sha1-"
2336 "cbc-aes-caam",
2337 .cra_blocksize = AES_BLOCK_SIZE,
8b4d43a4 2338 },
479bcc7c
HX		 2339 .setkey = aead_setkey,
2340 .setauthsize = aead_setauthsize,
2341 .encrypt = aead_encrypt,
2342 .decrypt = aead_decrypt,
2343 .ivsize = AES_BLOCK_SIZE,
2344 .maxauthsize = SHA1_DIGEST_SIZE,
2345 },
623814c0
HX		 2346 .aead.op = {
2347 .do_one_request = aead_do_one_req,
2348 },
479bcc7c
HX		 2349 .caam = {
2350 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2351 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2352 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2353 },
8b4d43a4 2354 },
8e8ec596 2355 {
623814c0 2356 .aead.base = {
479bcc7c
HX		 2357 .base = {
2358 .cra_name = "echainiv(authenc(hmac(sha1),"
2359 "cbc(aes)))",
2360 .cra_driver_name = "echainiv-authenc-"
2361 "hmac-sha1-cbc-aes-caam",
2362 .cra_blocksize = AES_BLOCK_SIZE,
2363 },
0e479300
YK		 2364 .setkey = aead_setkey,
2365 .setauthsize = aead_setauthsize,
479bcc7c 2366 .encrypt = aead_encrypt,
8b18e235 2367 .decrypt = aead_decrypt,
8e8ec596
KP		 2368 .ivsize = AES_BLOCK_SIZE,
2369 .maxauthsize = SHA1_DIGEST_SIZE,
479bcc7c 2370 },
623814c0
HX		 2371 .aead.op = {
2372 .do_one_request = aead_do_one_req,
2373 },
479bcc7c
HX		 2374 .caam = {
2375 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2376 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2377 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2378 .geniv = true,
2379 },
2380 },
2381 {
623814c0 2382 .aead.base = {
479bcc7c
HX		 2383 .base = {
2384 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2385 .cra_driver_name = "authenc-hmac-sha224-"
2386 "cbc-aes-caam",
2387 .cra_blocksize = AES_BLOCK_SIZE,
8e8ec596 2388 },
479bcc7c
HX		 2389 .setkey = aead_setkey,
2390 .setauthsize = aead_setauthsize,
2391 .encrypt = aead_encrypt,
2392 .decrypt = aead_decrypt,
2393 .ivsize = AES_BLOCK_SIZE,
2394 .maxauthsize = SHA224_DIGEST_SIZE,
2395 },
623814c0
HX		 2396 .aead.op = {
2397 .do_one_request = aead_do_one_req,
2398 },
479bcc7c
HX		 2399 .caam = {
2400 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2401 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2402 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2403 },
8e8ec596 2404 },
e863f9cc 2405 {
623814c0 2406 .aead.base = {
479bcc7c
HX		 2407 .base = {
2408 .cra_name = "echainiv(authenc(hmac(sha224),"
2409 "cbc(aes)))",
2410 .cra_driver_name = "echainiv-authenc-"
2411 "hmac-sha224-cbc-aes-caam",
2412 .cra_blocksize = AES_BLOCK_SIZE,
2413 },
e863f9cc
HA		 2414 .setkey = aead_setkey,
2415 .setauthsize = aead_setauthsize,
479bcc7c 2416 .encrypt = aead_encrypt,
8b18e235 2417 .decrypt = aead_decrypt,
e863f9cc
HA		 2418 .ivsize = AES_BLOCK_SIZE,
2419 .maxauthsize = SHA224_DIGEST_SIZE,
479bcc7c 2420 },
623814c0
HX		 2421 .aead.op = {
2422 .do_one_request = aead_do_one_req,
2423 },
479bcc7c
HX		 2424 .caam = {
2425 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2426 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2427 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2428 .geniv = true,
2429 },
2430 },
2431 {
623814c0 2432 .aead.base = {
479bcc7c
HX		 2433 .base = {
2434 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2435 .cra_driver_name = "authenc-hmac-sha256-"
2436 "cbc-aes-caam",
2437 .cra_blocksize = AES_BLOCK_SIZE,
e863f9cc 2438 },
479bcc7c
HX		 2439 .setkey = aead_setkey,
2440 .setauthsize = aead_setauthsize,
2441 .encrypt = aead_encrypt,
2442 .decrypt = aead_decrypt,
2443 .ivsize = AES_BLOCK_SIZE,
2444 .maxauthsize = SHA256_DIGEST_SIZE,
2445 },
623814c0
HX		 2446 .aead.op = {
2447 .do_one_request = aead_do_one_req,
2448 },
479bcc7c
HX		 2449 .caam = {
2450 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2451 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2452 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2453 },
e863f9cc 2454 },
8e8ec596 2455 {
623814c0 2456 .aead.base = {
479bcc7c
HX		 2457 .base = {
2458 .cra_name = "echainiv(authenc(hmac(sha256),"
2459 "cbc(aes)))",
2460 .cra_driver_name = "echainiv-authenc-"
2461 "hmac-sha256-cbc-aes-caam",
2462 .cra_blocksize = AES_BLOCK_SIZE,
2463 },
2464 .setkey = aead_setkey,
2465 .setauthsize = aead_setauthsize,
2466 .encrypt = aead_encrypt,
8b18e235 2467 .decrypt = aead_decrypt,
479bcc7c
HX		 2468 .ivsize = AES_BLOCK_SIZE,
2469 .maxauthsize = SHA256_DIGEST_SIZE,
2470 },
623814c0
HX		 2471 .aead.op = {
2472 .do_one_request = aead_do_one_req,
2473 },
479bcc7c
HX		 2474 .caam = {
2475 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2476 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2477 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2478 .geniv = true,
2479 },
2480 },
2481 {
623814c0 2482 .aead.base = {
479bcc7c
HX		 2483 .base = {
2484 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2485 .cra_driver_name = "authenc-hmac-sha384-"
2486 "cbc-aes-caam",
2487 .cra_blocksize = AES_BLOCK_SIZE,
2488 },
2489 .setkey = aead_setkey,
2490 .setauthsize = aead_setauthsize,
2491 .encrypt = aead_encrypt,
2492 .decrypt = aead_decrypt,
2493 .ivsize = AES_BLOCK_SIZE,
2494 .maxauthsize = SHA384_DIGEST_SIZE,
2495 },
623814c0
HX		 2496 .aead.op = {
2497 .do_one_request = aead_do_one_req,
2498 },
479bcc7c
HX		 2499 .caam = {
2500 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2501 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2502 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2503 },
2504 },
2505 {
623814c0 2506 .aead.base = {
479bcc7c
HX		 2507 .base = {
2508 .cra_name = "echainiv(authenc(hmac(sha384),"
2509 "cbc(aes)))",
2510 .cra_driver_name = "echainiv-authenc-"
2511 "hmac-sha384-cbc-aes-caam",
2512 .cra_blocksize = AES_BLOCK_SIZE,
2513 },
0e479300
YK		 2514 .setkey = aead_setkey,
2515 .setauthsize = aead_setauthsize,
479bcc7c 2516 .encrypt = aead_encrypt,
8b18e235 2517 .decrypt = aead_decrypt,
8e8ec596 2518 .ivsize = AES_BLOCK_SIZE,
479bcc7c
HX		 2519 .maxauthsize = SHA384_DIGEST_SIZE,
2520 },
623814c0
HX		 2521 .aead.op = {
2522 .do_one_request = aead_do_one_req,
2523 },
479bcc7c
HX		 2524 .caam = {
2525 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2526 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2527 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2528 .geniv = true,
2529 },
8e8ec596 2530 },
e863f9cc 2531 {
623814c0 2532 .aead.base = {
479bcc7c
HX		 2533 .base = {
2534 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2535 .cra_driver_name = "authenc-hmac-sha512-"
2536 "cbc-aes-caam",
2537 .cra_blocksize = AES_BLOCK_SIZE,
2538 },
e863f9cc
HA		 2539 .setkey = aead_setkey,
2540 .setauthsize = aead_setauthsize,
479bcc7c
HX		 2541 .encrypt = aead_encrypt,
2542 .decrypt = aead_decrypt,
e863f9cc 2543 .ivsize = AES_BLOCK_SIZE,
479bcc7c
HX		 2544 .maxauthsize = SHA512_DIGEST_SIZE,
2545 },
623814c0
HX		 2546 .aead.op = {
2547 .do_one_request = aead_do_one_req,
2548 },
479bcc7c
HX		 2549 .caam = {
2550 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2551 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2552 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2553 },
e863f9cc 2554 },
4427b1b4 2555 {
623814c0 2556 .aead.base = {
479bcc7c
HX		 2557 .base = {
2558 .cra_name = "echainiv(authenc(hmac(sha512),"
2559 "cbc(aes)))",
2560 .cra_driver_name = "echainiv-authenc-"
2561 "hmac-sha512-cbc-aes-caam",
2562 .cra_blocksize = AES_BLOCK_SIZE,
2563 },
0e479300
YK		 2564 .setkey = aead_setkey,
2565 .setauthsize = aead_setauthsize,
479bcc7c 2566 .encrypt = aead_encrypt,
8b18e235 2567 .decrypt = aead_decrypt,
4427b1b4
KP		 2568 .ivsize = AES_BLOCK_SIZE,
2569 .maxauthsize = SHA512_DIGEST_SIZE,
479bcc7c 2570 },
623814c0
HX		 2571 .aead.op = {
2572 .do_one_request = aead_do_one_req,
2573 },
479bcc7c
HX		 2574 .caam = {
2575 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2576 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2577 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2578 .geniv = true,
2579 },
2580 },
2581 {
623814c0 2582 .aead.base = {
479bcc7c
HX		 2583 .base = {
2584 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2585 .cra_driver_name = "authenc-hmac-md5-"
2586 "cbc-des3_ede-caam",
2587 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
4427b1b4 2588 },
1b52c409 2589 .setkey = des3_aead_setkey,
479bcc7c
HX		 2590 .setauthsize = aead_setauthsize,
2591 .encrypt = aead_encrypt,
2592 .decrypt = aead_decrypt,
2593 .ivsize = DES3_EDE_BLOCK_SIZE,
2594 .maxauthsize = MD5_DIGEST_SIZE,
2595 },
623814c0
HX		 2596 .aead.op = {
2597 .do_one_request = aead_do_one_req,
2598 },
479bcc7c
HX		 2599 .caam = {
2600 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2601 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2602 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2603 }
4427b1b4 2604 },
8b4d43a4 2605 {
623814c0 2606 .aead.base = {
479bcc7c
HX		 2607 .base = {
2608 .cra_name = "echainiv(authenc(hmac(md5),"
2609 "cbc(des3_ede)))",
2610 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2611 "cbc-des3_ede-caam",
2612 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2613 },
1b52c409 2614 .setkey = des3_aead_setkey,
8b4d43a4 2615 .setauthsize = aead_setauthsize,
479bcc7c 2616 .encrypt = aead_encrypt,
8b18e235 2617 .decrypt = aead_decrypt,
8b4d43a4
KP		 2618 .ivsize = DES3_EDE_BLOCK_SIZE,
2619 .maxauthsize = MD5_DIGEST_SIZE,
479bcc7c 2620 },
623814c0
HX		 2621 .aead.op = {
2622 .do_one_request = aead_do_one_req,
2623 },
479bcc7c
HX		 2624 .caam = {
2625 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2626 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2627 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2628 .geniv = true,
2629 }
2630 },
2631 {
623814c0 2632 .aead.base = {
479bcc7c
HX		 2633 .base = {
2634 .cra_name = "authenc(hmac(sha1),"
2635 "cbc(des3_ede))",
2636 .cra_driver_name = "authenc-hmac-sha1-"
2637 "cbc-des3_ede-caam",
2638 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
8b4d43a4 2639 },
1b52c409 2640 .setkey = des3_aead_setkey,
479bcc7c
HX		 2641 .setauthsize = aead_setauthsize,
2642 .encrypt = aead_encrypt,
2643 .decrypt = aead_decrypt,
2644 .ivsize = DES3_EDE_BLOCK_SIZE,
2645 .maxauthsize = SHA1_DIGEST_SIZE,
2646 },
623814c0
HX		 2647 .aead.op = {
2648 .do_one_request = aead_do_one_req,
2649 },
479bcc7c
HX		 2650 .caam = {
2651 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2652 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2653 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2654 },
8b4d43a4 2655 },
8e8ec596 2656 {
623814c0 2657 .aead.base = {
479bcc7c
HX		 2658 .base = {
2659 .cra_name = "echainiv(authenc(hmac(sha1),"
2660 "cbc(des3_ede)))",
2661 .cra_driver_name = "echainiv-authenc-"
2662 "hmac-sha1-"
2663 "cbc-des3_ede-caam",
2664 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2665 },
1b52c409 2666 .setkey = des3_aead_setkey,
0e479300 2667 .setauthsize = aead_setauthsize,
479bcc7c 2668 .encrypt = aead_encrypt,
8b18e235 2669 .decrypt = aead_decrypt,
8e8ec596
KP		 2670 .ivsize = DES3_EDE_BLOCK_SIZE,
2671 .maxauthsize = SHA1_DIGEST_SIZE,
479bcc7c 2672 },
623814c0
HX		 2673 .aead.op = {
2674 .do_one_request = aead_do_one_req,
2675 },
479bcc7c
HX		 2676 .caam = {
2677 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2678 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2679 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2680 .geniv = true,
2681 },
2682 },
2683 {
623814c0 2684 .aead.base = {
479bcc7c
HX		 2685 .base = {
2686 .cra_name = "authenc(hmac(sha224),"
2687 "cbc(des3_ede))",
2688 .cra_driver_name = "authenc-hmac-sha224-"
2689 "cbc-des3_ede-caam",
2690 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
8e8ec596 2691 },
1b52c409 2692 .setkey = des3_aead_setkey,
479bcc7c
HX		 2693 .setauthsize = aead_setauthsize,
2694 .encrypt = aead_encrypt,
2695 .decrypt = aead_decrypt,
2696 .ivsize = DES3_EDE_BLOCK_SIZE,
2697 .maxauthsize = SHA224_DIGEST_SIZE,
2698 },
623814c0
HX		 2699 .aead.op = {
2700 .do_one_request = aead_do_one_req,
2701 },
479bcc7c
HX		 2702 .caam = {
2703 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2704 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2705 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2706 },
8e8ec596 2707 },
e863f9cc 2708 {
623814c0 2709 .aead.base = {
479bcc7c
HX		 2710 .base = {
2711 .cra_name = "echainiv(authenc(hmac(sha224),"
2712 "cbc(des3_ede)))",
2713 .cra_driver_name = "echainiv-authenc-"
2714 "hmac-sha224-"
2715 "cbc-des3_ede-caam",
2716 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2717 },
1b52c409 2718 .setkey = des3_aead_setkey,
e863f9cc 2719 .setauthsize = aead_setauthsize,
479bcc7c 2720 .encrypt = aead_encrypt,
8b18e235 2721 .decrypt = aead_decrypt,
e863f9cc
HA		 2722 .ivsize = DES3_EDE_BLOCK_SIZE,
2723 .maxauthsize = SHA224_DIGEST_SIZE,
479bcc7c 2724 },
623814c0
HX		 2725 .aead.op = {
2726 .do_one_request = aead_do_one_req,
2727 },
479bcc7c
HX		 2728 .caam = {
2729 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2730 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2731 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2732 .geniv = true,
2733 },
2734 },
2735 {
623814c0 2736 .aead.base = {
479bcc7c
HX		 2737 .base = {
2738 .cra_name = "authenc(hmac(sha256),"
2739 "cbc(des3_ede))",
2740 .cra_driver_name = "authenc-hmac-sha256-"
2741 "cbc-des3_ede-caam",
2742 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
e863f9cc 2743 },
1b52c409 2744 .setkey = des3_aead_setkey,
479bcc7c
HX		 2745 .setauthsize = aead_setauthsize,
2746 .encrypt = aead_encrypt,
2747 .decrypt = aead_decrypt,
2748 .ivsize = DES3_EDE_BLOCK_SIZE,
2749 .maxauthsize = SHA256_DIGEST_SIZE,
2750 },
623814c0
HX		 2751 .aead.op = {
2752 .do_one_request = aead_do_one_req,
2753 },
479bcc7c
HX		 2754 .caam = {
2755 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2756 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2757 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2758 },
e863f9cc 2759 },
8e8ec596 2760 {
623814c0 2761 .aead.base = {
479bcc7c
HX		 2762 .base = {
2763 .cra_name = "echainiv(authenc(hmac(sha256),"
2764 "cbc(des3_ede)))",
2765 .cra_driver_name = "echainiv-authenc-"
2766 "hmac-sha256-"
2767 "cbc-des3_ede-caam",
2768 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2769 },
1b52c409 2770 .setkey = des3_aead_setkey,
0e479300 2771 .setauthsize = aead_setauthsize,
479bcc7c 2772 .encrypt = aead_encrypt,
8b18e235 2773 .decrypt = aead_decrypt,
8e8ec596
KP		 2774 .ivsize = DES3_EDE_BLOCK_SIZE,
2775 .maxauthsize = SHA256_DIGEST_SIZE,
479bcc7c 2776 },
623814c0
HX		 2777 .aead.op = {
2778 .do_one_request = aead_do_one_req,
2779 },
479bcc7c
HX		 2780 .caam = {
2781 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2782 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2783 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2784 .geniv = true,
2785 },
2786 },
2787 {
623814c0 2788 .aead.base = {
479bcc7c
HX		 2789 .base = {
2790 .cra_name = "authenc(hmac(sha384),"
2791 "cbc(des3_ede))",
2792 .cra_driver_name = "authenc-hmac-sha384-"
2793 "cbc-des3_ede-caam",
2794 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
8e8ec596 2795 },
1b52c409 2796 .setkey = des3_aead_setkey,
479bcc7c
HX		 2797 .setauthsize = aead_setauthsize,
2798 .encrypt = aead_encrypt,
2799 .decrypt = aead_decrypt,
2800 .ivsize = DES3_EDE_BLOCK_SIZE,
2801 .maxauthsize = SHA384_DIGEST_SIZE,
2802 },
623814c0
HX		 2803 .aead.op = {
2804 .do_one_request = aead_do_one_req,
2805 },
479bcc7c
HX		 2806 .caam = {
2807 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2808 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2809 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2810 },
8e8ec596 2811 },
e863f9cc 2812 {
623814c0 2813 .aead.base = {
479bcc7c
HX		 2814 .base = {
2815 .cra_name = "echainiv(authenc(hmac(sha384),"
2816 "cbc(des3_ede)))",
2817 .cra_driver_name = "echainiv-authenc-"
2818 "hmac-sha384-"
2819 "cbc-des3_ede-caam",
2820 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2821 },
1b52c409 2822 .setkey = des3_aead_setkey,
e863f9cc 2823 .setauthsize = aead_setauthsize,
479bcc7c 2824 .encrypt = aead_encrypt,
8b18e235 2825 .decrypt = aead_decrypt,
e863f9cc
HA		 2826 .ivsize = DES3_EDE_BLOCK_SIZE,
2827 .maxauthsize = SHA384_DIGEST_SIZE,
479bcc7c 2828 },
623814c0
HX		 2829 .aead.op = {
2830 .do_one_request = aead_do_one_req,
2831 },
479bcc7c
HX		 2832 .caam = {
2833 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2834 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2835 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2836 .geniv = true,
2837 },
2838 },
2839 {
623814c0 2840 .aead.base = {
479bcc7c
HX		 2841 .base = {
2842 .cra_name = "authenc(hmac(sha512),"
2843 "cbc(des3_ede))",
2844 .cra_driver_name = "authenc-hmac-sha512-"
2845 "cbc-des3_ede-caam",
2846 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
e863f9cc 2847 },
1b52c409 2848 .setkey = des3_aead_setkey,
479bcc7c
HX		 2849 .setauthsize = aead_setauthsize,
2850 .encrypt = aead_encrypt,
2851 .decrypt = aead_decrypt,
2852 .ivsize = DES3_EDE_BLOCK_SIZE,
2853 .maxauthsize = SHA512_DIGEST_SIZE,
2854 },
623814c0
HX		 2855 .aead.op = {
2856 .do_one_request = aead_do_one_req,
2857 },
479bcc7c
HX		 2858 .caam = {
2859 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2860 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2861 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2862 },
e863f9cc 2863 },
4427b1b4 2864 {
623814c0 2865 .aead.base = {
479bcc7c
HX		 2866 .base = {
2867 .cra_name = "echainiv(authenc(hmac(sha512),"
2868 "cbc(des3_ede)))",
2869 .cra_driver_name = "echainiv-authenc-"
2870 "hmac-sha512-"
2871 "cbc-des3_ede-caam",
2872 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2873 },
1b52c409 2874 .setkey = des3_aead_setkey,
0e479300 2875 .setauthsize = aead_setauthsize,
479bcc7c 2876 .encrypt = aead_encrypt,
8b18e235 2877 .decrypt = aead_decrypt,
4427b1b4
KP		 2878 .ivsize = DES3_EDE_BLOCK_SIZE,
2879 .maxauthsize = SHA512_DIGEST_SIZE,
479bcc7c 2880 },
623814c0
HX		 2881 .aead.op = {
2882 .do_one_request = aead_do_one_req,
2883 },
479bcc7c
HX		 2884 .caam = {
2885 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2886 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2887 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2888 .geniv = true,
2889 },
2890 },
2891 {
623814c0 2892 .aead.base = {
479bcc7c
HX		 2893 .base = {
2894 .cra_name = "authenc(hmac(md5),cbc(des))",
2895 .cra_driver_name = "authenc-hmac-md5-"
2896 "cbc-des-caam",
2897 .cra_blocksize = DES_BLOCK_SIZE,
4427b1b4 2898 },
479bcc7c
HX		 2899 .setkey = aead_setkey,
2900 .setauthsize = aead_setauthsize,
2901 .encrypt = aead_encrypt,
2902 .decrypt = aead_decrypt,
2903 .ivsize = DES_BLOCK_SIZE,
2904 .maxauthsize = MD5_DIGEST_SIZE,
2905 },
623814c0
HX		 2906 .aead.op = {
2907 .do_one_request = aead_do_one_req,
2908 },
479bcc7c
HX		 2909 .caam = {
2910 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2911 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2912 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2913 },
4427b1b4 2914 },
8b4d43a4 2915 {
623814c0 2916 .aead.base = {
479bcc7c
HX		 2917 .base = {
2918 .cra_name = "echainiv(authenc(hmac(md5),"
2919 "cbc(des)))",
2920 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2921 "cbc-des-caam",
2922 .cra_blocksize = DES_BLOCK_SIZE,
2923 },
8b4d43a4
KP		 2924 .setkey = aead_setkey,
2925 .setauthsize = aead_setauthsize,
479bcc7c 2926 .encrypt = aead_encrypt,
8b18e235 2927 .decrypt = aead_decrypt,
8b4d43a4
KP		 2928 .ivsize = DES_BLOCK_SIZE,
2929 .maxauthsize = MD5_DIGEST_SIZE,
479bcc7c 2930 },
623814c0
HX		 2931 .aead.op = {
2932 .do_one_request = aead_do_one_req,
2933 },
479bcc7c
HX		 2934 .caam = {
2935 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2936 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2937 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2938 .geniv = true,
2939 },
2940 },
2941 {
623814c0 2942 .aead.base = {
479bcc7c
HX		 2943 .base = {
2944 .cra_name = "authenc(hmac(sha1),cbc(des))",
2945 .cra_driver_name = "authenc-hmac-sha1-"
2946 "cbc-des-caam",
2947 .cra_blocksize = DES_BLOCK_SIZE,
8b4d43a4 2948 },
479bcc7c
HX		 2949 .setkey = aead_setkey,
2950 .setauthsize = aead_setauthsize,
2951 .encrypt = aead_encrypt,
2952 .decrypt = aead_decrypt,
2953 .ivsize = DES_BLOCK_SIZE,
2954 .maxauthsize = SHA1_DIGEST_SIZE,
2955 },
623814c0
HX		 2956 .aead.op = {
2957 .do_one_request = aead_do_one_req,
2958 },
479bcc7c
HX		 2959 .caam = {
2960 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2961 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2962 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 2963 },
8b4d43a4 2964 },
8e8ec596 2965 {
623814c0 2966 .aead.base = {
479bcc7c
HX		 2967 .base = {
2968 .cra_name = "echainiv(authenc(hmac(sha1),"
2969 "cbc(des)))",
2970 .cra_driver_name = "echainiv-authenc-"
2971 "hmac-sha1-cbc-des-caam",
2972 .cra_blocksize = DES_BLOCK_SIZE,
2973 },
0e479300
YK		 2974 .setkey = aead_setkey,
2975 .setauthsize = aead_setauthsize,
479bcc7c 2976 .encrypt = aead_encrypt,
8b18e235 2977 .decrypt = aead_decrypt,
8e8ec596
KP		 2978 .ivsize = DES_BLOCK_SIZE,
2979 .maxauthsize = SHA1_DIGEST_SIZE,
479bcc7c 2980 },
623814c0
HX		 2981 .aead.op = {
2982 .do_one_request = aead_do_one_req,
2983 },
479bcc7c
HX		 2984 .caam = {
2985 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2986 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2987 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 2988 .geniv = true,
2989 },
2990 },
2991 {
623814c0 2992 .aead.base = {
479bcc7c
HX		 2993 .base = {
2994 .cra_name = "authenc(hmac(sha224),cbc(des))",
2995 .cra_driver_name = "authenc-hmac-sha224-"
2996 "cbc-des-caam",
2997 .cra_blocksize = DES_BLOCK_SIZE,
8e8ec596 2998 },
479bcc7c
HX		 2999 .setkey = aead_setkey,
3000 .setauthsize = aead_setauthsize,
3001 .encrypt = aead_encrypt,
3002 .decrypt = aead_decrypt,
3003 .ivsize = DES_BLOCK_SIZE,
3004 .maxauthsize = SHA224_DIGEST_SIZE,
3005 },
623814c0
HX		 3006 .aead.op = {
3007 .do_one_request = aead_do_one_req,
3008 },
479bcc7c
HX		 3009 .caam = {
3010 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3011 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3012 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 3013 },
8e8ec596 3014 },
e863f9cc 3015 {
623814c0 3016 .aead.base = {
479bcc7c
HX		 3017 .base = {
3018 .cra_name = "echainiv(authenc(hmac(sha224),"
3019 "cbc(des)))",
3020 .cra_driver_name = "echainiv-authenc-"
3021 "hmac-sha224-cbc-des-caam",
3022 .cra_blocksize = DES_BLOCK_SIZE,
3023 },
e863f9cc
HA		 3024 .setkey = aead_setkey,
3025 .setauthsize = aead_setauthsize,
479bcc7c 3026 .encrypt = aead_encrypt,
8b18e235 3027 .decrypt = aead_decrypt,
e863f9cc
HA		 3028 .ivsize = DES_BLOCK_SIZE,
3029 .maxauthsize = SHA224_DIGEST_SIZE,
479bcc7c 3030 },
623814c0
HX		 3031 .aead.op = {
3032 .do_one_request = aead_do_one_req,
3033 },
479bcc7c
HX		 3034 .caam = {
3035 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3036 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3037 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3038 .geniv = true,
3039 },
3040 },
3041 {
623814c0 3042 .aead.base = {
479bcc7c
HX		 3043 .base = {
3044 .cra_name = "authenc(hmac(sha256),cbc(des))",
3045 .cra_driver_name = "authenc-hmac-sha256-"
3046 "cbc-des-caam",
3047 .cra_blocksize = DES_BLOCK_SIZE,
e863f9cc 3048 },
479bcc7c
HX		 3049 .setkey = aead_setkey,
3050 .setauthsize = aead_setauthsize,
3051 .encrypt = aead_encrypt,
3052 .decrypt = aead_decrypt,
3053 .ivsize = DES_BLOCK_SIZE,
3054 .maxauthsize = SHA256_DIGEST_SIZE,
3055 },
623814c0
HX		 3056 .aead.op = {
3057 .do_one_request = aead_do_one_req,
3058 },
479bcc7c
HX		 3059 .caam = {
3060 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3061 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3062 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 3063 },
e863f9cc 3064 },
8e8ec596 3065 {
623814c0 3066 .aead.base = {
479bcc7c
HX		 3067 .base = {
3068 .cra_name = "echainiv(authenc(hmac(sha256),"
3069 "cbc(des)))",
3070 .cra_driver_name = "echainiv-authenc-"
3071 "hmac-sha256-cbc-des-caam",
3072 .cra_blocksize = DES_BLOCK_SIZE,
3073 },
0e479300
YK		 3074 .setkey = aead_setkey,
3075 .setauthsize = aead_setauthsize,
479bcc7c 3076 .encrypt = aead_encrypt,
8b18e235 3077 .decrypt = aead_decrypt,
8e8ec596
KP		 3078 .ivsize = DES_BLOCK_SIZE,
3079 .maxauthsize = SHA256_DIGEST_SIZE,
479bcc7c 3080 },
623814c0
HX		 3081 .aead.op = {
3082 .do_one_request = aead_do_one_req,
3083 },
479bcc7c
HX		 3084 .caam = {
3085 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3086 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3087 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3088 .geniv = true,
3089 },
3090 },
3091 {
623814c0 3092 .aead.base = {
479bcc7c
HX		 3093 .base = {
3094 .cra_name = "authenc(hmac(sha384),cbc(des))",
3095 .cra_driver_name = "authenc-hmac-sha384-"
3096 "cbc-des-caam",
3097 .cra_blocksize = DES_BLOCK_SIZE,
8e8ec596 3098 },
479bcc7c
HX		 3099 .setkey = aead_setkey,
3100 .setauthsize = aead_setauthsize,
3101 .encrypt = aead_encrypt,
3102 .decrypt = aead_decrypt,
3103 .ivsize = DES_BLOCK_SIZE,
3104 .maxauthsize = SHA384_DIGEST_SIZE,
3105 },
623814c0
HX		 3106 .aead.op = {
3107 .do_one_request = aead_do_one_req,
3108 },
479bcc7c
HX		 3109 .caam = {
3110 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3111 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3112 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 3113 },
8e8ec596 3114 },
e863f9cc 3115 {
623814c0 3116 .aead.base = {
479bcc7c
HX		 3117 .base = {
3118 .cra_name = "echainiv(authenc(hmac(sha384),"
3119 "cbc(des)))",
3120 .cra_driver_name = "echainiv-authenc-"
3121 "hmac-sha384-cbc-des-caam",
3122 .cra_blocksize = DES_BLOCK_SIZE,
3123 },
e863f9cc
HA		 3124 .setkey = aead_setkey,
3125 .setauthsize = aead_setauthsize,
479bcc7c 3126 .encrypt = aead_encrypt,
8b18e235 3127 .decrypt = aead_decrypt,
e863f9cc
HA		 3128 .ivsize = DES_BLOCK_SIZE,
3129 .maxauthsize = SHA384_DIGEST_SIZE,
479bcc7c 3130 },
623814c0
HX		 3131 .aead.op = {
3132 .do_one_request = aead_do_one_req,
3133 },
479bcc7c
HX		 3134 .caam = {
3135 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3136 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3137 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3138 .geniv = true,
3139 },
3140 },
3141 {
623814c0 3142 .aead.base = {
479bcc7c
HX		 3143 .base = {
3144 .cra_name = "authenc(hmac(sha512),cbc(des))",
3145 .cra_driver_name = "authenc-hmac-sha512-"
3146 "cbc-des-caam",
3147 .cra_blocksize = DES_BLOCK_SIZE,
e863f9cc 3148 },
479bcc7c
HX		 3149 .setkey = aead_setkey,
3150 .setauthsize = aead_setauthsize,
3151 .encrypt = aead_encrypt,
3152 .decrypt = aead_decrypt,
3153 .ivsize = DES_BLOCK_SIZE,
3154 .maxauthsize = SHA512_DIGEST_SIZE,
3155 },
623814c0
HX		 3156 .aead.op = {
3157 .do_one_request = aead_do_one_req,
3158 },
479bcc7c
HX		 3159 .caam = {
3160 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3161 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3162 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 3163 },
e863f9cc 3164 },
4427b1b4 3165 {
623814c0 3166 .aead.base = {
479bcc7c
HX		 3167 .base = {
3168 .cra_name = "echainiv(authenc(hmac(sha512),"
3169 "cbc(des)))",
3170 .cra_driver_name = "echainiv-authenc-"
3171 "hmac-sha512-cbc-des-caam",
3172 .cra_blocksize = DES_BLOCK_SIZE,
3173 },
0e479300
YK		 3174 .setkey = aead_setkey,
3175 .setauthsize = aead_setauthsize,
479bcc7c 3176 .encrypt = aead_encrypt,
8b18e235 3177 .decrypt = aead_decrypt,
4427b1b4
KP		 3178 .ivsize = DES_BLOCK_SIZE,
3179 .maxauthsize = SHA512_DIGEST_SIZE,
479bcc7c 3180 },
623814c0
HX		 3181 .aead.op = {
3182 .do_one_request = aead_do_one_req,
3183 },
479bcc7c
HX		 3184 .caam = {
3185 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3186 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3187 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3188 .geniv = true,
3189 },
4427b1b4 3190 },
daebc465 3191 {
623814c0 3192 .aead.base = {
479bcc7c
HX		 3193 .base = {
3194 .cra_name = "authenc(hmac(md5),"
3195 "rfc3686(ctr(aes)))",
3196 .cra_driver_name = "authenc-hmac-md5-"
3197 "rfc3686-ctr-aes-caam",
3198 .cra_blocksize = 1,
3199 },
daebc465
CV		 3200 .setkey = aead_setkey,
3201 .setauthsize = aead_setauthsize,
479bcc7c
HX		 3202 .encrypt = aead_encrypt,
3203 .decrypt = aead_decrypt,
daebc465
CV		 3204 .ivsize = CTR_RFC3686_IV_SIZE,
3205 .maxauthsize = MD5_DIGEST_SIZE,
479bcc7c 3206 },
623814c0
HX		 3207 .aead.op = {
3208 .do_one_request = aead_do_one_req,
3209 },
479bcc7c
HX		 3210 .caam = {
3211 .class1_alg_type = OP_ALG_ALGSEL_AES |
3212 OP_ALG_AAI_CTR_MOD128,
3213 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3214 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3215 .rfc3686 = true,
3216 },
daebc465
CV		 3217 },
3218 {
623814c0 3219 .aead.base = {
479bcc7c
HX		 3220 .base = {
3221 .cra_name = "seqiv(authenc("
3222 "hmac(md5),rfc3686(ctr(aes))))",
3223 .cra_driver_name = "seqiv-authenc-hmac-md5-"
3224 "rfc3686-ctr-aes-caam",
3225 .cra_blocksize = 1,
3226 },
daebc465
CV		 3227 .setkey = aead_setkey,
3228 .setauthsize = aead_setauthsize,
479bcc7c 3229 .encrypt = aead_encrypt,
8b18e235 3230 .decrypt = aead_decrypt,
daebc465 3231 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3232 .maxauthsize = MD5_DIGEST_SIZE,
3233 },
623814c0
HX		 3234 .aead.op = {
3235 .do_one_request = aead_do_one_req,
3236 },
479bcc7c
HX		 3237 .caam = {
3238 .class1_alg_type = OP_ALG_ALGSEL_AES |
3239 OP_ALG_AAI_CTR_MOD128,
3240 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3241 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3242 .rfc3686 = true,
3243 .geniv = true,
3244 },
daebc465
CV		 3245 },
3246 {
623814c0 3247 .aead.base = {
479bcc7c
HX		 3248 .base = {
3249 .cra_name = "authenc(hmac(sha1),"
3250 "rfc3686(ctr(aes)))",
3251 .cra_driver_name = "authenc-hmac-sha1-"
3252 "rfc3686-ctr-aes-caam",
3253 .cra_blocksize = 1,
3254 },
daebc465
CV		 3255 .setkey = aead_setkey,
3256 .setauthsize = aead_setauthsize,
479bcc7c
HX		 3257 .encrypt = aead_encrypt,
3258 .decrypt = aead_decrypt,
daebc465 3259 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3260 .maxauthsize = SHA1_DIGEST_SIZE,
3261 },
623814c0
HX		 3262 .aead.op = {
3263 .do_one_request = aead_do_one_req,
3264 },
479bcc7c
HX		 3265 .caam = {
3266 .class1_alg_type = OP_ALG_ALGSEL_AES |
3267 OP_ALG_AAI_CTR_MOD128,
3268 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3269 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3270 .rfc3686 = true,
3271 },
daebc465
CV		 3272 },
3273 {
623814c0 3274 .aead.base = {
479bcc7c
HX		 3275 .base = {
3276 .cra_name = "seqiv(authenc("
3277 "hmac(sha1),rfc3686(ctr(aes))))",
3278 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3279 "rfc3686-ctr-aes-caam",
3280 .cra_blocksize = 1,
3281 },
daebc465
CV		 3282 .setkey = aead_setkey,
3283 .setauthsize = aead_setauthsize,
479bcc7c 3284 .encrypt = aead_encrypt,
8b18e235 3285 .decrypt = aead_decrypt,
daebc465 3286 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3287 .maxauthsize = SHA1_DIGEST_SIZE,
3288 },
623814c0
HX		 3289 .aead.op = {
3290 .do_one_request = aead_do_one_req,
3291 },
479bcc7c
HX		 3292 .caam = {
3293 .class1_alg_type = OP_ALG_ALGSEL_AES |
3294 OP_ALG_AAI_CTR_MOD128,
3295 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3296 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3297 .rfc3686 = true,
3298 .geniv = true,
3299 },
daebc465
CV		 3300 },
3301 {
623814c0 3302 .aead.base = {
479bcc7c
HX		 3303 .base = {
3304 .cra_name = "authenc(hmac(sha224),"
3305 "rfc3686(ctr(aes)))",
3306 .cra_driver_name = "authenc-hmac-sha224-"
3307 "rfc3686-ctr-aes-caam",
3308 .cra_blocksize = 1,
3309 },
daebc465
CV		 3310 .setkey = aead_setkey,
3311 .setauthsize = aead_setauthsize,
479bcc7c
HX		 3312 .encrypt = aead_encrypt,
3313 .decrypt = aead_decrypt,
daebc465 3314 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3315 .maxauthsize = SHA224_DIGEST_SIZE,
3316 },
623814c0
HX		 3317 .aead.op = {
3318 .do_one_request = aead_do_one_req,
3319 },
479bcc7c
HX		 3320 .caam = {
3321 .class1_alg_type = OP_ALG_ALGSEL_AES |
3322 OP_ALG_AAI_CTR_MOD128,
3323 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3324 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3325 .rfc3686 = true,
3326 },
daebc465
CV		 3327 },
3328 {
623814c0 3329 .aead.base = {
479bcc7c
HX		 3330 .base = {
3331 .cra_name = "seqiv(authenc("
3332 "hmac(sha224),rfc3686(ctr(aes))))",
3333 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3334 "rfc3686-ctr-aes-caam",
3335 .cra_blocksize = 1,
3336 },
daebc465
CV		 3337 .setkey = aead_setkey,
3338 .setauthsize = aead_setauthsize,
479bcc7c 3339 .encrypt = aead_encrypt,
8b18e235 3340 .decrypt = aead_decrypt,
daebc465 3341 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3342 .maxauthsize = SHA224_DIGEST_SIZE,
3343 },
623814c0
HX		 3344 .aead.op = {
3345 .do_one_request = aead_do_one_req,
3346 },
479bcc7c
HX		 3347 .caam = {
3348 .class1_alg_type = OP_ALG_ALGSEL_AES |
3349 OP_ALG_AAI_CTR_MOD128,
3350 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3351 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3352 .rfc3686 = true,
3353 .geniv = true,
3354 },
acdca31d
YK		 3355 },
3356 {
623814c0 3357 .aead.base = {
479bcc7c
HX		 3358 .base = {
3359 .cra_name = "authenc(hmac(sha256),"
3360 "rfc3686(ctr(aes)))",
3361 .cra_driver_name = "authenc-hmac-sha256-"
3362 "rfc3686-ctr-aes-caam",
3363 .cra_blocksize = 1,
acdca31d 3364 },
479bcc7c
HX		 3365 .setkey = aead_setkey,
3366 .setauthsize = aead_setauthsize,
3367 .encrypt = aead_encrypt,
3368 .decrypt = aead_decrypt,
3369 .ivsize = CTR_RFC3686_IV_SIZE,
3370 .maxauthsize = SHA256_DIGEST_SIZE,
3371 },
623814c0
HX		 3372 .aead.op = {
3373 .do_one_request = aead_do_one_req,
3374 },
479bcc7c
HX		 3375 .caam = {
3376 .class1_alg_type = OP_ALG_ALGSEL_AES |
3377 OP_ALG_AAI_CTR_MOD128,
3378 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3379 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3380 .rfc3686 = true,
3381 },
acdca31d
YK		 3382 },
3383 {
623814c0 3384 .aead.base = {
479bcc7c
HX		 3385 .base = {
3386 .cra_name = "seqiv(authenc(hmac(sha256),"
3387 "rfc3686(ctr(aes))))",
3388 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3389 "rfc3686-ctr-aes-caam",
3390 .cra_blocksize = 1,
acdca31d 3391 },
479bcc7c
HX		 3392 .setkey = aead_setkey,
3393 .setauthsize = aead_setauthsize,
3394 .encrypt = aead_encrypt,
8b18e235 3395 .decrypt = aead_decrypt,
479bcc7c
HX		 3396 .ivsize = CTR_RFC3686_IV_SIZE,
3397 .maxauthsize = SHA256_DIGEST_SIZE,
3398 },
623814c0
HX		 3399 .aead.op = {
3400 .do_one_request = aead_do_one_req,
3401 },
479bcc7c
HX		 3402 .caam = {
3403 .class1_alg_type = OP_ALG_ALGSEL_AES |
3404 OP_ALG_AAI_CTR_MOD128,
3405 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3406 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3407 .rfc3686 = true,
3408 .geniv = true,
3409 },
2b22f6c5
CV		 3410 },
3411 {
623814c0 3412 .aead.base = {
479bcc7c
HX		 3413 .base = {
3414 .cra_name = "authenc(hmac(sha384),"
3415 "rfc3686(ctr(aes)))",
3416 .cra_driver_name = "authenc-hmac-sha384-"
3417 "rfc3686-ctr-aes-caam",
3418 .cra_blocksize = 1,
2b22f6c5 3419 },
479bcc7c
HX		 3420 .setkey = aead_setkey,
3421 .setauthsize = aead_setauthsize,
3422 .encrypt = aead_encrypt,
3423 .decrypt = aead_decrypt,
a5f57cff 3424 .ivsize = CTR_RFC3686_IV_SIZE,
479bcc7c
HX		 3425 .maxauthsize = SHA384_DIGEST_SIZE,
3426 },
623814c0
HX		 3427 .aead.op = {
3428 .do_one_request = aead_do_one_req,
3429 },
479bcc7c
HX		 3430 .caam = {
3431 .class1_alg_type = OP_ALG_ALGSEL_AES |
3432 OP_ALG_AAI_CTR_MOD128,
3433 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3434 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3435 .rfc3686 = true,
3436 },
3437 },
f2147b88 3438 {
623814c0 3439 .aead.base = {
f2147b88 3440 .base = {
479bcc7c
HX		 3441 .cra_name = "seqiv(authenc(hmac(sha384),"
3442 "rfc3686(ctr(aes))))",
3443 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3444 "rfc3686-ctr-aes-caam",
f2147b88
HX		 3445 .cra_blocksize = 1,
3446 },
479bcc7c
HX		 3447 .setkey = aead_setkey,
3448 .setauthsize = aead_setauthsize,
3449 .encrypt = aead_encrypt,
8b18e235 3450 .decrypt = aead_decrypt,
479bcc7c
HX		 3451 .ivsize = CTR_RFC3686_IV_SIZE,
3452 .maxauthsize = SHA384_DIGEST_SIZE,
f2147b88 3453 },
623814c0
HX		 3454 .aead.op = {
3455 .do_one_request = aead_do_one_req,
3456 },
f2147b88 3457 .caam = {
479bcc7c
HX		 3458 .class1_alg_type = OP_ALG_ALGSEL_AES |
3459 OP_ALG_AAI_CTR_MOD128,
3460 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3461 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3462 .rfc3686 = true,
3463 .geniv = true,
f2147b88
HX		 3464 },
3465 },
3466 {
623814c0 3467 .aead.base = {
f2147b88 3468 .base = {
479bcc7c
HX		 3469 .cra_name = "authenc(hmac(sha512),"
3470 "rfc3686(ctr(aes)))",
3471 .cra_driver_name = "authenc-hmac-sha512-"
3472 "rfc3686-ctr-aes-caam",
f2147b88
HX		 3473 .cra_blocksize = 1,
3474 },
479bcc7c
HX		 3475 .setkey = aead_setkey,
3476 .setauthsize = aead_setauthsize,
3477 .encrypt = aead_encrypt,
3478 .decrypt = aead_decrypt,
3479 .ivsize = CTR_RFC3686_IV_SIZE,
3480 .maxauthsize = SHA512_DIGEST_SIZE,
f2147b88 3481 },
623814c0
HX		 3482 .aead.op = {
3483 .do_one_request = aead_do_one_req,
3484 },
f2147b88 3485 .caam = {
479bcc7c
HX		 3486 .class1_alg_type = OP_ALG_ALGSEL_AES |
3487 OP_ALG_AAI_CTR_MOD128,
3488 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3489 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c 3490 .rfc3686 = true,
f2147b88
HX		 3491 },
3492 },
f2147b88 3493 {
623814c0 3494 .aead.base = {
f2147b88 3495 .base = {
479bcc7c
HX		 3496 .cra_name = "seqiv(authenc(hmac(sha512),"
3497 "rfc3686(ctr(aes))))",
3498 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3499 "rfc3686-ctr-aes-caam",
f2147b88
HX		 3500 .cra_blocksize = 1,
3501 },
479bcc7c
HX		 3502 .setkey = aead_setkey,
3503 .setauthsize = aead_setauthsize,
3504 .encrypt = aead_encrypt,
8b18e235 3505 .decrypt = aead_decrypt,
479bcc7c
HX		 3506 .ivsize = CTR_RFC3686_IV_SIZE,
3507 .maxauthsize = SHA512_DIGEST_SIZE,
f2147b88 3508 },
623814c0
HX		 3509 .aead.op = {
3510 .do_one_request = aead_do_one_req,
3511 },
f2147b88 3512 .caam = {
479bcc7c
HX		 3513 .class1_alg_type = OP_ALG_ALGSEL_AES |
3514 OP_ALG_AAI_CTR_MOD128,
3515 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3516 OP_ALG_AAI_HMAC_PRECOMP,
479bcc7c
HX		 3517 .rfc3686 = true,
3518 .geniv = true,
f2147b88
HX		 3519 },
3520 },
d6bbd4ee 3521 {
623814c0 3522 .aead.base = {
d6bbd4ee
HG		 3523 .base = {
3524 .cra_name = "rfc7539(chacha20,poly1305)",
3525 .cra_driver_name = "rfc7539-chacha20-poly1305-"
3526 "caam",
3527 .cra_blocksize = 1,
3528 },
3529 .setkey = chachapoly_setkey,
3530 .setauthsize = chachapoly_setauthsize,
3531 .encrypt = chachapoly_encrypt,
3532 .decrypt = chachapoly_decrypt,
3533 .ivsize = CHACHAPOLY_IV_SIZE,
3534 .maxauthsize = POLY1305_DIGEST_SIZE,
3535 },
623814c0
HX		 3536 .aead.op = {
3537 .do_one_request = aead_do_one_req,
3538 },
d6bbd4ee
HG		 3539 .caam = {
3540 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3541 OP_ALG_AAI_AEAD,
3542 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3543 OP_ALG_AAI_AEAD,
24586b5f 3544 .nodkp = true,
d6bbd4ee
HG		 3545 },
3546 },
3547 {
623814c0 3548 .aead.base = {
d6bbd4ee
HG		 3549 .base = {
3550 .cra_name = "rfc7539esp(chacha20,poly1305)",
3551 .cra_driver_name = "rfc7539esp-chacha20-"
3552 "poly1305-caam",
3553 .cra_blocksize = 1,
3554 },
3555 .setkey = chachapoly_setkey,
3556 .setauthsize = chachapoly_setauthsize,
3557 .encrypt = chachapoly_encrypt,
3558 .decrypt = chachapoly_decrypt,
3559 .ivsize = 8,
3560 .maxauthsize = POLY1305_DIGEST_SIZE,
3561 },
623814c0
HX		 3562 .aead.op = {
3563 .do_one_request = aead_do_one_req,
3564 },
d6bbd4ee
HG		 3565 .caam = {
3566 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3567 OP_ALG_AAI_AEAD,
3568 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3569 OP_ALG_AAI_AEAD,
24586b5f 3570 .nodkp = true,
d6bbd4ee
HG		 3571 },
3572 },
f2147b88
HX		 3573};
3574
7e0880b9
HG		 3575static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3576 bool uses_dkp)
8e8ec596 3577{
bbf22344 3578 dma_addr_t dma_addr;
7e0880b9 3579 struct caam_drv_private *priv;
ee38767f
IP		 3580 const size_t sh_desc_enc_offset = offsetof(struct caam_ctx,
3581 sh_desc_enc);
bbf22344 3582
cfc6f11b
RG		 3583 ctx->jrdev = caam_jr_alloc();
3584 if (IS_ERR(ctx->jrdev)) {
3585 pr_err("Job Ring Device allocation for transform failed\n");
3586 return PTR_ERR(ctx->jrdev);
3587 }
8e8ec596 3588
7e0880b9
HG		 3589 priv = dev_get_drvdata(ctx->jrdev->parent);
3590 if (priv->era >= 6 && uses_dkp)
3591 ctx->dir = DMA_BIDIRECTIONAL;
3592 else
3593 ctx->dir = DMA_TO_DEVICE;
3594
bbf22344
HG		 3595 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3596 offsetof(struct caam_ctx,
ee38767f
IP		 3597 sh_desc_enc_dma) -
3598 sh_desc_enc_offset,
7e0880b9 3599 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
bbf22344
HG		 3600 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3601 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3602 caam_jr_free(ctx->jrdev);
3603 return -ENOMEM;
3604 }
3605
3606 ctx->sh_desc_enc_dma = dma_addr;
3607 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
ee38767f
IP		 3608 sh_desc_dec) -
3609 sh_desc_enc_offset;
3610 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) -
3611 sh_desc_enc_offset;
bbf22344 3612
8e8ec596 3613 /* copy descriptor header template value */
db57656b
HG		 3614 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3615 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
8e8ec596
KP		 3616
3617 return 0;
3618}
3619
5ca7badb 3620static int caam_cra_init(struct crypto_skcipher *tfm)
8e8ec596 3621{
5ca7badb
HG		 3622 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3623 struct caam_skcipher_alg *caam_alg =
623814c0 3624 container_of(alg, typeof(*caam_alg), skcipher.base);
4cb4f7c1 3625 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
9d9b14db
AB		 3626 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3627 int ret = 0;
ee38767f 3628
9d9b14db
AB		 3629 if (alg_aai == OP_ALG_AAI_XTS) {
3630 const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
3631 struct crypto_skcipher *fallback;
3632
3633 fallback = crypto_alloc_skcipher(tfm_name, 0,
3634 CRYPTO_ALG_NEED_FALLBACK);
3635 if (IS_ERR(fallback)) {
ab95bd2a
HG		 3636 pr_err("Failed to allocate %s fallback: %ld\n",
3637 tfm_name, PTR_ERR(fallback));
9d9b14db
AB		 3638 return PTR_ERR(fallback);
3639 }
3640
3641 ctx->fallback = fallback;
3642 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
3643 crypto_skcipher_reqsize(fallback));
3644 } else {
3645 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
3646 }
3647
3648 ret = caam_init_common(ctx, &caam_alg->caam, false);
3649 if (ret && ctx->fallback)
3650 crypto_free_skcipher(ctx->fallback);
3651
3652 return ret;
f2147b88
HX		 3653}
3654
3655static int caam_aead_init(struct crypto_aead *tfm)
3656{
3657 struct aead_alg *alg = crypto_aead_alg(tfm);
3658 struct caam_aead_alg *caam_alg =
623814c0 3659 container_of(alg, struct caam_aead_alg, aead.base);
4cb4f7c1 3660 struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm);
f2147b88 3661
1c240226
IP		 3662 crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx));
3663
24586b5f 3664 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
f2147b88
HX		 3665}
3666
3667static void caam_exit_common(struct caam_ctx *ctx)
3668{
bbf22344 3669 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
ee38767f
IP		 3670 offsetof(struct caam_ctx, sh_desc_enc_dma) -
3671 offsetof(struct caam_ctx, sh_desc_enc),
7e0880b9 3672 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
cfc6f11b 3673 caam_jr_free(ctx->jrdev);
8e8ec596
KP		 3674}
3675
5ca7badb 3676static void caam_cra_exit(struct crypto_skcipher *tfm)
f2147b88 3677{
4cb4f7c1 3678 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
9d9b14db
AB		 3679
3680 if (ctx->fallback)
3681 crypto_free_skcipher(ctx->fallback);
3682 caam_exit_common(ctx);
f2147b88
HX		 3683}
3684
3685static void caam_aead_exit(struct crypto_aead *tfm)
3686{
4cb4f7c1 3687 caam_exit_common(crypto_aead_ctx_dma(tfm));
f2147b88
HX		 3688}
3689
1b46c90c 3690void caam_algapi_exit(void)
8e8ec596 3691{
f2147b88
HX		 3692 int i;
3693
3694 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3695 struct caam_aead_alg *t_alg = driver_aeads + i;
3696
3697 if (t_alg->registered)
623814c0 3698 crypto_engine_unregister_aead(&t_alg->aead);
f2147b88 3699 }
8e8ec596 3700
5ca7badb
HG		 3701 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3702 struct caam_skcipher_alg *t_alg = driver_algs + i;
8e8ec596 3703
5ca7badb 3704 if (t_alg->registered)
623814c0 3705 crypto_engine_unregister_skcipher(&t_alg->skcipher);
8e8ec596 3706 }
8e8ec596
KP		 3707}
3708
5ca7badb 3709static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
8e8ec596 3710{
623814c0 3711 struct skcipher_alg *alg = &t_alg->skcipher.base;
8e8ec596 3712
5ca7badb
HG		 3713 alg->base.cra_module = THIS_MODULE;
3714 alg->base.cra_priority = CAAM_CRA_PRIORITY;
4cb4f7c1 3715 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
9d9b14db
AB		 3716 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3717 CRYPTO_ALG_KERN_DRIVER_ONLY);
8e8ec596 3718
5ca7badb
HG		 3719 alg->init = caam_cra_init;
3720 alg->exit = caam_cra_exit;
8e8ec596
KP		 3721}
3722
f2147b88
HX		 3723static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3724{
623814c0 3725 struct aead_alg *alg = &t_alg->aead.base;
f2147b88
HX		 3726
3727 alg->base.cra_module = THIS_MODULE;
3728 alg->base.cra_priority = CAAM_CRA_PRIORITY;
4cb4f7c1 3729 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
b8aa7dc5
MP		 3730 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3731 CRYPTO_ALG_KERN_DRIVER_ONLY;
f2147b88
HX		 3732
3733 alg->init = caam_aead_init;
3734 alg->exit = caam_aead_exit;
3735}
3736
1b46c90c 3737int caam_algapi_init(struct device *ctrldev)
8e8ec596 3738{
1b46c90c 3739 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
8e8ec596 3740 int i = 0, err = 0;
d6bbd4ee 3741 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
bf83490e 3742 unsigned int md_limit = SHA512_DIGEST_SIZE;
df80bfd3 3743 bool registered = false, gcm_support;
8e8ec596 3744
bf83490e
VM		 3745 /*
3746 * Register crypto algorithms the device supports.
3747 * First, detect presence and attributes of DES, AES, and MD blocks.
3748 */
d239b10d 3749 if (priv->era < 10) {
ae1dd17d 3750 struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon;
df80bfd3 3751 u32 cha_vid, cha_inst, aes_rn;
d239b10d 3752
ae1dd17d 3753 cha_vid = rd_reg32(&perfmon->cha_id_ls);
d239b10d
HG		 3754 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
3755 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3756
ae1dd17d 3757 cha_inst = rd_reg32(&perfmon->cha_num_ls);
d239b10d
HG		 3758 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
3759 CHA_ID_LS_DES_SHIFT;
3760 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
3761 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
d6bbd4ee
HG		 3762 ccha_inst = 0;
3763 ptha_inst = 0;
df80bfd3 3764
ae1dd17d 3765 aes_rn = rd_reg32(&perfmon->cha_rev_ls) & CHA_ID_LS_AES_MASK;
df80bfd3 3766 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
d239b10d 3767 } else {
ae1dd17d 3768 struct version_regs __iomem *vreg = &priv->jr[0]->vreg;
d239b10d
HG		 3769 u32 aesa, mdha;
3770
ae1dd17d
HG		 3771 aesa = rd_reg32(&vreg->aesa);
3772 mdha = rd_reg32(&vreg->mdha);
d239b10d
HG		 3773
3774 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3775 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3776
ae1dd17d 3777 des_inst = rd_reg32(&vreg->desa) & CHA_VER_NUM_MASK;
d239b10d
HG		 3778 aes_inst = aesa & CHA_VER_NUM_MASK;
3779 md_inst = mdha & CHA_VER_NUM_MASK;
ae1dd17d
HG		 3780 ccha_inst = rd_reg32(&vreg->ccha) & CHA_VER_NUM_MASK;
3781 ptha_inst = rd_reg32(&vreg->ptha) & CHA_VER_NUM_MASK;
df80bfd3
HG		 3782
3783 gcm_support = aesa & CHA_VER_MISC_AES_GCM;
d239b10d 3784 }
bf83490e
VM		 3785
3786 /* If MD is present, limit digest size based on LP256 */
d239b10d 3787 if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
bf83490e
VM		 3788 md_limit = SHA256_DIGEST_SIZE;
3789
8e8ec596 3790 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5ca7badb
HG		 3791 struct caam_skcipher_alg *t_alg = driver_algs + i;
3792 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
bf83490e
VM		 3793
3794 /* Skip DES algorithms if not supported by device */
3795 if (!des_inst &&
3796 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3797 (alg_sel == OP_ALG_ALGSEL_DES)))
3798 continue;
3799
3800 /* Skip AES algorithms if not supported by device */
3801 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3802 continue;
eaed71a4 3803
83d2c9a9
SE		 3804 /*
3805 * Check support for AES modes not available
3806 * on LP devices.
3807 */
d239b10d
HG		 3808 if (aes_vid == CHA_VER_VID_AES_LP &&
3809 (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
3810 OP_ALG_AAI_XTS)
3811 continue;
83d2c9a9 3812
5ca7badb 3813 caam_skcipher_alg_init(t_alg);
8e8ec596 3814
623814c0 3815 err = crypto_engine_register_skcipher(&t_alg->skcipher);
8e8ec596 3816 if (err) {
cfc6f11b 3817 pr_warn("%s alg registration failed\n",
623814c0 3818 t_alg->skcipher.base.base.cra_driver_name);
f2147b88
HX		 3819 continue;
3820 }
3821
5ca7badb 3822 t_alg->registered = true;
f2147b88
HX		 3823 registered = true;
3824 }
3825
3826 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3827 struct caam_aead_alg *t_alg = driver_aeads + i;
bf83490e
VM		 3828 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3829 OP_ALG_ALGSEL_MASK;
3830 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3831 OP_ALG_ALGSEL_MASK;
3832 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3833
3834 /* Skip DES algorithms if not supported by device */
3835 if (!des_inst &&
3836 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3837 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3838 continue;
3839
3840 /* Skip AES algorithms if not supported by device */
3841 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3842 continue;
3843
d6bbd4ee
HG		 3844 /* Skip CHACHA20 algorithms if not supported by device */
3845 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
3846 continue;
3847
3848 /* Skip POLY1305 algorithms if not supported by device */
3849 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
3850 continue;
3851
df80bfd3
HG		 3852 /* Skip GCM algorithms if not supported by device */
3853 if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
3854 alg_aai == OP_ALG_AAI_GCM && !gcm_support)
d239b10d 3855 continue;
bf83490e
VM		 3856
3857 /*
3858 * Skip algorithms requiring message digests
3859 * if MD or MD size is not supported by device.
3860 */
2dd3fde4 3861 if (is_mdha(c2_alg_sel) &&
623814c0 3862 (!md_inst || t_alg->aead.base.maxauthsize > md_limit))
d6bbd4ee 3863 continue;
f2147b88
HX		 3864
3865 caam_aead_alg_init(t_alg);
3866
623814c0 3867 err = crypto_engine_register_aead(&t_alg->aead);
f2147b88
HX		 3868 if (err) {
3869 pr_warn("%s alg registration failed\n",
623814c0 3870 t_alg->aead.base.base.cra_driver_name);
f2147b88
HX		 3871 continue;
3872 }
3873
3874 t_alg->registered = true;
3875 registered = true;
8e8ec596 3876 }
f2147b88
HX		 3877
3878 if (registered)
cfc6f11b 3879 pr_info("caam algorithms registered in /proc/crypto\n");
8e8ec596
KP		 3880
3881 return err;
3882}
Linux 6.x block layer and io_uring tree(s)