projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: ccree - avoid implicit enum conversion
[linux-2.6-block.git] / drivers / crypto / chelsio / chcr_algo.c
CommitLineData
324429d7
HS		 1/*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42#define pr_fmt(fmt) "chcr:" fmt
43
44#include <linux/kernel.h>
45#include <linux/module.h>
46#include <linux/crypto.h>
47#include <linux/cryptohash.h>
48#include <linux/skbuff.h>
49#include <linux/rtnetlink.h>
50#include <linux/highmem.h>
51#include <linux/scatterlist.h>
52
53#include <crypto/aes.h>
54#include <crypto/algapi.h>
55#include <crypto/hash.h>
8f6acb7f 56#include <crypto/gcm.h>
324429d7 57#include <crypto/sha.h>
2debd332 58#include <crypto/authenc.h>
b8fd1f41
HJ		 59#include <crypto/ctr.h>
60#include <crypto/gf128mul.h>
2debd332
HJ		 61#include <crypto/internal/aead.h>
62#include <crypto/null.h>
63#include <crypto/internal/skcipher.h>
64#include <crypto/aead.h>
65#include <crypto/scatterwalk.h>
324429d7
HS		 66#include <crypto/internal/hash.h>
67
68#include "t4fw_api.h"
69#include "t4_msg.h"
70#include "chcr_core.h"
71#include "chcr_algo.h"
72#include "chcr_crypto.h"
73
2f47d580
HJ		 74#define IV AES_BLOCK_SIZE
75
8579e076
CIK		 76static unsigned int sgl_ent_len[] = {
77 0, 0, 16, 24, 40, 48, 64, 72, 88,
78 96, 112, 120, 136, 144, 160, 168, 184,
79 192, 208, 216, 232, 240, 256, 264, 280,
80 288, 304, 312, 328, 336, 352, 360, 376
81};
6dad4e8a 82
8579e076
CIK		 83static unsigned int dsgl_ent_len[] = {
84 0, 32, 32, 48, 48, 64, 64, 80, 80,
85 112, 112, 128, 128, 144, 144, 160, 160,
86 192, 192, 208, 208, 224, 224, 240, 240,
87 272, 272, 288, 288, 304, 304, 320, 320
88};
6dad4e8a
AG		 89
90static u32 round_constant[11] = {
91 0x01000000, 0x02000000, 0x04000000, 0x08000000,
92 0x10000000, 0x20000000, 0x40000000, 0x80000000,
93 0x1B000000, 0x36000000, 0x6C000000
94};
95
96static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
97 unsigned char *input, int err);
98
2debd332
HJ		 99static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
100{
101 return ctx->crypto_ctx->aeadctx;
102}
103
324429d7
HS		 104static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
105{
106 return ctx->crypto_ctx->ablkctx;
107}
108
109static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
110{
111 return ctx->crypto_ctx->hmacctx;
112}
113
2debd332
HJ		 114static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
115{
116 return gctx->ctx->gcm;
117}
118
119static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
120{
121 return gctx->ctx->authenc;
122}
123
324429d7
HS		 124static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
125{
126 return ctx->dev->u_ctx;
127}
128
129static inline int is_ofld_imm(const struct sk_buff *skb)
130{
2f47d580 131 return (skb->len <= SGE_MAX_WR_LEN);
324429d7
HS		 132}
133
5110e655
HJ		 134static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
135{
136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
137}
138
2f47d580
HJ		 139static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
140 unsigned int entlen,
141 unsigned int skip)
2956f36c
HJ		 142{
143 int nents = 0;
144 unsigned int less;
2f47d580 145 unsigned int skip_len = 0;
2956f36c 146
2f47d580
HJ		 147 while (sg && skip) {
148 if (sg_dma_len(sg) <= skip) {
149 skip -= sg_dma_len(sg);
150 skip_len = 0;
151 sg = sg_next(sg);
152 } else {
153 skip_len = skip;
154 skip = 0;
155 }
2956f36c
HJ		 156 }
157
2f47d580
HJ		 158 while (sg && reqlen) {
159 less = min(reqlen, sg_dma_len(sg) - skip_len);
160 nents += DIV_ROUND_UP(less, entlen);
161 reqlen -= less;
162 skip_len = 0;
163 sg = sg_next(sg);
164 }
2956f36c
HJ		 165 return nents;
166}
167
6dad4e8a 168static inline int get_aead_subtype(struct crypto_aead *aead)
2f47d580 169{
6dad4e8a
AG		 170 struct aead_alg *alg = crypto_aead_alg(aead);
171 struct chcr_alg_template *chcr_crypto_alg =
172 container_of(alg, struct chcr_alg_template, alg.aead);
173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
2f47d580 174}
2f47d580 175
6dad4e8a 176void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
2debd332
HJ		 177{
178 u8 temp[SHA512_DIGEST_SIZE];
179 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
180 int authsize = crypto_aead_authsize(tfm);
181 struct cpl_fw6_pld *fw6_pld;
182 int cmp = 0;
183
184 fw6_pld = (struct cpl_fw6_pld *)input;
185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
d600fc8a 187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
2debd332
HJ		 188 } else {
189
190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
191 authsize, req->assoclen +
192 req->cryptlen - authsize);
d600fc8a 193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
2debd332
HJ		 194 }
195 if (cmp)
196 *err = -EBADMSG;
197 else
198 *err = 0;
199}
200
6dad4e8a
AG		 201static inline void chcr_handle_aead_resp(struct aead_request *req,
202 unsigned char *input,
203 int err)
204{
205 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
6dad4e8a 206
4262c98a 207 chcr_aead_common_exit(req);
6dad4e8a
AG		 208 if (reqctx->verify == VERIFY_SW) {
209 chcr_verify_tag(req, input, &err);
210 reqctx->verify = VERIFY_HW;
211 }
212 req->base.complete(&req->base, err);
213}
214
2f47d580 215static void get_aes_decrypt_key(unsigned char *dec_key,
39f91a34
HJ		 216 const unsigned char *key,
217 unsigned int keylength)
218{
219 u32 temp;
220 u32 w_ring[MAX_NK];
221 int i, j, k;
222 u8 nr, nk;
223
224 switch (keylength) {
225 case AES_KEYLENGTH_128BIT:
226 nk = KEYLENGTH_4BYTES;
227 nr = NUMBER_OF_ROUNDS_10;
228 break;
229 case AES_KEYLENGTH_192BIT:
230 nk = KEYLENGTH_6BYTES;
231 nr = NUMBER_OF_ROUNDS_12;
232 break;
233 case AES_KEYLENGTH_256BIT:
234 nk = KEYLENGTH_8BYTES;
235 nr = NUMBER_OF_ROUNDS_14;
236 break;
237 default:
238 return;
239 }
240 for (i = 0; i < nk; i++)
241 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
242
243 i = 0;
244 temp = w_ring[nk - 1];
245 while (i + nk < (nr + 1) * 4) {
246 if (!(i % nk)) {
247 /* RotWord(temp) */
248 temp = (temp << 8) | (temp >> 24);
249 temp = aes_ks_subword(temp);
250 temp ^= round_constant[i / nk];
251 } else if (nk == 8 && (i % 4 == 0)) {
252 temp = aes_ks_subword(temp);
253 }
254 w_ring[i % nk] ^= temp;
255 temp = w_ring[i % nk];
256 i++;
257 }
258 i--;
259 for (k = 0, j = i % nk; k < nk; k++) {
260 *((u32 *)dec_key + k) = htonl(w_ring[j]);
261 j--;
262 if (j < 0)
263 j += nk;
264 }
265}
266
e7922729 267static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
324429d7 268{
ec1bca94 269 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
324429d7
HS		 270
271 switch (ds) {
272 case SHA1_DIGEST_SIZE:
e7922729 273 base_hash = crypto_alloc_shash("sha1", 0, 0);
324429d7
HS		 274 break;
275 case SHA224_DIGEST_SIZE:
e7922729 276 base_hash = crypto_alloc_shash("sha224", 0, 0);
324429d7
HS		 277 break;
278 case SHA256_DIGEST_SIZE:
e7922729 279 base_hash = crypto_alloc_shash("sha256", 0, 0);
324429d7
HS		 280 break;
281 case SHA384_DIGEST_SIZE:
e7922729 282 base_hash = crypto_alloc_shash("sha384", 0, 0);
324429d7
HS		 283 break;
284 case SHA512_DIGEST_SIZE:
e7922729 285 base_hash = crypto_alloc_shash("sha512", 0, 0);
324429d7
HS		 286 break;
287 }
324429d7 288
e7922729 289 return base_hash;
324429d7
HS		 290}
291
292static int chcr_compute_partial_hash(struct shash_desc *desc,
293 char *iopad, char *result_hash,
294 int digest_size)
295{
296 struct sha1_state sha1_st;
297 struct sha256_state sha256_st;
298 struct sha512_state sha512_st;
299 int error;
300
301 if (digest_size == SHA1_DIGEST_SIZE) {
302 error = crypto_shash_init(desc) ?:
303 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
304 crypto_shash_export(desc, (void *)&sha1_st);
305 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
306 } else if (digest_size == SHA224_DIGEST_SIZE) {
307 error = crypto_shash_init(desc) ?:
308 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
309 crypto_shash_export(desc, (void *)&sha256_st);
310 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
311
312 } else if (digest_size == SHA256_DIGEST_SIZE) {
313 error = crypto_shash_init(desc) ?:
314 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
315 crypto_shash_export(desc, (void *)&sha256_st);
316 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
317
318 } else if (digest_size == SHA384_DIGEST_SIZE) {
319 error = crypto_shash_init(desc) ?:
320 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
321 crypto_shash_export(desc, (void *)&sha512_st);
322 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
323
324 } else if (digest_size == SHA512_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha512_st);
328 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
329 } else {
330 error = -EINVAL;
331 pr_err("Unknown digest size %d\n", digest_size);
332 }
333 return error;
334}
335
336static void chcr_change_order(char *buf, int ds)
337{
338 int i;
339
340 if (ds == SHA512_DIGEST_SIZE) {
341 for (i = 0; i < (ds / sizeof(u64)); i++)
342 *((__be64 *)buf + i) =
343 cpu_to_be64(*((u64 *)buf + i));
344 } else {
345 for (i = 0; i < (ds / sizeof(u32)); i++)
346 *((__be32 *)buf + i) =
347 cpu_to_be32(*((u32 *)buf + i));
348 }
349}
350
351static inline int is_hmac(struct crypto_tfm *tfm)
352{
353 struct crypto_alg *alg = tfm->__crt_alg;
354 struct chcr_alg_template *chcr_crypto_alg =
355 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
356 alg.hash);
5c86a8ff 357 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
324429d7
HS		 358 return 1;
359 return 0;
360}
361
2f47d580
HJ		 362static inline void dsgl_walk_init(struct dsgl_walk *walk,
363 struct cpl_rx_phys_dsgl *dsgl)
324429d7 364{
2f47d580
HJ		 365 walk->dsgl = dsgl;
366 walk->nents = 0;
367 walk->to = (struct phys_sge_pairs *)(dsgl + 1);
368}
369
add92a81
HJ		 370static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
371 int pci_chan_id)
2f47d580
HJ		 372{
373 struct cpl_rx_phys_dsgl *phys_cpl;
374
375 phys_cpl = walk->dsgl;
324429d7
HS		 376
377 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
378 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
2f47d580
HJ		 379 phys_cpl->pcirlxorder_to_noofsgentr =
380 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
381 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
382 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
383 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
384 CPL_RX_PHYS_DSGL_DCAID_V(0) |
385 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
386 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
387 phys_cpl->rss_hdr_int.qid = htons(qid);
388 phys_cpl->rss_hdr_int.hash_val = 0;
add92a81 389 phys_cpl->rss_hdr_int.channel = pci_chan_id;
2f47d580
HJ		 390}
391
392static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
393 size_t size,
394 dma_addr_t *addr)
395{
396 int j;
397
398 if (!size)
399 return;
400 j = walk->nents;
401 walk->to->len[j % 8] = htons(size);
402 walk->to->addr[j % 8] = cpu_to_be64(*addr);
403 j++;
404 if ((j % 8) == 0)
405 walk->to++;
406 walk->nents = j;
407}
408
409static void dsgl_walk_add_sg(struct dsgl_walk *walk,
410 struct scatterlist *sg,
411 unsigned int slen,
412 unsigned int skip)
413{
414 int skip_len = 0;
415 unsigned int left_size = slen, len = 0;
416 unsigned int j = walk->nents;
417 int offset, ent_len;
418
419 if (!slen)
420 return;
421 while (sg && skip) {
422 if (sg_dma_len(sg) <= skip) {
423 skip -= sg_dma_len(sg);
424 skip_len = 0;
425 sg = sg_next(sg);
426 } else {
427 skip_len = skip;
428 skip = 0;
429 }
430 }
431
2956f36c 432 while (left_size && sg) {
2f47d580 433 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
2956f36c
HJ		 434 offset = 0;
435 while (len) {
2f47d580
HJ		 436 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
437 walk->to->len[j % 8] = htons(ent_len);
438 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
439 offset + skip_len);
2956f36c
HJ		 440 offset += ent_len;
441 len -= ent_len;
442 j++;
443 if ((j % 8) == 0)
2f47d580 444 walk->to++;
2956f36c 445 }
2f47d580
HJ		 446 walk->last_sg = sg;
447 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
448 skip_len) + skip_len;
449 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
450 skip_len = 0;
2956f36c
HJ		 451 sg = sg_next(sg);
452 }
2f47d580
HJ		 453 walk->nents = j;
454}
455
456static inline void ulptx_walk_init(struct ulptx_walk *walk,
457 struct ulptx_sgl *ulp)
458{
459 walk->sgl = ulp;
460 walk->nents = 0;
461 walk->pair_idx = 0;
462 walk->pair = ulp->sge;
463 walk->last_sg = NULL;
464 walk->last_sg_len = 0;
465}
466
467static inline void ulptx_walk_end(struct ulptx_walk *walk)
468{
469 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
470 ULPTX_NSGE_V(walk->nents));
471}
2956f36c 472
2f47d580
HJ		 473
474static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
475 size_t size,
476 dma_addr_t *addr)
477{
478 if (!size)
479 return;
480
481 if (walk->nents == 0) {
482 walk->sgl->len0 = cpu_to_be32(size);
483 walk->sgl->addr0 = cpu_to_be64(*addr);
484 } else {
485 walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr);
486 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
487 walk->pair_idx = !walk->pair_idx;
488 if (!walk->pair_idx)
489 walk->pair++;
490 }
491 walk->nents++;
324429d7
HS		 492}
493
2f47d580 494static void ulptx_walk_add_sg(struct ulptx_walk *walk,
adf1ca61 495 struct scatterlist *sg,
2f47d580
HJ		 496 unsigned int len,
497 unsigned int skip)
324429d7 498{
2f47d580
HJ		 499 int small;
500 int skip_len = 0;
501 unsigned int sgmin;
324429d7 502
2f47d580
HJ		 503 if (!len)
504 return;
2f47d580
HJ		 505 while (sg && skip) {
506 if (sg_dma_len(sg) <= skip) {
507 skip -= sg_dma_len(sg);
508 skip_len = 0;
509 sg = sg_next(sg);
510 } else {
511 skip_len = skip;
512 skip = 0;
513 }
514 }
8daa32b9
HJ		 515 WARN(!sg, "SG should not be null here\n");
516 if (sg && (walk->nents == 0)) {
2f47d580
HJ		 517 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
518 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
519 walk->sgl->len0 = cpu_to_be32(sgmin);
520 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
521 walk->nents++;
522 len -= sgmin;
523 walk->last_sg = sg;
524 walk->last_sg_len = sgmin + skip_len;
525 skip_len += sgmin;
526 if (sg_dma_len(sg) == skip_len) {
527 sg = sg_next(sg);
528 skip_len = 0;
529 }
530 }
531
532 while (sg && len) {
533 small = min(sg_dma_len(sg) - skip_len, len);
534 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
535 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
536 walk->pair->addr[walk->pair_idx] =
537 cpu_to_be64(sg_dma_address(sg) + skip_len);
538 walk->pair_idx = !walk->pair_idx;
539 walk->nents++;
540 if (!walk->pair_idx)
541 walk->pair++;
542 len -= sgmin;
543 skip_len += sgmin;
544 walk->last_sg = sg;
545 walk->last_sg_len = skip_len;
546 if (sg_dma_len(sg) == skip_len) {
547 sg = sg_next(sg);
548 skip_len = 0;
549 }
324429d7 550 }
324429d7
HS		 551}
552
553static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
554{
555 struct crypto_alg *alg = tfm->__crt_alg;
556 struct chcr_alg_template *chcr_crypto_alg =
557 container_of(alg, struct chcr_alg_template, alg.crypto);
558
559 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
560}
561
b8fd1f41
HJ		 562static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
563{
564 struct adapter *adap = netdev2adap(dev);
565 struct sge_uld_txq_info *txq_info =
566 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
567 struct sge_uld_txq *txq;
568 int ret = 0;
569
570 local_bh_disable();
571 txq = &txq_info->uldtxq[idx];
572 spin_lock(&txq->sendq.lock);
573 if (txq->full)
574 ret = -1;
575 spin_unlock(&txq->sendq.lock);
576 local_bh_enable();
577 return ret;
578}
579
324429d7
HS		 580static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
581 struct _key_ctx *key_ctx)
582{
583 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
cc1b156d 584 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
324429d7
HS		 585 } else {
586 memcpy(key_ctx->key,
587 ablkctx->key + (ablkctx->enckey_len >> 1),
588 ablkctx->enckey_len >> 1);
cc1b156d
HJ		 589 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
590 ablkctx->rrkey, ablkctx->enckey_len >> 1);
324429d7
HS		 591 }
592 return 0;
593}
5110e655
HJ		 594
595static int chcr_hash_ent_in_wr(struct scatterlist *src,
596 unsigned int minsg,
597 unsigned int space,
598 unsigned int srcskip)
599{
600 int srclen = 0;
601 int srcsg = minsg;
602 int soffset = 0, sless;
603
604 if (sg_dma_len(src) == srcskip) {
605 src = sg_next(src);
606 srcskip = 0;
607 }
608 while (src && space > (sgl_ent_len[srcsg + 1])) {
609 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
610 CHCR_SRC_SG_SIZE);
611 srclen += sless;
612 soffset += sless;
613 srcsg++;
614 if (sg_dma_len(src) == (soffset + srcskip)) {
615 src = sg_next(src);
616 soffset = 0;
617 srcskip = 0;
618 }
619 }
620 return srclen;
621}
622
b8fd1f41
HJ		 623static int chcr_sg_ent_in_wr(struct scatterlist *src,
624 struct scatterlist *dst,
625 unsigned int minsg,
2f47d580
HJ		 626 unsigned int space,
627 unsigned int srcskip,
628 unsigned int dstskip)
b8fd1f41
HJ		 629{
630 int srclen = 0, dstlen = 0;
2f47d580 631 int srcsg = minsg, dstsg = minsg;
1d693cf6 632 int offset = 0, soffset = 0, less, sless = 0;
b8fd1f41 633
2f47d580
HJ		 634 if (sg_dma_len(src) == srcskip) {
635 src = sg_next(src);
636 srcskip = 0;
637 }
2f47d580
HJ		 638 if (sg_dma_len(dst) == dstskip) {
639 dst = sg_next(dst);
640 dstskip = 0;
641 }
642
643 while (src && dst &&
b8fd1f41 644 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
1d693cf6
HJ		 645 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
646 CHCR_SRC_SG_SIZE);
647 srclen += sless;
b8fd1f41 648 srcsg++;
2956f36c 649 offset = 0;
b8fd1f41
HJ		 650 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
651 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
652 if (srclen <= dstlen)
653 break;
2f47d580 654 less = min_t(unsigned int, sg_dma_len(dst) - offset -
db6deea4 655 dstskip, CHCR_DST_SG_SIZE);
2956f36c
HJ		 656 dstlen += less;
657 offset += less;
1d693cf6 658 if ((offset + dstskip) == sg_dma_len(dst)) {
2956f36c
HJ		 659 dst = sg_next(dst);
660 offset = 0;
661 }
b8fd1f41 662 dstsg++;
2f47d580 663 dstskip = 0;
b8fd1f41 664 }
1d693cf6
HJ		 665 soffset += sless;
666 if ((soffset + srcskip) == sg_dma_len(src)) {
667 src = sg_next(src);
668 srcskip = 0;
669 soffset = 0;
670 }
671
b8fd1f41 672 }
b8fd1f41
HJ		 673 return min(srclen, dstlen);
674}
675
28874f26 676static int chcr_cipher_fallback(struct crypto_sync_skcipher *cipher,
b8fd1f41
HJ		 677 u32 flags,
678 struct scatterlist *src,
679 struct scatterlist *dst,
680 unsigned int nbytes,
681 u8 *iv,
682 unsigned short op_type)
683{
684 int err;
685
28874f26 686 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
6faa0f57 687
28874f26 688 skcipher_request_set_sync_tfm(subreq, cipher);
b8fd1f41
HJ		 689 skcipher_request_set_callback(subreq, flags, NULL, NULL);
690 skcipher_request_set_crypt(subreq, src, dst,
691 nbytes, iv);
692
693 err = op_type ? crypto_skcipher_decrypt(subreq) :
694 crypto_skcipher_encrypt(subreq);
695 skcipher_request_zero(subreq);
696
697 return err;
324429d7 698
b8fd1f41 699}
324429d7 700static inline void create_wreq(struct chcr_context *ctx,
358961d1 701 struct chcr_wr *chcr_req,
2f47d580
HJ		 702 struct crypto_async_request *req,
703 unsigned int imm,
570265bf 704 int hash_sz,
2f47d580 705 unsigned int len16,
2512a624
HJ		 706 unsigned int sc_len,
707 unsigned int lcb)
324429d7
HS		 708{
709 struct uld_ctx *u_ctx = ULD_CTX(ctx);
72a56ca9 710 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
324429d7 711
324429d7 712
570265bf 713 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
358961d1 714 chcr_req->wreq.pld_size_hash_size =
570265bf 715 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
358961d1 716 chcr_req->wreq.len16_pkd =
2f47d580 717 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
358961d1
HJ		 718 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
719 chcr_req->wreq.rx_chid_to_rx_q_id =
8a13449f 720 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
570265bf 721 !!lcb, ctx->tx_qidx);
324429d7 722
add92a81 723 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->tx_chan_id,
8a13449f 724 qid);
2f47d580
HJ		 725 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
726 ((sizeof(chcr_req->wreq)) >> 4)));
324429d7 727
2f47d580 728 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
358961d1 729 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
2f47d580 730 sizeof(chcr_req->key_ctx) + sc_len);
324429d7
HS		 731}
732
733/**
734 * create_cipher_wr - form the WR for cipher operations
735 * @req: cipher req.
736 * @ctx: crypto driver context of the request.
737 * @qid: ingress qid where response of this WR should be received.
738 * @op_type: encryption or decryption
739 */
b8fd1f41 740static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
324429d7 741{
b8fd1f41 742 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
2f47d580 743 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
324429d7 744 struct sk_buff *skb = NULL;
358961d1 745 struct chcr_wr *chcr_req;
324429d7 746 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580 747 struct ulptx_sgl *ulptx;
b8fd1f41
HJ		 748 struct chcr_blkcipher_req_ctx *reqctx =
749 ablkcipher_request_ctx(wrparam->req);
2f47d580 750 unsigned int temp = 0, transhdr_len, dst_size;
b8fd1f41 751 int error;
2956f36c 752 int nents;
2f47d580 753 unsigned int kctx_len;
b8fd1f41
HJ		 754 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
755 GFP_KERNEL : GFP_ATOMIC;
2f47d580 756 struct adapter *adap = padap(c_ctx(tfm)->dev);
324429d7 757
2f47d580
HJ		 758 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
759 reqctx->dst_ofst);
335bcc4a 760 dst_size = get_space_for_phys_dsgl(nents);
125d01ca 761 kctx_len = roundup(ablkctx->enckey_len, 16);
2f47d580
HJ		 762 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
763 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
764 CHCR_SRC_SG_SIZE, reqctx->src_ofst);
335bcc4a
HJ		 765 temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
766 (sgl_len(nents) * 8);
2f47d580 767 transhdr_len += temp;
125d01ca 768 transhdr_len = roundup(transhdr_len, 16);
2f47d580 769 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
b8fd1f41
HJ		 770 if (!skb) {
771 error = -ENOMEM;
772 goto err;
773 }
de77b966 774 chcr_req = __skb_put_zero(skb, transhdr_len);
358961d1 775 chcr_req->sec_cpl.op_ivinsrtofst =
2f47d580 776 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1);
358961d1 777
2f47d580 778 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
358961d1 779 chcr_req->sec_cpl.aadstart_cipherstop_hi =
2f47d580 780 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
358961d1
HJ		 781
782 chcr_req->sec_cpl.cipherstop_lo_authinsert =
783 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
b8fd1f41 784 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
324429d7 785 ablkctx->ciph_mode,
2f47d580 786 0, 0, IV >> 1);
358961d1 787 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
335bcc4a 788 0, 1, dst_size);
324429d7 789
358961d1 790 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
b8fd1f41
HJ		 791 if ((reqctx->op == CHCR_DECRYPT_OP) &&
792 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
793 CRYPTO_ALG_SUB_TYPE_CTR)) &&
794 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
795 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
358961d1 796 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
324429d7 797 } else {
b8fd1f41
HJ		 798 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
799 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
358961d1
HJ		 800 memcpy(chcr_req->key_ctx.key, ablkctx->key,
801 ablkctx->enckey_len);
324429d7 802 } else {
358961d1 803 memcpy(chcr_req->key_ctx.key, ablkctx->key +
324429d7
HS		 804 (ablkctx->enckey_len >> 1),
805 ablkctx->enckey_len >> 1);
358961d1 806 memcpy(chcr_req->key_ctx.key +
324429d7
HS		 807 (ablkctx->enckey_len >> 1),
808 ablkctx->key,
809 ablkctx->enckey_len >> 1);
810 }
811 }
358961d1 812 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2f47d580
HJ		 813 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
814 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
815 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
324429d7 816
ee0863ba 817 atomic_inc(&adap->chcr_stats.cipher_rqst);
335bcc4a
HJ		 818 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
819 + (reqctx->imm ? (wrparam->bytes) : 0);
2f47d580
HJ		 820 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
821 transhdr_len, temp,
2512a624 822 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
5c86a8ff 823 reqctx->skb = skb;
5fb78dba
HJ		 824
825 if (reqctx->op && (ablkctx->ciph_mode ==
826 CHCR_SCMD_CIPHER_MODE_AES_CBC))
827 sg_pcopy_to_buffer(wrparam->req->src,
828 sg_nents(wrparam->req->src), wrparam->req->info, 16,
829 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
830
324429d7 831 return skb;
b8fd1f41
HJ		 832err:
833 return ERR_PTR(error);
834}
835
836static inline int chcr_keyctx_ck_size(unsigned int keylen)
837{
838 int ck_size = 0;
839
840 if (keylen == AES_KEYSIZE_128)
841 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
842 else if (keylen == AES_KEYSIZE_192)
843 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
844 else if (keylen == AES_KEYSIZE_256)
845 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
846 else
847 ck_size = 0;
848
849 return ck_size;
850}
851static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
852 const u8 *key,
853 unsigned int keylen)
854{
855 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
2f47d580 856 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41
HJ		 857 int err = 0;
858
28874f26
KC		 859 crypto_sync_skcipher_clear_flags(ablkctx->sw_cipher,
860 CRYPTO_TFM_REQ_MASK);
861 crypto_sync_skcipher_set_flags(ablkctx->sw_cipher,
862 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
863 err = crypto_sync_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
b8fd1f41
HJ		 864 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
865 tfm->crt_flags |=
28874f26 866 crypto_sync_skcipher_get_flags(ablkctx->sw_cipher) &
b8fd1f41
HJ		 867 CRYPTO_TFM_RES_MASK;
868 return err;
324429d7
HS		 869}
870
b8fd1f41
HJ		 871static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
872 const u8 *key,
324429d7
HS		 873 unsigned int keylen)
874{
2f47d580 875 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
324429d7
HS		 876 unsigned int ck_size, context_size;
877 u16 alignment = 0;
b8fd1f41 878 int err;
324429d7 879
b8fd1f41
HJ		 880 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
881 if (err)
324429d7 882 goto badkey_err;
b8fd1f41
HJ		 883
884 ck_size = chcr_keyctx_ck_size(keylen);
885 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
cc1b156d
HJ		 886 memcpy(ablkctx->key, key, keylen);
887 ablkctx->enckey_len = keylen;
888 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
324429d7
HS		 889 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
890 keylen + alignment) >> 4;
891
892 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
893 0, 0, context_size);
894 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
895 return 0;
896badkey_err:
b8fd1f41 897 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
324429d7 898 ablkctx->enckey_len = 0;
b8fd1f41
HJ		 899
900 return err;
324429d7
HS		 901}
902
b8fd1f41
HJ		 903static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
904 const u8 *key,
905 unsigned int keylen)
324429d7 906{
2f47d580 907 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41
HJ		 908 unsigned int ck_size, context_size;
909 u16 alignment = 0;
910 int err;
911
912 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
913 if (err)
914 goto badkey_err;
915 ck_size = chcr_keyctx_ck_size(keylen);
916 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
917 memcpy(ablkctx->key, key, keylen);
918 ablkctx->enckey_len = keylen;
919 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
920 keylen + alignment) >> 4;
921
922 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
923 0, 0, context_size);
924 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
925
926 return 0;
927badkey_err:
928 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
929 ablkctx->enckey_len = 0;
930
931 return err;
932}
933
934static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
935 const u8 *key,
936 unsigned int keylen)
937{
2f47d580 938 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41
HJ		 939 unsigned int ck_size, context_size;
940 u16 alignment = 0;
941 int err;
942
943 if (keylen < CTR_RFC3686_NONCE_SIZE)
944 return -EINVAL;
945 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
946 CTR_RFC3686_NONCE_SIZE);
947
948 keylen -= CTR_RFC3686_NONCE_SIZE;
949 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
950 if (err)
951 goto badkey_err;
952
953 ck_size = chcr_keyctx_ck_size(keylen);
954 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
955 memcpy(ablkctx->key, key, keylen);
956 ablkctx->enckey_len = keylen;
957 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
958 keylen + alignment) >> 4;
959
960 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
961 0, 0, context_size);
962 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
963
964 return 0;
965badkey_err:
966 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
967 ablkctx->enckey_len = 0;
968
969 return err;
970}
971static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
972{
973 unsigned int size = AES_BLOCK_SIZE;
974 __be32 *b = (__be32 *)(dstiv + size);
975 u32 c, prev;
976
977 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
978 for (; size >= 4; size -= 4) {
979 prev = be32_to_cpu(*--b);
980 c = prev + add;
981 *b = cpu_to_be32(c);
982 if (prev < c)
983 break;
984 add = 1;
985 }
986
987}
988
989static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
990{
991 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
992 u64 c;
993 u32 temp = be32_to_cpu(*--b);
994
995 temp = ~temp;
996 c = (u64)temp + 1; // No of block can processed withou overflow
997 if ((bytes / AES_BLOCK_SIZE) > c)
998 bytes = c * AES_BLOCK_SIZE;
999 return bytes;
1000}
1001
209897d5
HJ		 1002static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
1003 u32 isfinal)
b8fd1f41
HJ		 1004{
1005 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
2f47d580 1006 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
b8fd1f41
HJ		 1007 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1008 struct crypto_cipher *cipher;
1009 int ret, i;
1010 u8 *key;
1011 unsigned int keylen;
de1a00ac
HJ		 1012 int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1013 int round8 = round / 8;
b8fd1f41 1014
d3f1d2f7 1015 cipher = ablkctx->aes_generic;
de1a00ac 1016 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
b8fd1f41 1017
b8fd1f41
HJ		 1018 keylen = ablkctx->enckey_len / 2;
1019 key = ablkctx->key + keylen;
1020 ret = crypto_cipher_setkey(cipher, key, keylen);
1021 if (ret)
d3f1d2f7 1022 goto out;
335bcc4a 1023 crypto_cipher_encrypt_one(cipher, iv, iv);
de1a00ac
HJ		 1024 for (i = 0; i < round8; i++)
1025 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1026
1027 for (i = 0; i < (round % 8); i++)
b8fd1f41
HJ		 1028 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1029
209897d5
HJ		 1030 if (!isfinal)
1031 crypto_cipher_decrypt_one(cipher, iv, iv);
b8fd1f41
HJ		 1032out:
1033 return ret;
1034}
1035
1036static int chcr_update_cipher_iv(struct ablkcipher_request *req,
1037 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1038{
1039 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1040 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1041 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
ab677ff4 1042 int ret = 0;
324429d7 1043
b8fd1f41
HJ		 1044 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1045 ctr_add_iv(iv, req->info, (reqctx->processed /
1046 AES_BLOCK_SIZE));
1047 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1048 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1049 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1050 AES_BLOCK_SIZE) + 1);
1051 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
209897d5 1052 ret = chcr_update_tweak(req, iv, 0);
b8fd1f41
HJ		 1053 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1054 if (reqctx->op)
5fb78dba
HJ		 1055 /*Updated before sending last WR*/
1056 memcpy(iv, req->info, AES_BLOCK_SIZE);
b8fd1f41
HJ		 1057 else
1058 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1059 }
1060
324429d7 1061 return ret;
b8fd1f41 1062
324429d7
HS		 1063}
1064
b8fd1f41
HJ		 1065/* We need separate function for final iv because in rfc3686 Initial counter
1066 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1067 * for subsequent update requests
1068 */
1069
1070static int chcr_final_cipher_iv(struct ablkcipher_request *req,
1071 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1072{
1073 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1074 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1075 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
1076 int ret = 0;
1077
1078 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1079 ctr_add_iv(iv, req->info, (reqctx->processed /
1080 AES_BLOCK_SIZE));
1081 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
209897d5 1082 ret = chcr_update_tweak(req, iv, 1);
b8fd1f41 1083 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
5fb78dba
HJ		 1084 /*Already updated for Decrypt*/
1085 if (!reqctx->op)
b8fd1f41
HJ		 1086 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1087
1088 }
1089 return ret;
1090
1091}
1092
b8fd1f41
HJ		 1093static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
1094 unsigned char *input, int err)
324429d7
HS		 1095{
1096 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
2f47d580
HJ		 1097 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1098 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
324429d7 1099 struct sk_buff *skb;
b8fd1f41
HJ		 1100 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1101 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1102 struct cipher_wr_param wrparam;
1103 int bytes;
1104
b8fd1f41 1105 if (err)
2f47d580 1106 goto unmap;
b8fd1f41 1107 if (req->nbytes == reqctx->processed) {
2f47d580
HJ		 1108 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1109 req);
b8fd1f41
HJ		 1110 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1111 goto complete;
1112 }
1113
2f47d580 1114 if (!reqctx->imm) {
335bcc4a 1115 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
5110e655 1116 CIP_SPACE_LEFT(ablkctx->enckey_len),
2f47d580 1117 reqctx->src_ofst, reqctx->dst_ofst);
db6deea4
HJ		 1118 if ((bytes + reqctx->processed) >= req->nbytes)
1119 bytes = req->nbytes - reqctx->processed;
1120 else
125d01ca 1121 bytes = rounddown(bytes, 16);
2f47d580
HJ		 1122 } else {
1123 /*CTR mode counter overfloa*/
1124 bytes = req->nbytes - reqctx->processed;
1125 }
b8fd1f41
HJ		 1126 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1127 if (err)
2f47d580 1128 goto unmap;
b8fd1f41
HJ		 1129
1130 if (unlikely(bytes == 0)) {
2f47d580
HJ		 1131 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1132 req);
b8fd1f41
HJ		 1133 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1134 req->base.flags,
2f47d580
HJ		 1135 req->src,
1136 req->dst,
1137 req->nbytes,
1138 req->info,
b8fd1f41
HJ		 1139 reqctx->op);
1140 goto complete;
1141 }
1142
1143 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1144 CRYPTO_ALG_SUB_TYPE_CTR)
1145 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
2f47d580 1146 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx];
b8fd1f41
HJ		 1147 wrparam.req = req;
1148 wrparam.bytes = bytes;
1149 skb = create_cipher_wr(&wrparam);
1150 if (IS_ERR(skb)) {
1151 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1152 err = PTR_ERR(skb);
2f47d580 1153 goto unmap;
b8fd1f41
HJ		 1154 }
1155 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1156 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
b8fd1f41 1157 chcr_send_wr(skb);
2f47d580
HJ		 1158 reqctx->last_req_len = bytes;
1159 reqctx->processed += bytes;
b8fd1f41 1160 return 0;
2f47d580
HJ		 1161unmap:
1162 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
b8fd1f41
HJ		 1163complete:
1164 req->base.complete(&req->base, err);
1165 return err;
1166}
1167
1168static int process_cipher(struct ablkcipher_request *req,
1169 unsigned short qid,
1170 struct sk_buff **skb,
1171 unsigned short op_type)
1172{
1173 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1174 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1175 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2f47d580 1176 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
b8fd1f41 1177 struct cipher_wr_param wrparam;
2956f36c 1178 int bytes, err = -EINVAL;
b8fd1f41 1179
b8fd1f41
HJ		 1180 reqctx->processed = 0;
1181 if (!req->info)
1182 goto error;
1183 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1184 (req->nbytes == 0) ||
1185 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1186 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1187 ablkctx->enckey_len, req->nbytes, ivsize);
1188 goto error;
1189 }
2f47d580
HJ		 1190 chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1191 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1192 AES_MIN_KEY_SIZE +
1193 sizeof(struct cpl_rx_phys_dsgl) +
1194 /*Min dsgl size*/
1195 32))) {
1196 /* Can be sent as Imm*/
1197 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1198
1199 dnents = sg_nents_xlen(req->dst, req->nbytes,
1200 CHCR_DST_SG_SIZE, 0);
2f47d580 1201 phys_dsgl = get_space_for_phys_dsgl(dnents);
125d01ca 1202 kctx_len = roundup(ablkctx->enckey_len, 16);
2f47d580
HJ		 1203 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1204 reqctx->imm = (transhdr_len + IV + req->nbytes) <=
1205 SGE_MAX_WR_LEN;
1206 bytes = IV + req->nbytes;
1207
1208 } else {
1209 reqctx->imm = 0;
1210 }
1211
1212 if (!reqctx->imm) {
335bcc4a 1213 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
5110e655 1214 CIP_SPACE_LEFT(ablkctx->enckey_len),
2f47d580 1215 0, 0);
db6deea4
HJ		 1216 if ((bytes + reqctx->processed) >= req->nbytes)
1217 bytes = req->nbytes - reqctx->processed;
1218 else
125d01ca 1219 bytes = rounddown(bytes, 16);
2f47d580 1220 } else {
b8fd1f41 1221 bytes = req->nbytes;
2f47d580 1222 }
b8fd1f41 1223 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
db6deea4 1224 CRYPTO_ALG_SUB_TYPE_CTR) {
b8fd1f41
HJ		 1225 bytes = adjust_ctr_overflow(req->info, bytes);
1226 }
1227 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1228 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1229 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1230 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1231 CTR_RFC3686_IV_SIZE);
1232
1233 /* initialize counter portion of counter block */
1234 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1235 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1236
1237 } else {
1238
2f47d580 1239 memcpy(reqctx->iv, req->info, IV);
b8fd1f41
HJ		 1240 }
1241 if (unlikely(bytes == 0)) {
2f47d580
HJ		 1242 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1243 req);
b8fd1f41
HJ		 1244 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1245 req->base.flags,
1246 req->src,
1247 req->dst,
1248 req->nbytes,
7ffb9118 1249 reqctx->iv,
b8fd1f41
HJ		 1250 op_type);
1251 goto error;
1252 }
b8fd1f41 1253 reqctx->op = op_type;
2f47d580
HJ		 1254 reqctx->srcsg = req->src;
1255 reqctx->dstsg = req->dst;
1256 reqctx->src_ofst = 0;
1257 reqctx->dst_ofst = 0;
b8fd1f41
HJ		 1258 wrparam.qid = qid;
1259 wrparam.req = req;
1260 wrparam.bytes = bytes;
1261 *skb = create_cipher_wr(&wrparam);
1262 if (IS_ERR(*skb)) {
1263 err = PTR_ERR(*skb);
2f47d580 1264 goto unmap;
b8fd1f41 1265 }
2f47d580
HJ		 1266 reqctx->processed = bytes;
1267 reqctx->last_req_len = bytes;
b8fd1f41
HJ		 1268
1269 return 0;
2f47d580
HJ		 1270unmap:
1271 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
b8fd1f41
HJ		 1272error:
1273 return err;
1274}
1275
1276static int chcr_aes_encrypt(struct ablkcipher_request *req)
1277{
1278 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
b8fd1f41 1279 struct sk_buff *skb = NULL;
6faa0f57 1280 int err, isfull = 0;
2f47d580 1281 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
324429d7
HS		 1282
1283 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2f47d580 1284 c_ctx(tfm)->tx_qidx))) {
6faa0f57 1285 isfull = 1;
324429d7 1286 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 1287 return -ENOSPC;
324429d7
HS		 1288 }
1289
2f47d580
HJ		 1290 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1291 &skb, CHCR_ENCRYPT_OP);
b8fd1f41
HJ		 1292 if (err || !skb)
1293 return err;
324429d7 1294 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1295 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
324429d7 1296 chcr_send_wr(skb);
6faa0f57 1297 return isfull ? -EBUSY : -EINPROGRESS;
324429d7
HS		 1298}
1299
1300static int chcr_aes_decrypt(struct ablkcipher_request *req)
1301{
1302 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
2f47d580 1303 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
b8fd1f41 1304 struct sk_buff *skb = NULL;
6faa0f57 1305 int err, isfull = 0;
324429d7
HS		 1306
1307 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2f47d580 1308 c_ctx(tfm)->tx_qidx))) {
6faa0f57 1309 isfull = 1;
324429d7 1310 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 1311 return -ENOSPC;
324429d7
HS		 1312 }
1313
2f47d580
HJ		 1314 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1315 &skb, CHCR_DECRYPT_OP);
b8fd1f41
HJ		 1316 if (err || !skb)
1317 return err;
324429d7 1318 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1319 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
324429d7 1320 chcr_send_wr(skb);
6faa0f57 1321 return isfull ? -EBUSY : -EINPROGRESS;
324429d7
HS		 1322}
1323
1324static int chcr_device_init(struct chcr_context *ctx)
1325{
14c19b17 1326 struct uld_ctx *u_ctx = NULL;
72a56ca9 1327 struct adapter *adap;
324429d7 1328 unsigned int id;
72a56ca9 1329 int txq_perchan, txq_idx, ntxq;
324429d7
HS		 1330 int err = 0, rxq_perchan, rxq_idx;
1331
1332 id = smp_processor_id();
1333 if (!ctx->dev) {
14c19b17
HJ		 1334 u_ctx = assign_chcr_device();
1335 if (!u_ctx) {
324429d7
HS		 1336 pr_err("chcr device assignment fails\n");
1337 goto out;
1338 }
14c19b17 1339 ctx->dev = u_ctx->dev;
72a56ca9
HJ		 1340 adap = padap(ctx->dev);
1341 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1342 adap->vres.ncrypto_fc);
324429d7 1343 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
72a56ca9 1344 txq_perchan = ntxq / u_ctx->lldi.nchan;
324429d7 1345 spin_lock(&ctx->dev->lock_chcr_dev);
add92a81 1346 ctx->tx_chan_id = ctx->dev->tx_channel_id;
ab677ff4 1347 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
8a13449f 1348 ctx->dev->rx_channel_id = 0;
324429d7 1349 spin_unlock(&ctx->dev->lock_chcr_dev);
add92a81
HJ		 1350 rxq_idx = ctx->tx_chan_id * rxq_perchan;
1351 rxq_idx += id % rxq_perchan;
1352 txq_idx = ctx->tx_chan_id * txq_perchan;
1353 txq_idx += id % txq_perchan;
1354 ctx->rx_qidx = rxq_idx;
1355 ctx->tx_qidx = txq_idx;
1356 /* Channel Id used by SGE to forward packet to Host.
1357 * Same value should be used in cpl_fw6_pld RSS_CH field
1358 * by FW. Driver programs PCI channel ID to be used in fw
1359 * at the time of queue allocation with value "pi->tx_chan"
1360 */
1361 ctx->pci_chan_id = txq_idx / txq_perchan;
324429d7
HS		 1362 }
1363out:
1364 return err;
1365}
1366
1367static int chcr_cra_init(struct crypto_tfm *tfm)
1368{
b8fd1f41
HJ		 1369 struct crypto_alg *alg = tfm->__crt_alg;
1370 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1371 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1372
28874f26
KC		 1373 ablkctx->sw_cipher = crypto_alloc_sync_skcipher(alg->cra_name, 0,
1374 CRYPTO_ALG_NEED_FALLBACK);
b8fd1f41
HJ		 1375 if (IS_ERR(ablkctx->sw_cipher)) {
1376 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1377 return PTR_ERR(ablkctx->sw_cipher);
1378 }
d3f1d2f7
HJ		 1379
1380 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
1381 /* To update tweak*/
1382 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
1383 if (IS_ERR(ablkctx->aes_generic)) {
1384 pr_err("failed to allocate aes cipher for tweak\n");
1385 return PTR_ERR(ablkctx->aes_generic);
1386 }
1387 } else
1388 ablkctx->aes_generic = NULL;
1389
324429d7
HS		 1390 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1391 return chcr_device_init(crypto_tfm_ctx(tfm));
1392}
1393
b8fd1f41
HJ		 1394static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1395{
1396 struct crypto_alg *alg = tfm->__crt_alg;
1397 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1398 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1399
1400 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1401 * cannot be used as fallback in chcr_handle_cipher_response
1402 */
28874f26
KC		 1403 ablkctx->sw_cipher = crypto_alloc_sync_skcipher("ctr(aes)", 0,
1404 CRYPTO_ALG_NEED_FALLBACK);
b8fd1f41
HJ		 1405 if (IS_ERR(ablkctx->sw_cipher)) {
1406 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1407 return PTR_ERR(ablkctx->sw_cipher);
1408 }
1409 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1410 return chcr_device_init(crypto_tfm_ctx(tfm));
1411}
1412
1413
1414static void chcr_cra_exit(struct crypto_tfm *tfm)
1415{
1416 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1417 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1418
28874f26 1419 crypto_free_sync_skcipher(ablkctx->sw_cipher);
d3f1d2f7
HJ		 1420 if (ablkctx->aes_generic)
1421 crypto_free_cipher(ablkctx->aes_generic);
b8fd1f41
HJ		 1422}
1423
324429d7
HS		 1424static int get_alg_config(struct algo_param *params,
1425 unsigned int auth_size)
1426{
1427 switch (auth_size) {
1428 case SHA1_DIGEST_SIZE:
1429 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1430 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1431 params->result_size = SHA1_DIGEST_SIZE;
1432 break;
1433 case SHA224_DIGEST_SIZE:
1434 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1435 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1436 params->result_size = SHA256_DIGEST_SIZE;
1437 break;
1438 case SHA256_DIGEST_SIZE:
1439 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1440 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1441 params->result_size = SHA256_DIGEST_SIZE;
1442 break;
1443 case SHA384_DIGEST_SIZE:
1444 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1445 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1446 params->result_size = SHA512_DIGEST_SIZE;
1447 break;
1448 case SHA512_DIGEST_SIZE:
1449 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1450 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1451 params->result_size = SHA512_DIGEST_SIZE;
1452 break;
1453 default:
1454 pr_err("chcr : ERROR, unsupported digest size\n");
1455 return -EINVAL;
1456 }
1457 return 0;
1458}
1459
e7922729 1460static inline void chcr_free_shash(struct crypto_shash *base_hash)
324429d7 1461{
e7922729 1462 crypto_free_shash(base_hash);
324429d7
HS		 1463}
1464
1465/**
358961d1 1466 * create_hash_wr - Create hash work request
324429d7
HS		 1467 * @req - Cipher req base
1468 */
358961d1 1469static struct sk_buff *create_hash_wr(struct ahash_request *req,
2debd332 1470 struct hash_wr_param *param)
324429d7
HS		 1471{
1472 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1473 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2f47d580 1474 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
324429d7 1475 struct sk_buff *skb = NULL;
2f47d580 1476 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
358961d1 1477 struct chcr_wr *chcr_req;
2f47d580 1478 struct ulptx_sgl *ulptx;
5110e655
HJ		 1479 unsigned int nents = 0, transhdr_len;
1480 unsigned int temp = 0;
358961d1
HJ		 1481 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1482 GFP_ATOMIC;
2f47d580
HJ		 1483 struct adapter *adap = padap(h_ctx(tfm)->dev);
1484 int error = 0;
324429d7 1485
5110e655
HJ		 1486 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1487 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1488 param->sg_len) <= SGE_MAX_WR_LEN;
1489 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
1490 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
2f47d580 1491 nents += param->bfr_len ? 1 : 0;
5110e655
HJ		 1492 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1493 param->sg_len, 16) : (sgl_len(nents) * 8);
125d01ca 1494 transhdr_len = roundup(transhdr_len, 16);
2f47d580 1495
5110e655 1496 skb = alloc_skb(transhdr_len, flags);
324429d7 1497 if (!skb)
2f47d580 1498 return ERR_PTR(-ENOMEM);
de77b966 1499 chcr_req = __skb_put_zero(skb, transhdr_len);
324429d7 1500
358961d1 1501 chcr_req->sec_cpl.op_ivinsrtofst =
2f47d580 1502 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0);
358961d1 1503 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
324429d7 1504
358961d1 1505 chcr_req->sec_cpl.aadstart_cipherstop_hi =
324429d7 1506 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
358961d1 1507 chcr_req->sec_cpl.cipherstop_lo_authinsert =
324429d7 1508 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
358961d1 1509 chcr_req->sec_cpl.seqno_numivs =
324429d7 1510 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
358961d1 1511 param->opad_needed, 0);
324429d7 1512
358961d1 1513 chcr_req->sec_cpl.ivgen_hdrlen =
324429d7
HS		 1514 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1515
358961d1
HJ		 1516 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1517 param->alg_prm.result_size);
324429d7
HS		 1518
1519 if (param->opad_needed)
358961d1
HJ		 1520 memcpy(chcr_req->key_ctx.key +
1521 ((param->alg_prm.result_size <= 32) ? 32 :
1522 CHCR_HASH_MAX_DIGEST_SIZE),
324429d7
HS		 1523 hmacctx->opad, param->alg_prm.result_size);
1524
358961d1 1525 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
324429d7
HS		 1526 param->alg_prm.mk_size, 0,
1527 param->opad_needed,
5110e655 1528 ((param->kctx_len +
358961d1
HJ		 1529 sizeof(chcr_req->key_ctx)) >> 4));
1530 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
5110e655 1531 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
2f47d580
HJ		 1532 DUMMY_BYTES);
1533 if (param->bfr_len != 0) {
5110e655
HJ		 1534 req_ctx->hctx_wr.dma_addr =
1535 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1536 param->bfr_len, DMA_TO_DEVICE);
2f47d580 1537 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
5110e655 1538 req_ctx->hctx_wr. dma_addr)) {
2f47d580
HJ		 1539 error = -ENOMEM;
1540 goto err;
1541 }
5110e655 1542 req_ctx->hctx_wr.dma_len = param->bfr_len;
2f47d580 1543 } else {
5110e655 1544 req_ctx->hctx_wr.dma_addr = 0;
2f47d580
HJ		 1545 }
1546 chcr_add_hash_src_ent(req, ulptx, param);
1547 /* Request upto max wr size */
5110e655
HJ		 1548 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1549 (param->sg_len + param->bfr_len) : 0);
ee0863ba 1550 atomic_inc(&adap->chcr_stats.digest_rqst);
5110e655
HJ		 1551 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
1552 param->hash_size, transhdr_len,
2f47d580 1553 temp, 0);
5110e655 1554 req_ctx->hctx_wr.skb = skb;
324429d7 1555 return skb;
2f47d580
HJ		 1556err:
1557 kfree_skb(skb);
1558 return ERR_PTR(error);
324429d7
HS		 1559}
1560
1561static int chcr_ahash_update(struct ahash_request *req)
1562{
1563 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1564 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
324429d7
HS		 1565 struct uld_ctx *u_ctx = NULL;
1566 struct sk_buff *skb;
1567 u8 remainder = 0, bs;
1568 unsigned int nbytes = req->nbytes;
1569 struct hash_wr_param params;
6faa0f57 1570 int error, isfull = 0;
324429d7
HS		 1571
1572 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2f47d580 1573 u_ctx = ULD_CTX(h_ctx(rtfm));
324429d7 1574 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2f47d580 1575 h_ctx(rtfm)->tx_qidx))) {
6faa0f57 1576 isfull = 1;
324429d7 1577 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 1578 return -ENOSPC;
324429d7
HS		 1579 }
1580
44fce12a
HJ		 1581 if (nbytes + req_ctx->reqlen >= bs) {
1582 remainder = (nbytes + req_ctx->reqlen) % bs;
1583 nbytes = nbytes + req_ctx->reqlen - remainder;
324429d7 1584 } else {
44fce12a
HJ		 1585 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1586 + req_ctx->reqlen, nbytes, 0);
1587 req_ctx->reqlen += nbytes;
324429d7
HS		 1588 return 0;
1589 }
5110e655 1590 chcr_init_hctx_per_wr(req_ctx);
2f47d580
HJ		 1591 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1592 if (error)
1593 return -ENOMEM;
5110e655
HJ		 1594 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1595 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1596 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1597 HASH_SPACE_LEFT(params.kctx_len), 0);
1598 if (params.sg_len > req->nbytes)
1599 params.sg_len = req->nbytes;
1600 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1601 req_ctx->reqlen;
324429d7
HS		 1602 params.opad_needed = 0;
1603 params.more = 1;
1604 params.last = 0;
44fce12a 1605 params.bfr_len = req_ctx->reqlen;
324429d7 1606 params.scmd1 = 0;
5110e655
HJ		 1607 req_ctx->hctx_wr.srcsg = req->src;
1608
1609 params.hash_size = params.alg_prm.result_size;
324429d7 1610 req_ctx->data_len += params.sg_len + params.bfr_len;
358961d1 1611 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1612 if (IS_ERR(skb)) {
1613 error = PTR_ERR(skb);
1614 goto unmap;
1615 }
324429d7 1616
5110e655 1617 req_ctx->hctx_wr.processed += params.sg_len;
44fce12a 1618 if (remainder) {
44fce12a 1619 /* Swap buffers */
abfa2b37 1620 swap(req_ctx->reqbfr, req_ctx->skbfr);
324429d7 1621 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
44fce12a 1622 req_ctx->reqbfr, remainder, req->nbytes -
324429d7 1623 remainder);
44fce12a
HJ		 1624 }
1625 req_ctx->reqlen = remainder;
324429d7 1626 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1627 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
324429d7
HS		 1628 chcr_send_wr(skb);
1629
6faa0f57 1630 return isfull ? -EBUSY : -EINPROGRESS;
2f47d580
HJ		 1631unmap:
1632 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1633 return error;
324429d7
HS		 1634}
1635
1636static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1637{
1638 memset(bfr_ptr, 0, bs);
1639 *bfr_ptr = 0x80;
1640 if (bs == 64)
1641 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1642 else
1643 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1644}
1645
1646static int chcr_ahash_final(struct ahash_request *req)
1647{
1648 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1649 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
324429d7
HS		 1650 struct hash_wr_param params;
1651 struct sk_buff *skb;
1652 struct uld_ctx *u_ctx = NULL;
1653 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1654
5110e655 1655 chcr_init_hctx_per_wr(req_ctx);
2f47d580 1656 u_ctx = ULD_CTX(h_ctx(rtfm));
324429d7
HS		 1657 if (is_hmac(crypto_ahash_tfm(rtfm)))
1658 params.opad_needed = 1;
1659 else
1660 params.opad_needed = 0;
1661 params.sg_len = 0;
5110e655 1662 req_ctx->hctx_wr.isfinal = 1;
324429d7 1663 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
5110e655
HJ		 1664 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1665 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1666 params.opad_needed = 1;
1667 params.kctx_len *= 2;
1668 } else {
1669 params.opad_needed = 0;
1670 }
1671
1672 req_ctx->hctx_wr.result = 1;
44fce12a 1673 params.bfr_len = req_ctx->reqlen;
324429d7 1674 req_ctx->data_len += params.bfr_len + params.sg_len;
5110e655 1675 req_ctx->hctx_wr.srcsg = req->src;
44fce12a
HJ		 1676 if (req_ctx->reqlen == 0) {
1677 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1678 params.last = 0;
1679 params.more = 1;
1680 params.scmd1 = 0;
1681 params.bfr_len = bs;
1682
1683 } else {
1684 params.scmd1 = req_ctx->data_len;
1685 params.last = 1;
1686 params.more = 0;
1687 }
5110e655 1688 params.hash_size = crypto_ahash_digestsize(rtfm);
358961d1 1689 skb = create_hash_wr(req, &params);
40cdbe1a
YG		 1690 if (IS_ERR(skb))
1691 return PTR_ERR(skb);
5110e655 1692 req_ctx->reqlen = 0;
324429d7 1693 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1694 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
324429d7
HS		 1695 chcr_send_wr(skb);
1696 return -EINPROGRESS;
1697}
1698
1699static int chcr_ahash_finup(struct ahash_request *req)
1700{
1701 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1702 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
324429d7
HS		 1703 struct uld_ctx *u_ctx = NULL;
1704 struct sk_buff *skb;
1705 struct hash_wr_param params;
1706 u8 bs;
6faa0f57 1707 int error, isfull = 0;
324429d7
HS		 1708
1709 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2f47d580 1710 u_ctx = ULD_CTX(h_ctx(rtfm));
324429d7
HS		 1711
1712 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2f47d580 1713 h_ctx(rtfm)->tx_qidx))) {
6faa0f57 1714 isfull = 1;
324429d7 1715 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 1716 return -ENOSPC;
324429d7 1717 }
5110e655
HJ		 1718 chcr_init_hctx_per_wr(req_ctx);
1719 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1720 if (error)
1721 return -ENOMEM;
324429d7 1722
5110e655
HJ		 1723 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1724 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1725 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1726 params.kctx_len *= 2;
324429d7 1727 params.opad_needed = 1;
5110e655 1728 } else {
324429d7 1729 params.opad_needed = 0;
5110e655 1730 }
324429d7 1731
5110e655
HJ		 1732 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1733 HASH_SPACE_LEFT(params.kctx_len), 0);
1734 if (params.sg_len < req->nbytes) {
1735 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1736 params.kctx_len /= 2;
1737 params.opad_needed = 0;
1738 }
1739 params.last = 0;
1740 params.more = 1;
1741 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1742 - req_ctx->reqlen;
1743 params.hash_size = params.alg_prm.result_size;
1744 params.scmd1 = 0;
1745 } else {
1746 params.last = 1;
1747 params.more = 0;
1748 params.sg_len = req->nbytes;
1749 params.hash_size = crypto_ahash_digestsize(rtfm);
1750 params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1751 params.sg_len;
1752 }
44fce12a 1753 params.bfr_len = req_ctx->reqlen;
324429d7 1754 req_ctx->data_len += params.bfr_len + params.sg_len;
5110e655
HJ		 1755 req_ctx->hctx_wr.result = 1;
1756 req_ctx->hctx_wr.srcsg = req->src;
44fce12a
HJ		 1757 if ((req_ctx->reqlen + req->nbytes) == 0) {
1758 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1759 params.last = 0;
1760 params.more = 1;
1761 params.scmd1 = 0;
1762 params.bfr_len = bs;
324429d7 1763 }
358961d1 1764 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1765 if (IS_ERR(skb)) {
1766 error = PTR_ERR(skb);
1767 goto unmap;
1768 }
5110e655
HJ		 1769 req_ctx->reqlen = 0;
1770 req_ctx->hctx_wr.processed += params.sg_len;
324429d7 1771 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1772 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
324429d7
HS		 1773 chcr_send_wr(skb);
1774
6faa0f57 1775 return isfull ? -EBUSY : -EINPROGRESS;
2f47d580
HJ		 1776unmap:
1777 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1778 return error;
324429d7
HS		 1779}
1780
1781static int chcr_ahash_digest(struct ahash_request *req)
1782{
1783 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1784 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
324429d7
HS		 1785 struct uld_ctx *u_ctx = NULL;
1786 struct sk_buff *skb;
1787 struct hash_wr_param params;
1788 u8 bs;
6faa0f57 1789 int error, isfull = 0;
324429d7
HS		 1790
1791 rtfm->init(req);
1792 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1793
2f47d580 1794 u_ctx = ULD_CTX(h_ctx(rtfm));
324429d7 1795 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2f47d580 1796 h_ctx(rtfm)->tx_qidx))) {
6faa0f57 1797 isfull = 1;
324429d7 1798 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 1799 return -ENOSPC;
324429d7
HS		 1800 }
1801
5110e655 1802 chcr_init_hctx_per_wr(req_ctx);
2f47d580
HJ		 1803 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1804 if (error)
1805 return -ENOMEM;
324429d7 1806
324429d7 1807 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
5110e655
HJ		 1808 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1809 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1810 params.kctx_len *= 2;
1811 params.opad_needed = 1;
1812 } else {
1813 params.opad_needed = 0;
1814 }
1815 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1816 HASH_SPACE_LEFT(params.kctx_len), 0);
1817 if (params.sg_len < req->nbytes) {
1818 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1819 params.kctx_len /= 2;
1820 params.opad_needed = 0;
1821 }
1822 params.last = 0;
1823 params.more = 1;
1824 params.scmd1 = 0;
1825 params.sg_len = rounddown(params.sg_len, bs);
1826 params.hash_size = params.alg_prm.result_size;
1827 } else {
1828 params.sg_len = req->nbytes;
1829 params.hash_size = crypto_ahash_digestsize(rtfm);
1830 params.last = 1;
1831 params.more = 0;
1832 params.scmd1 = req->nbytes + req_ctx->data_len;
1833
1834 }
1835 params.bfr_len = 0;
1836 req_ctx->hctx_wr.result = 1;
1837 req_ctx->hctx_wr.srcsg = req->src;
324429d7
HS		 1838 req_ctx->data_len += params.bfr_len + params.sg_len;
1839
44fce12a
HJ		 1840 if (req->nbytes == 0) {
1841 create_last_hash_block(req_ctx->reqbfr, bs, 0);
324429d7
HS		 1842 params.more = 1;
1843 params.bfr_len = bs;
1844 }
1845
358961d1 1846 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1847 if (IS_ERR(skb)) {
1848 error = PTR_ERR(skb);
1849 goto unmap;
1850 }
5110e655 1851 req_ctx->hctx_wr.processed += params.sg_len;
324429d7 1852 skb->dev = u_ctx->lldi.ports[0];
2f47d580 1853 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
324429d7 1854 chcr_send_wr(skb);
6faa0f57 1855 return isfull ? -EBUSY : -EINPROGRESS;
2f47d580
HJ		 1856unmap:
1857 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1858 return error;
324429d7
HS		 1859}
1860
6f76672b
HJ		 1861static int chcr_ahash_continue(struct ahash_request *req)
1862{
1863 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
1864 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
1865 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1866 struct uld_ctx *u_ctx = NULL;
1867 struct sk_buff *skb;
1868 struct hash_wr_param params;
1869 u8 bs;
1870 int error;
1871
1872 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1873 u_ctx = ULD_CTX(h_ctx(rtfm));
6f76672b
HJ		 1874 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1875 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1876 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1877 params.kctx_len *= 2;
1878 params.opad_needed = 1;
1879 } else {
1880 params.opad_needed = 0;
1881 }
1882 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
1883 HASH_SPACE_LEFT(params.kctx_len),
1884 hctx_wr->src_ofst);
1885 if ((params.sg_len + hctx_wr->processed) > req->nbytes)
1886 params.sg_len = req->nbytes - hctx_wr->processed;
1887 if (!hctx_wr->result ||
1888 ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
1889 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1890 params.kctx_len /= 2;
1891 params.opad_needed = 0;
1892 }
1893 params.last = 0;
1894 params.more = 1;
1895 params.sg_len = rounddown(params.sg_len, bs);
1896 params.hash_size = params.alg_prm.result_size;
1897 params.scmd1 = 0;
1898 } else {
1899 params.last = 1;
1900 params.more = 0;
1901 params.hash_size = crypto_ahash_digestsize(rtfm);
1902 params.scmd1 = reqctx->data_len + params.sg_len;
1903 }
1904 params.bfr_len = 0;
1905 reqctx->data_len += params.sg_len;
1906 skb = create_hash_wr(req, &params);
1907 if (IS_ERR(skb)) {
1908 error = PTR_ERR(skb);
1909 goto err;
1910 }
1911 hctx_wr->processed += params.sg_len;
1912 skb->dev = u_ctx->lldi.ports[0];
1913 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1914 chcr_send_wr(skb);
1915 return 0;
1916err:
1917 return error;
1918}
1919
1920static inline void chcr_handle_ahash_resp(struct ahash_request *req,
1921 unsigned char *input,
1922 int err)
1923{
1924 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
1925 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
1926 int digestsize, updated_digestsize;
1927 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1928 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
1929
1930 if (input == NULL)
1931 goto out;
1932 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
1933 updated_digestsize = digestsize;
1934 if (digestsize == SHA224_DIGEST_SIZE)
1935 updated_digestsize = SHA256_DIGEST_SIZE;
1936 else if (digestsize == SHA384_DIGEST_SIZE)
1937 updated_digestsize = SHA512_DIGEST_SIZE;
1938
1939 if (hctx_wr->dma_addr) {
1940 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
1941 hctx_wr->dma_len, DMA_TO_DEVICE);
1942 hctx_wr->dma_addr = 0;
1943 }
1944 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
1945 req->nbytes)) {
1946 if (hctx_wr->result == 1) {
1947 hctx_wr->result = 0;
1948 memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
1949 digestsize);
1950 } else {
1951 memcpy(reqctx->partial_hash,
1952 input + sizeof(struct cpl_fw6_pld),
1953 updated_digestsize);
1954
1955 }
1956 goto unmap;
1957 }
1958 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
1959 updated_digestsize);
1960
1961 err = chcr_ahash_continue(req);
1962 if (err)
1963 goto unmap;
1964 return;
1965unmap:
1966 if (hctx_wr->is_sg_map)
1967 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1968
1969
1970out:
1971 req->base.complete(&req->base, err);
1972}
1973
1974/*
1975 * chcr_handle_resp - Unmap the DMA buffers associated with the request
1976 * @req: crypto request
1977 */
1978int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
1979 int err)
1980{
1981 struct crypto_tfm *tfm = req->tfm;
1982 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1983 struct adapter *adap = padap(ctx->dev);
1984
1985 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
1986 case CRYPTO_ALG_TYPE_AEAD:
1987 chcr_handle_aead_resp(aead_request_cast(req), input, err);
1988 break;
1989
1990 case CRYPTO_ALG_TYPE_ABLKCIPHER:
1991 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
1992 input, err);
1993 break;
1994
1995 case CRYPTO_ALG_TYPE_AHASH:
1996 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
1997 }
1998 atomic_inc(&adap->chcr_stats.complete);
1999 return err;
2000}
324429d7
HS		 2001static int chcr_ahash_export(struct ahash_request *areq, void *out)
2002{
2003 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2004 struct chcr_ahash_req_ctx *state = out;
2005
44fce12a 2006 state->reqlen = req_ctx->reqlen;
324429d7 2007 state->data_len = req_ctx->data_len;
44fce12a 2008 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
324429d7
HS		 2009 memcpy(state->partial_hash, req_ctx->partial_hash,
2010 CHCR_HASH_MAX_DIGEST_SIZE);
5110e655 2011 chcr_init_hctx_per_wr(state);
44fce12a 2012 return 0;
324429d7
HS		 2013}
2014
2015static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2016{
2017 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2018 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2019
44fce12a 2020 req_ctx->reqlen = state->reqlen;
324429d7 2021 req_ctx->data_len = state->data_len;
44fce12a
HJ		 2022 req_ctx->reqbfr = req_ctx->bfr1;
2023 req_ctx->skbfr = req_ctx->bfr2;
2024 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
324429d7
HS		 2025 memcpy(req_ctx->partial_hash, state->partial_hash,
2026 CHCR_HASH_MAX_DIGEST_SIZE);
5110e655 2027 chcr_init_hctx_per_wr(req_ctx);
324429d7
HS		 2028 return 0;
2029}
2030
2031static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2032 unsigned int keylen)
2033{
2f47d580 2034 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
324429d7
HS		 2035 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2036 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2037 unsigned int i, err = 0, updated_digestsize;
2038
e7922729
HJ		 2039 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
2040
2041 /* use the key to calculate the ipad and opad. ipad will sent with the
324429d7
HS		 2042 * first request's data. opad will be sent with the final hash result
2043 * ipad in hmacctx->ipad and opad in hmacctx->opad location
2044 */
e7922729
HJ		 2045 shash->tfm = hmacctx->base_hash;
2046 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
324429d7 2047 if (keylen > bs) {
e7922729 2048 err = crypto_shash_digest(shash, key, keylen,
324429d7
HS		 2049 hmacctx->ipad);
2050 if (err)
2051 goto out;
2052 keylen = digestsize;
2053 } else {
2054 memcpy(hmacctx->ipad, key, keylen);
2055 }
2056 memset(hmacctx->ipad + keylen, 0, bs - keylen);
2057 memcpy(hmacctx->opad, hmacctx->ipad, bs);
2058
2059 for (i = 0; i < bs / sizeof(int); i++) {
2060 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
2061 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
2062 }
2063
2064 updated_digestsize = digestsize;
2065 if (digestsize == SHA224_DIGEST_SIZE)
2066 updated_digestsize = SHA256_DIGEST_SIZE;
2067 else if (digestsize == SHA384_DIGEST_SIZE)
2068 updated_digestsize = SHA512_DIGEST_SIZE;
e7922729 2069 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
324429d7
HS		 2070 hmacctx->ipad, digestsize);
2071 if (err)
2072 goto out;
2073 chcr_change_order(hmacctx->ipad, updated_digestsize);
2074
e7922729 2075 err = chcr_compute_partial_hash(shash, hmacctx->opad,
324429d7
HS		 2076 hmacctx->opad, digestsize);
2077 if (err)
2078 goto out;
2079 chcr_change_order(hmacctx->opad, updated_digestsize);
2080out:
2081 return err;
2082}
2083
b8fd1f41 2084static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
324429d7
HS		 2085 unsigned int key_len)
2086{
2f47d580 2087 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
324429d7 2088 unsigned short context_size = 0;
b8fd1f41 2089 int err;
324429d7 2090
b8fd1f41
HJ		 2091 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
2092 if (err)
2093 goto badkey_err;
cc1b156d
HJ		 2094
2095 memcpy(ablkctx->key, key, key_len);
2096 ablkctx->enckey_len = key_len;
2097 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
2098 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2099 ablkctx->key_ctx_hdr =
2100 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2101 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2102 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2103 CHCR_KEYCTX_NO_KEY, 1,
2104 0, context_size);
2105 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2106 return 0;
b8fd1f41
HJ		 2107badkey_err:
2108 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
2109 ablkctx->enckey_len = 0;
2110
2111 return err;
324429d7
HS		 2112}
2113
2114static int chcr_sha_init(struct ahash_request *areq)
2115{
2116 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2117 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2118 int digestsize = crypto_ahash_digestsize(tfm);
2119
2120 req_ctx->data_len = 0;
44fce12a
HJ		 2121 req_ctx->reqlen = 0;
2122 req_ctx->reqbfr = req_ctx->bfr1;
2123 req_ctx->skbfr = req_ctx->bfr2;
324429d7 2124 copy_hash_init_values(req_ctx->partial_hash, digestsize);
5110e655 2125
324429d7
HS		 2126 return 0;
2127}
2128
2129static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2130{
2131 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2132 sizeof(struct chcr_ahash_req_ctx));
2133 return chcr_device_init(crypto_tfm_ctx(tfm));
2134}
2135
2136static int chcr_hmac_init(struct ahash_request *areq)
2137{
2138 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2139 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
2f47d580 2140 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
324429d7
HS		 2141 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
2142 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2143
2144 chcr_sha_init(areq);
2145 req_ctx->data_len = bs;
2146 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2147 if (digestsize == SHA224_DIGEST_SIZE)
2148 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2149 SHA256_DIGEST_SIZE);
2150 else if (digestsize == SHA384_DIGEST_SIZE)
2151 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2152 SHA512_DIGEST_SIZE);
2153 else
2154 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2155 digestsize);
2156 }
2157 return 0;
2158}
2159
2160static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2161{
2162 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2163 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2164 unsigned int digestsize =
2165 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
2166
2167 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2168 sizeof(struct chcr_ahash_req_ctx));
e7922729
HJ		 2169 hmacctx->base_hash = chcr_alloc_shash(digestsize);
2170 if (IS_ERR(hmacctx->base_hash))
2171 return PTR_ERR(hmacctx->base_hash);
324429d7
HS		 2172 return chcr_device_init(crypto_tfm_ctx(tfm));
2173}
2174
324429d7
HS		 2175static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2176{
2177 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2178 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2179
e7922729
HJ		 2180 if (hmacctx->base_hash) {
2181 chcr_free_shash(hmacctx->base_hash);
2182 hmacctx->base_hash = NULL;
324429d7
HS		 2183 }
2184}
2185
4262c98a
HJ		 2186inline void chcr_aead_common_exit(struct aead_request *req)
2187{
2188 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2189 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2190 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
2191
2192 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
2193}
2194
2195static int chcr_aead_common_init(struct aead_request *req)
2debd332 2196{
2f47d580
HJ		 2197 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2198 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2199 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2f47d580 2200 unsigned int authsize = crypto_aead_authsize(tfm);
4262c98a 2201 int error = -EINVAL;
2debd332 2202
2f47d580
HJ		 2203 /* validate key size */
2204 if (aeadctx->enckey_len == 0)
2205 goto err;
4262c98a 2206 if (reqctx->op && req->cryptlen < authsize)
2f47d580 2207 goto err;
4262c98a
HJ		 2208 if (reqctx->b0_len)
2209 reqctx->scratch_pad = reqctx->iv + IV;
2210 else
2211 reqctx->scratch_pad = NULL;
2212
2f47d580 2213 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
4262c98a 2214 reqctx->op);
2f47d580
HJ		 2215 if (error) {
2216 error = -ENOMEM;
2217 goto err;
2218 }
2219 reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen,
2220 CHCR_SRC_SG_SIZE, 0);
2221 reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen,
2222 CHCR_SRC_SG_SIZE, req->assoclen);
2223 return 0;
2224err:
2225 return error;
2debd332 2226}
2f47d580
HJ		 2227
2228static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
0e93708d
HJ		 2229 int aadmax, int wrlen,
2230 unsigned short op_type)
2231{
2232 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2233
2234 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2f47d580 2235 dst_nents > MAX_DSGL_ENT ||
0e93708d 2236 (req->assoclen > aadmax) ||
2f47d580 2237 (wrlen > SGE_MAX_WR_LEN))
0e93708d
HJ		 2238 return 1;
2239 return 0;
2240}
2debd332 2241
0e93708d
HJ		 2242static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2243{
2244 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2245 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d
HJ		 2246 struct aead_request *subreq = aead_request_ctx(req);
2247
2248 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2249 aead_request_set_callback(subreq, req->base.flags,
2250 req->base.complete, req->base.data);
4262c98a 2251 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
0e93708d
HJ		 2252 req->iv);
2253 aead_request_set_ad(subreq, req->assoclen);
2254 return op_type ? crypto_aead_decrypt(subreq) :
2255 crypto_aead_encrypt(subreq);
2256}
2debd332
HJ		 2257
2258static struct sk_buff *create_authenc_wr(struct aead_request *req,
2259 unsigned short qid,
4262c98a 2260 int size)
2debd332
HJ		 2261{
2262 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2263 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2264 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2265 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2266 struct sk_buff *skb = NULL;
2267 struct chcr_wr *chcr_req;
2268 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580
HJ		 2269 struct ulptx_sgl *ulptx;
2270 unsigned int transhdr_len;
3d64bd67 2271 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
2f47d580 2272 unsigned int kctx_len = 0, dnents;
2debd332
HJ		 2273 unsigned int assoclen = req->assoclen;
2274 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 2275 int error = -EINVAL;
2debd332
HJ		 2276 int null = 0;
2277 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2278 GFP_ATOMIC;
2f47d580 2279 struct adapter *adap = padap(a_ctx(tfm)->dev);
2debd332 2280
2f47d580
HJ		 2281 if (req->cryptlen == 0)
2282 return NULL;
2debd332 2283
4262c98a
HJ		 2284 reqctx->b0_len = 0;
2285 error = chcr_aead_common_init(req);
2286 if (error)
2287 return ERR_PTR(error);
2288
3d64bd67 2289 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
4262c98a 2290 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2debd332
HJ		 2291 null = 1;
2292 assoclen = 0;
4262c98a 2293 reqctx->aad_nents = 0;
2debd332 2294 }
5abc8db0
HJ		 2295 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2296 dnents += sg_nents_xlen(req->dst, req->cryptlen +
4262c98a 2297 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE,
5abc8db0
HJ		 2298 req->assoclen);
2299 dnents += MIN_AUTH_SG; // For IV
2f47d580
HJ		 2300
2301 dst_size = get_space_for_phys_dsgl(dnents);
2debd332
HJ		 2302 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
2303 - sizeof(chcr_req->key_ctx);
2304 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2f47d580
HJ		 2305 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <
2306 SGE_MAX_WR_LEN;
125d01ca
HJ		 2307 temp = reqctx->imm ? roundup(assoclen + IV + req->cryptlen, 16)
2308 : (sgl_len(reqctx->src_nents + reqctx->aad_nents
2f47d580
HJ		 2309 + MIN_GCM_SG) * 8);
2310 transhdr_len += temp;
125d01ca 2311 transhdr_len = roundup(transhdr_len, 16);
2f47d580
HJ		 2312
2313 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
4262c98a 2314 transhdr_len, reqctx->op)) {
ee0863ba 2315 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 2316 chcr_aead_common_exit(req);
2317 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 2318 }
2f47d580 2319 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
5fe8c711
HJ		 2320 if (!skb) {
2321 error = -ENOMEM;
2debd332 2322 goto err;
5fe8c711 2323 }
2debd332 2324
de77b966 2325 chcr_req = __skb_put_zero(skb, transhdr_len);
2debd332 2326
4262c98a 2327 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2debd332
HJ		 2328
2329 /*
2330 * Input order is AAD,IV and Payload. where IV should be included as
2331 * the part of authdata. All other fields should be filled according
2332 * to the hardware spec
2333 */
2334 chcr_req->sec_cpl.op_ivinsrtofst =
2f47d580
HJ		 2335 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2,
2336 assoclen + 1);
2337 chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen);
2debd332
HJ		 2338 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2339 assoclen ? 1 : 0, assoclen,
2f47d580
HJ		 2340 assoclen + IV + 1,
2341 (temp & 0x1F0) >> 4);
2debd332 2342 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2f47d580
HJ		 2343 temp & 0xF,
2344 null ? 0 : assoclen + IV + 1,
2345 temp, temp);
3d64bd67
HJ		 2346 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2347 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2348 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2349 else
2350 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
4262c98a
HJ		 2351 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2352 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
3d64bd67 2353 temp,
2debd332 2354 actx->auth_mode, aeadctx->hmac_ctrl,
2f47d580 2355 IV >> 1);
2debd332 2356 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2f47d580 2357 0, 0, dst_size);
2debd332
HJ		 2358
2359 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
4262c98a 2360 if (reqctx->op == CHCR_ENCRYPT_OP ||
3d64bd67
HJ		 2361 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2362 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2debd332
HJ		 2363 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2364 aeadctx->enckey_len);
2365 else
2366 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2367 aeadctx->enckey_len);
2368
125d01ca
HJ		 2369 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2370 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
3d64bd67
HJ		 2371 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2372 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2373 memcpy(reqctx->iv, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2374 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
2375 CTR_RFC3686_IV_SIZE);
2376 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
2377 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2378 } else {
2379 memcpy(reqctx->iv, req->iv, IV);
2380 }
2debd332 2381 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2f47d580 2382 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
4262c98a
HJ		 2383 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
2384 chcr_add_aead_src_ent(req, ulptx, assoclen);
ee0863ba 2385 atomic_inc(&adap->chcr_stats.cipher_rqst);
2f47d580
HJ		 2386 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2387 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
2388 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2389 transhdr_len, temp, 0);
2debd332 2390 reqctx->skb = skb;
2debd332
HJ		 2391
2392 return skb;
2debd332 2393err:
4262c98a 2394 chcr_aead_common_exit(req);
2f47d580 2395
5fe8c711 2396 return ERR_PTR(error);
2debd332
HJ		 2397}
2398
6dad4e8a
AG		 2399int chcr_aead_dma_map(struct device *dev,
2400 struct aead_request *req,
2401 unsigned short op_type)
2f47d580
HJ		 2402{
2403 int error;
2404 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2405 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2406 unsigned int authsize = crypto_aead_authsize(tfm);
2407 int dst_size;
2408
2409 dst_size = req->assoclen + req->cryptlen + (op_type ?
2410 -authsize : authsize);
2411 if (!req->cryptlen || !dst_size)
2412 return 0;
4262c98a 2413 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2f47d580
HJ		 2414 DMA_BIDIRECTIONAL);
2415 if (dma_mapping_error(dev, reqctx->iv_dma))
2416 return -ENOMEM;
4262c98a
HJ		 2417 if (reqctx->b0_len)
2418 reqctx->b0_dma = reqctx->iv_dma + IV;
2419 else
2420 reqctx->b0_dma = 0;
2f47d580
HJ		 2421 if (req->src == req->dst) {
2422 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2423 DMA_BIDIRECTIONAL);
2424 if (!error)
2425 goto err;
2426 } else {
2427 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2428 DMA_TO_DEVICE);
2429 if (!error)
2430 goto err;
2431 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2432 DMA_FROM_DEVICE);
2433 if (!error) {
2434 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2435 DMA_TO_DEVICE);
2436 goto err;
2437 }
2438 }
2439
2440 return 0;
2441err:
2442 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2443 return -ENOMEM;
2444}
2445
6dad4e8a
AG		 2446void chcr_aead_dma_unmap(struct device *dev,
2447 struct aead_request *req,
2448 unsigned short op_type)
2f47d580
HJ		 2449{
2450 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2451 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2452 unsigned int authsize = crypto_aead_authsize(tfm);
2453 int dst_size;
2454
2455 dst_size = req->assoclen + req->cryptlen + (op_type ?
2456 -authsize : authsize);
2457 if (!req->cryptlen || !dst_size)
2458 return;
2459
4262c98a 2460 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2f47d580
HJ		 2461 DMA_BIDIRECTIONAL);
2462 if (req->src == req->dst) {
2463 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2464 DMA_BIDIRECTIONAL);
2465 } else {
2466 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2467 DMA_TO_DEVICE);
2468 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2469 DMA_FROM_DEVICE);
2470 }
2471}
2472
6dad4e8a
AG		 2473void chcr_add_aead_src_ent(struct aead_request *req,
2474 struct ulptx_sgl *ulptx,
4262c98a 2475 unsigned int assoclen)
2f47d580
HJ		 2476{
2477 struct ulptx_walk ulp_walk;
2478 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2479
2480 if (reqctx->imm) {
2481 u8 *buf = (u8 *)ulptx;
2482
4262c98a 2483 if (reqctx->b0_len) {
2f47d580
HJ		 2484 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2485 buf += reqctx->b0_len;
2486 }
2487 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2488 buf, assoclen, 0);
2489 buf += assoclen;
2490 memcpy(buf, reqctx->iv, IV);
2491 buf += IV;
2492 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2493 buf, req->cryptlen, req->assoclen);
2494 } else {
2495 ulptx_walk_init(&ulp_walk, ulptx);
4262c98a 2496 if (reqctx->b0_len)
2f47d580
HJ		 2497 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2498 &reqctx->b0_dma);
2499 ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0);
2500 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
2501 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen,
2502 req->assoclen);
2503 ulptx_walk_end(&ulp_walk);
2504 }
2505}
2506
6dad4e8a
AG		 2507void chcr_add_aead_dst_ent(struct aead_request *req,
2508 struct cpl_rx_phys_dsgl *phys_cpl,
2509 unsigned int assoclen,
6dad4e8a 2510 unsigned short qid)
2f47d580
HJ		 2511{
2512 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2513 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2514 struct dsgl_walk dsgl_walk;
2515 unsigned int authsize = crypto_aead_authsize(tfm);
add92a81 2516 struct chcr_context *ctx = a_ctx(tfm);
2f47d580
HJ		 2517 u32 temp;
2518
2519 dsgl_walk_init(&dsgl_walk, phys_cpl);
4262c98a 2520 if (reqctx->b0_len)
2f47d580
HJ		 2521 dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma);
2522 dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0);
2523 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
4262c98a 2524 temp = req->cryptlen + (reqctx->op ? -authsize : authsize);
2f47d580 2525 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen);
add92a81 2526 dsgl_walk_end(&dsgl_walk, qid, ctx->pci_chan_id);
2f47d580
HJ		 2527}
2528
6dad4e8a 2529void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
335bcc4a 2530 void *ulptx,
6dad4e8a 2531 struct cipher_wr_param *wrparam)
2f47d580
HJ		 2532{
2533 struct ulptx_walk ulp_walk;
2534 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
335bcc4a 2535 u8 *buf = ulptx;
2f47d580 2536
335bcc4a
HJ		 2537 memcpy(buf, reqctx->iv, IV);
2538 buf += IV;
2f47d580 2539 if (reqctx->imm) {
2f47d580
HJ		 2540 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2541 buf, wrparam->bytes, reqctx->processed);
2542 } else {
335bcc4a 2543 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
2f47d580
HJ		 2544 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2545 reqctx->src_ofst);
2546 reqctx->srcsg = ulp_walk.last_sg;
2547 reqctx->src_ofst = ulp_walk.last_sg_len;
2548 ulptx_walk_end(&ulp_walk);
2549 }
2550}
2551
6dad4e8a
AG		 2552void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
2553 struct cpl_rx_phys_dsgl *phys_cpl,
2554 struct cipher_wr_param *wrparam,
2555 unsigned short qid)
2f47d580
HJ		 2556{
2557 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
add92a81
HJ		 2558 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
2559 struct chcr_context *ctx = c_ctx(tfm);
2f47d580
HJ		 2560 struct dsgl_walk dsgl_walk;
2561
2562 dsgl_walk_init(&dsgl_walk, phys_cpl);
2f47d580
HJ		 2563 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2564 reqctx->dst_ofst);
2565 reqctx->dstsg = dsgl_walk.last_sg;
2566 reqctx->dst_ofst = dsgl_walk.last_sg_len;
2567
add92a81 2568 dsgl_walk_end(&dsgl_walk, qid, ctx->pci_chan_id);
2f47d580
HJ		 2569}
2570
6dad4e8a
AG		 2571void chcr_add_hash_src_ent(struct ahash_request *req,
2572 struct ulptx_sgl *ulptx,
2573 struct hash_wr_param *param)
2f47d580
HJ		 2574{
2575 struct ulptx_walk ulp_walk;
2576 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2577
5110e655 2578 if (reqctx->hctx_wr.imm) {
2f47d580
HJ		 2579 u8 *buf = (u8 *)ulptx;
2580
2581 if (param->bfr_len) {
2582 memcpy(buf, reqctx->reqbfr, param->bfr_len);
2583 buf += param->bfr_len;
2584 }
5110e655
HJ		 2585
2586 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
2587 sg_nents(reqctx->hctx_wr.srcsg), buf,
2588 param->sg_len, 0);
2f47d580
HJ		 2589 } else {
2590 ulptx_walk_init(&ulp_walk, ulptx);
2591 if (param->bfr_len)
2592 ulptx_walk_add_page(&ulp_walk, param->bfr_len,
5110e655
HJ		 2593 &reqctx->hctx_wr.dma_addr);
2594 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
2595 param->sg_len, reqctx->hctx_wr.src_ofst);
2596 reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2597 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
db6deea4 2598 ulptx_walk_end(&ulp_walk);
2f47d580
HJ		 2599 }
2600}
2601
6dad4e8a
AG		 2602int chcr_hash_dma_map(struct device *dev,
2603 struct ahash_request *req)
2f47d580
HJ		 2604{
2605 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2606 int error = 0;
2607
2608 if (!req->nbytes)
2609 return 0;
2610 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2611 DMA_TO_DEVICE);
2612 if (!error)
7814f552 2613 return -ENOMEM;
5110e655 2614 req_ctx->hctx_wr.is_sg_map = 1;
2f47d580
HJ		 2615 return 0;
2616}
2617
6dad4e8a
AG		 2618void chcr_hash_dma_unmap(struct device *dev,
2619 struct ahash_request *req)
2f47d580
HJ		 2620{
2621 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2622
2623 if (!req->nbytes)
2624 return;
2625
2626 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2627 DMA_TO_DEVICE);
5110e655 2628 req_ctx->hctx_wr.is_sg_map = 0;
2f47d580
HJ		 2629
2630}
2631
6dad4e8a
AG		 2632int chcr_cipher_dma_map(struct device *dev,
2633 struct ablkcipher_request *req)
2f47d580
HJ		 2634{
2635 int error;
2f47d580
HJ		 2636
2637 if (req->src == req->dst) {
2638 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2639 DMA_BIDIRECTIONAL);
2640 if (!error)
2641 goto err;
2642 } else {
2643 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2644 DMA_TO_DEVICE);
2645 if (!error)
2646 goto err;
2647 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2648 DMA_FROM_DEVICE);
2649 if (!error) {
2650 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2651 DMA_TO_DEVICE);
2652 goto err;
2653 }
2654 }
2655
2656 return 0;
2657err:
2f47d580
HJ		 2658 return -ENOMEM;
2659}
6dad4e8a
AG		 2660
2661void chcr_cipher_dma_unmap(struct device *dev,
2662 struct ablkcipher_request *req)
2f47d580 2663{
2f47d580
HJ		 2664 if (req->src == req->dst) {
2665 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2666 DMA_BIDIRECTIONAL);
2667 } else {
2668 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2669 DMA_TO_DEVICE);
2670 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2671 DMA_FROM_DEVICE);
2672 }
2673}
2674
2debd332
HJ		 2675static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2676{
2677 __be32 data;
2678
2679 memset(block, 0, csize);
2680 block += csize;
2681
2682 if (csize >= 4)
2683 csize = 4;
2684 else if (msglen > (unsigned int)(1 << (8 * csize)))
2685 return -EOVERFLOW;
2686
2687 data = cpu_to_be32(msglen);
2688 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2689
2690 return 0;
2691}
2692
2693static void generate_b0(struct aead_request *req,
2694 struct chcr_aead_ctx *aeadctx,
2695 unsigned short op_type)
2696{
2697 unsigned int l, lp, m;
2698 int rc;
2699 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2700 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2701 u8 *b0 = reqctx->scratch_pad;
2702
2703 m = crypto_aead_authsize(aead);
2704
2705 memcpy(b0, reqctx->iv, 16);
2706
2707 lp = b0[0];
2708 l = lp + 1;
2709
2710 /* set m, bits 3-5 */
2711 *b0 |= (8 * ((m - 2) / 2));
2712
2713 /* set adata, bit 6, if associated data is used */
2714 if (req->assoclen)
2715 *b0 |= 64;
2716 rc = set_msg_len(b0 + 16 - l,
2717 (op_type == CHCR_DECRYPT_OP) ?
2718 req->cryptlen - m : req->cryptlen, l);
2719}
2720
2721static inline int crypto_ccm_check_iv(const u8 *iv)
2722{
2723 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2724 if (iv[0] < 1 || iv[0] > 7)
2725 return -EINVAL;
2726
2727 return 0;
2728}
2729
2730static int ccm_format_packet(struct aead_request *req,
2731 struct chcr_aead_ctx *aeadctx,
2732 unsigned int sub_type,
4262c98a
HJ		 2733 unsigned short op_type,
2734 unsigned int assoclen)
2debd332
HJ		 2735{
2736 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2737 int rc = 0;
2738
2debd332
HJ		 2739 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2740 reqctx->iv[0] = 3;
2741 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2742 memcpy(reqctx->iv + 4, req->iv, 8);
2743 memset(reqctx->iv + 12, 0, 4);
2debd332
HJ		 2744 } else {
2745 memcpy(reqctx->iv, req->iv, 16);
2debd332 2746 }
4262c98a
HJ		 2747 if (assoclen)
2748 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2749 htons(assoclen);
2750
2debd332
HJ		 2751 generate_b0(req, aeadctx, op_type);
2752 /* zero the ctr value */
2753 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2754 return rc;
2755}
2756
2757static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2758 unsigned int dst_size,
2759 struct aead_request *req,
2f47d580 2760 unsigned short op_type)
2debd332
HJ		 2761{
2762 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2763 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2764 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2765 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2f47d580 2766 unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id;
2debd332
HJ		 2767 unsigned int ccm_xtra;
2768 unsigned char tag_offset = 0, auth_offset = 0;
2debd332
HJ		 2769 unsigned int assoclen;
2770
2771 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2772 assoclen = req->assoclen - 8;
2773 else
2774 assoclen = req->assoclen;
2775 ccm_xtra = CCM_B0_SIZE +
2776 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2777
2778 auth_offset = req->cryptlen ?
2f47d580 2779 (assoclen + IV + 1 + ccm_xtra) : 0;
2debd332
HJ		 2780 if (op_type == CHCR_DECRYPT_OP) {
2781 if (crypto_aead_authsize(tfm) != req->cryptlen)
2782 tag_offset = crypto_aead_authsize(tfm);
2783 else
2784 auth_offset = 0;
2785 }
2786
2787
2788 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2f47d580 2789 2, assoclen + 1 + ccm_xtra);
2debd332 2790 sec_cpl->pldlen =
2f47d580 2791 htonl(assoclen + IV + req->cryptlen + ccm_xtra);
2debd332
HJ		 2792 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2793 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2794 1, assoclen + ccm_xtra, assoclen
2f47d580 2795 + IV + 1 + ccm_xtra, 0);
2debd332
HJ		 2796
2797 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2798 auth_offset, tag_offset,
2799 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2800 crypto_aead_authsize(tfm));
2801 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2802 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
0a7bd30c 2803 cipher_mode, mac_mode,
2f47d580 2804 aeadctx->hmac_ctrl, IV >> 1);
2debd332
HJ		 2805
2806 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2f47d580 2807 0, dst_size);
2debd332
HJ		 2808}
2809
1efb892b
CIK		 2810static int aead_ccm_validate_input(unsigned short op_type,
2811 struct aead_request *req,
2812 struct chcr_aead_ctx *aeadctx,
2813 unsigned int sub_type)
2debd332
HJ		 2814{
2815 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2816 if (crypto_ccm_check_iv(req->iv)) {
2817 pr_err("CCM: IV check fails\n");
2818 return -EINVAL;
2819 }
2820 } else {
2821 if (req->assoclen != 16 && req->assoclen != 20) {
2822 pr_err("RFC4309: Invalid AAD length %d\n",
2823 req->assoclen);
2824 return -EINVAL;
2825 }
2826 }
2debd332
HJ		 2827 return 0;
2828}
2829
2debd332
HJ		 2830static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2831 unsigned short qid,
4262c98a 2832 int size)
2debd332
HJ		 2833{
2834 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2835 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2836 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2837 struct sk_buff *skb = NULL;
2838 struct chcr_wr *chcr_req;
2839 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580
HJ		 2840 struct ulptx_sgl *ulptx;
2841 unsigned int transhdr_len;
2842 unsigned int dst_size = 0, kctx_len, dnents, temp;
2843 unsigned int sub_type, assoclen = req->assoclen;
2debd332 2844 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 2845 int error = -EINVAL;
2debd332
HJ		 2846 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2847 GFP_ATOMIC;
2f47d580 2848 struct adapter *adap = padap(a_ctx(tfm)->dev);
2debd332 2849
2f47d580
HJ		 2850 sub_type = get_aead_subtype(tfm);
2851 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2852 assoclen -= 8;
4262c98a
HJ		 2853 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2854 error = chcr_aead_common_init(req);
2f47d580
HJ		 2855 if (error)
2856 return ERR_PTR(error);
0e93708d 2857
4262c98a 2858 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
5fe8c711 2859 if (error)
2debd332 2860 goto err;
e1a018e6
HJ		 2861 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2862 dnents += sg_nents_xlen(req->dst, req->cryptlen
4262c98a 2863 + (reqctx->op ? -authsize : authsize),
e1a018e6
HJ		 2864 CHCR_DST_SG_SIZE, req->assoclen);
2865 dnents += MIN_CCM_SG; // For IV and B0
2f47d580 2866 dst_size = get_space_for_phys_dsgl(dnents);
125d01ca 2867 kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
2debd332 2868 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2f47d580
HJ		 2869 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen +
2870 reqctx->b0_len) <= SGE_MAX_WR_LEN;
125d01ca
HJ		 2871 temp = reqctx->imm ? roundup(assoclen + IV + req->cryptlen +
2872 reqctx->b0_len, 16) :
2f47d580
HJ		 2873 (sgl_len(reqctx->src_nents + reqctx->aad_nents +
2874 MIN_CCM_SG) * 8);
2875 transhdr_len += temp;
125d01ca 2876 transhdr_len = roundup(transhdr_len, 16);
2f47d580
HJ		 2877
2878 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
4262c98a 2879 reqctx->b0_len, transhdr_len, reqctx->op)) {
ee0863ba 2880 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 2881 chcr_aead_common_exit(req);
2882 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 2883 }
2f47d580 2884 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2debd332 2885
5fe8c711
HJ		 2886 if (!skb) {
2887 error = -ENOMEM;
2debd332 2888 goto err;
5fe8c711 2889 }
2debd332 2890
2f47d580 2891 chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len);
2debd332 2892
4262c98a 2893 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
2debd332
HJ		 2894
2895 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2896 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
125d01ca
HJ		 2897 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2898 aeadctx->key, aeadctx->enckey_len);
2debd332
HJ		 2899
2900 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2f47d580 2901 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
4262c98a 2902 error = ccm_format_packet(req, aeadctx, sub_type, reqctx->op, assoclen);
5fe8c711 2903 if (error)
2debd332 2904 goto dstmap_fail;
4262c98a
HJ		 2905 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
2906 chcr_add_aead_src_ent(req, ulptx, assoclen);
2debd332 2907
ee0863ba 2908 atomic_inc(&adap->chcr_stats.aead_rqst);
2f47d580
HJ		 2909 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2910 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen +
2911 reqctx->b0_len) : 0);
2912 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
2913 transhdr_len, temp, 0);
2debd332 2914 reqctx->skb = skb;
2f47d580 2915
2debd332
HJ		 2916 return skb;
2917dstmap_fail:
2918 kfree_skb(skb);
2debd332 2919err:
4262c98a 2920 chcr_aead_common_exit(req);
5fe8c711 2921 return ERR_PTR(error);
2debd332
HJ		 2922}
2923
2924static struct sk_buff *create_gcm_wr(struct aead_request *req,
2925 unsigned short qid,
4262c98a 2926 int size)
2debd332
HJ		 2927{
2928 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2929 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2930 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2931 struct sk_buff *skb = NULL;
2932 struct chcr_wr *chcr_req;
2933 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580
HJ		 2934 struct ulptx_sgl *ulptx;
2935 unsigned int transhdr_len, dnents = 0;
2936 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
2debd332 2937 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 2938 int error = -EINVAL;
2debd332
HJ		 2939 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2940 GFP_ATOMIC;
2f47d580 2941 struct adapter *adap = padap(a_ctx(tfm)->dev);
2debd332 2942
2f47d580
HJ		 2943 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2944 assoclen = req->assoclen - 8;
2debd332 2945
4262c98a
HJ		 2946 reqctx->b0_len = 0;
2947 error = chcr_aead_common_init(req);
e1a018e6
HJ		 2948 if (error)
2949 return ERR_PTR(error);
2950 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2951 dnents += sg_nents_xlen(req->dst, req->cryptlen +
4262c98a 2952 (reqctx->op ? -authsize : authsize),
2f47d580 2953 CHCR_DST_SG_SIZE, req->assoclen);
e1a018e6 2954 dnents += MIN_GCM_SG; // For IV
2f47d580 2955 dst_size = get_space_for_phys_dsgl(dnents);
125d01ca 2956 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
2debd332 2957 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2f47d580
HJ		 2958 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <=
2959 SGE_MAX_WR_LEN;
125d01ca
HJ		 2960 temp = reqctx->imm ? roundup(assoclen + IV + req->cryptlen, 16) :
2961 (sgl_len(reqctx->src_nents +
2962 reqctx->aad_nents + MIN_GCM_SG) * 8);
2f47d580 2963 transhdr_len += temp;
125d01ca 2964 transhdr_len = roundup(transhdr_len, 16);
2f47d580 2965 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
4262c98a
HJ		 2966 transhdr_len, reqctx->op)) {
2967
ee0863ba 2968 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 2969 chcr_aead_common_exit(req);
2970 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 2971 }
2f47d580 2972 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
5fe8c711
HJ		 2973 if (!skb) {
2974 error = -ENOMEM;
2debd332 2975 goto err;
5fe8c711 2976 }
2debd332 2977
de77b966 2978 chcr_req = __skb_put_zero(skb, transhdr_len);
2debd332 2979
2f47d580 2980 //Offset of tag from end
4262c98a 2981 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2debd332 2982 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2f47d580
HJ		 2983 a_ctx(tfm)->dev->rx_channel_id, 2,
2984 (assoclen + 1));
0e93708d 2985 chcr_req->sec_cpl.pldlen =
2f47d580 2986 htonl(assoclen + IV + req->cryptlen);
2debd332 2987 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
d600fc8a 2988 assoclen ? 1 : 0, assoclen,
2f47d580 2989 assoclen + IV + 1, 0);
e1a018e6 2990 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2f47d580
HJ		 2991 FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
2992 temp, temp);
e1a018e6 2993 chcr_req->sec_cpl.seqno_numivs =
4262c98a 2994 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
2debd332
HJ		 2995 CHCR_ENCRYPT_OP) ? 1 : 0,
2996 CHCR_SCMD_CIPHER_MODE_AES_GCM,
0a7bd30c 2997 CHCR_SCMD_AUTH_MODE_GHASH,
2f47d580 2998 aeadctx->hmac_ctrl, IV >> 1);
2debd332 2999 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2f47d580 3000 0, 0, dst_size);
2debd332
HJ		 3001 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3002 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
125d01ca
HJ		 3003 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3004 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2debd332
HJ		 3005
3006 /* prepare a 16 byte iv */
3007 /* S A L T | IV | 0x00000001 */
3008 if (get_aead_subtype(tfm) ==
3009 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
3010 memcpy(reqctx->iv, aeadctx->salt, 4);
8f6acb7f 3011 memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE);
2debd332 3012 } else {
8f6acb7f 3013 memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE);
2debd332
HJ		 3014 }
3015 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
3016
3017 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2f47d580 3018 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2debd332 3019
4262c98a
HJ		 3020 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, qid);
3021 chcr_add_aead_src_ent(req, ulptx, assoclen);
ee0863ba 3022 atomic_inc(&adap->chcr_stats.aead_rqst);
2f47d580
HJ		 3023 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
3024 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
3025 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
3026 transhdr_len, temp, reqctx->verify);
2debd332 3027 reqctx->skb = skb;
2debd332
HJ		 3028 return skb;
3029
2debd332 3030err:
4262c98a 3031 chcr_aead_common_exit(req);
5fe8c711 3032 return ERR_PTR(error);
2debd332
HJ		 3033}
3034
3035
3036
3037static int chcr_aead_cra_init(struct crypto_aead *tfm)
3038{
2f47d580 3039 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d
HJ		 3040 struct aead_alg *alg = crypto_aead_alg(tfm);
3041
3042 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
5fe8c711
HJ		 3043 CRYPTO_ALG_NEED_FALLBACK |
3044 CRYPTO_ALG_ASYNC);
0e93708d
HJ		 3045 if (IS_ERR(aeadctx->sw_cipher))
3046 return PTR_ERR(aeadctx->sw_cipher);
3047 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
3048 sizeof(struct aead_request) +
3049 crypto_aead_reqsize(aeadctx->sw_cipher)));
2f47d580 3050 return chcr_device_init(a_ctx(tfm));
2debd332
HJ		 3051}
3052
3053static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3054{
2f47d580 3055 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d 3056
0e93708d 3057 crypto_free_aead(aeadctx->sw_cipher);
2debd332
HJ		 3058}
3059
3060static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3061 unsigned int authsize)
3062{
2f47d580 3063 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3064
3065 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3066 aeadctx->mayverify = VERIFY_HW;
0e93708d 3067 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3068}
3069static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3070 unsigned int authsize)
3071{
2f47d580 3072 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3073 u32 maxauth = crypto_aead_maxauthsize(tfm);
3074
3075 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3076 * true for sha1. authsize == 12 condition should be before
3077 * authsize == (maxauth >> 1)
3078 */
3079 if (authsize == ICV_4) {
3080 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3081 aeadctx->mayverify = VERIFY_HW;
3082 } else if (authsize == ICV_6) {
3083 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3084 aeadctx->mayverify = VERIFY_HW;
3085 } else if (authsize == ICV_10) {
3086 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3087 aeadctx->mayverify = VERIFY_HW;
3088 } else if (authsize == ICV_12) {
3089 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3090 aeadctx->mayverify = VERIFY_HW;
3091 } else if (authsize == ICV_14) {
3092 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3093 aeadctx->mayverify = VERIFY_HW;
3094 } else if (authsize == (maxauth >> 1)) {
3095 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3096 aeadctx->mayverify = VERIFY_HW;
3097 } else if (authsize == maxauth) {
3098 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3099 aeadctx->mayverify = VERIFY_HW;
3100 } else {
3101 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3102 aeadctx->mayverify = VERIFY_SW;
3103 }
0e93708d 3104 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3105}
3106
3107
3108static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3109{
2f47d580 3110 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3111
3112 switch (authsize) {
3113 case ICV_4:
3114 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3115 aeadctx->mayverify = VERIFY_HW;
3116 break;
3117 case ICV_8:
3118 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3119 aeadctx->mayverify = VERIFY_HW;
3120 break;
3121 case ICV_12:
3122 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3123 aeadctx->mayverify = VERIFY_HW;
3124 break;
3125 case ICV_14:
3126 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3127 aeadctx->mayverify = VERIFY_HW;
3128 break;
3129 case ICV_16:
3130 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3131 aeadctx->mayverify = VERIFY_HW;
3132 break;
3133 case ICV_13:
3134 case ICV_15:
3135 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3136 aeadctx->mayverify = VERIFY_SW;
3137 break;
3138 default:
3139
3140 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
3141 CRYPTO_TFM_RES_BAD_KEY_LEN);
3142 return -EINVAL;
3143 }
0e93708d 3144 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3145}
3146
3147static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3148 unsigned int authsize)
3149{
2f47d580 3150 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3151
3152 switch (authsize) {
3153 case ICV_8:
3154 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3155 aeadctx->mayverify = VERIFY_HW;
3156 break;
3157 case ICV_12:
3158 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3159 aeadctx->mayverify = VERIFY_HW;
3160 break;
3161 case ICV_16:
3162 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3163 aeadctx->mayverify = VERIFY_HW;
3164 break;
3165 default:
3166 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3167 CRYPTO_TFM_RES_BAD_KEY_LEN);
3168 return -EINVAL;
3169 }
0e93708d 3170 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3171}
3172
3173static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3174 unsigned int authsize)
3175{
2f47d580 3176 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3177
3178 switch (authsize) {
3179 case ICV_4:
3180 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3181 aeadctx->mayverify = VERIFY_HW;
3182 break;
3183 case ICV_6:
3184 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3185 aeadctx->mayverify = VERIFY_HW;
3186 break;
3187 case ICV_8:
3188 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3189 aeadctx->mayverify = VERIFY_HW;
3190 break;
3191 case ICV_10:
3192 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3193 aeadctx->mayverify = VERIFY_HW;
3194 break;
3195 case ICV_12:
3196 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3197 aeadctx->mayverify = VERIFY_HW;
3198 break;
3199 case ICV_14:
3200 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3201 aeadctx->mayverify = VERIFY_HW;
3202 break;
3203 case ICV_16:
3204 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3205 aeadctx->mayverify = VERIFY_HW;
3206 break;
3207 default:
3208 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3209 CRYPTO_TFM_RES_BAD_KEY_LEN);
3210 return -EINVAL;
3211 }
0e93708d 3212 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3213}
3214
0e93708d 3215static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2debd332
HJ		 3216 const u8 *key,
3217 unsigned int keylen)
3218{
2f47d580 3219 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
2debd332
HJ		 3220 unsigned char ck_size, mk_size;
3221 int key_ctx_size = 0;
3222
125d01ca 3223 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
2debd332 3224 if (keylen == AES_KEYSIZE_128) {
2debd332 3225 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
125d01ca 3226 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
2debd332
HJ		 3227 } else if (keylen == AES_KEYSIZE_192) {
3228 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3229 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3230 } else if (keylen == AES_KEYSIZE_256) {
3231 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3232 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3233 } else {
3234 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3235 CRYPTO_TFM_RES_BAD_KEY_LEN);
3236 aeadctx->enckey_len = 0;
3237 return -EINVAL;
3238 }
3239 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3240 key_ctx_size >> 4);
0e93708d
HJ		 3241 memcpy(aeadctx->key, key, keylen);
3242 aeadctx->enckey_len = keylen;
3243
2debd332
HJ		 3244 return 0;
3245}
3246
0e93708d
HJ		 3247static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3248 const u8 *key,
3249 unsigned int keylen)
3250{
2f47d580 3251 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
0e93708d
HJ		 3252 int error;
3253
3254 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3255 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3256 CRYPTO_TFM_REQ_MASK);
3257 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3258 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3259 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3260 CRYPTO_TFM_RES_MASK);
3261 if (error)
3262 return error;
3263 return chcr_ccm_common_setkey(aead, key, keylen);
3264}
3265
2debd332
HJ		 3266static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3267 unsigned int keylen)
3268{
2f47d580 3269 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
4dbeae42 3270 int error;
2debd332
HJ		 3271
3272 if (keylen < 3) {
3273 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3274 CRYPTO_TFM_RES_BAD_KEY_LEN);
3275 aeadctx->enckey_len = 0;
3276 return -EINVAL;
3277 }
4dbeae42
HJ		 3278 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3279 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3280 CRYPTO_TFM_REQ_MASK);
3281 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3282 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3283 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3284 CRYPTO_TFM_RES_MASK);
3285 if (error)
3286 return error;
2debd332
HJ		 3287 keylen -= 3;
3288 memcpy(aeadctx->salt, key + keylen, 3);
0e93708d 3289 return chcr_ccm_common_setkey(aead, key, keylen);
2debd332
HJ		 3290}
3291
3292static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3293 unsigned int keylen)
3294{
2f47d580 3295 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
2debd332 3296 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
8356ea51 3297 struct crypto_cipher *cipher;
2debd332
HJ		 3298 unsigned int ck_size;
3299 int ret = 0, key_ctx_size = 0;
3300
0e93708d
HJ		 3301 aeadctx->enckey_len = 0;
3302 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3303 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3304 & CRYPTO_TFM_REQ_MASK);
3305 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3306 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3307 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3308 CRYPTO_TFM_RES_MASK);
3309 if (ret)
3310 goto out;
3311
7c2cf1c4
HJ		 3312 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3313 keylen > 3) {
2debd332
HJ		 3314 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3315 memcpy(aeadctx->salt, key + keylen, 4);
3316 }
3317 if (keylen == AES_KEYSIZE_128) {
3318 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3319 } else if (keylen == AES_KEYSIZE_192) {
3320 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3321 } else if (keylen == AES_KEYSIZE_256) {
3322 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3323 } else {
3324 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3325 CRYPTO_TFM_RES_BAD_KEY_LEN);
0e93708d 3326 pr_err("GCM: Invalid key length %d\n", keylen);
2debd332
HJ		 3327 ret = -EINVAL;
3328 goto out;
3329 }
3330
3331 memcpy(aeadctx->key, key, keylen);
3332 aeadctx->enckey_len = keylen;
125d01ca 3333 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
2debd332 3334 AEAD_H_SIZE;
125d01ca 3335 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2debd332
HJ		 3336 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3337 0, 0,
3338 key_ctx_size >> 4);
8356ea51
HJ		 3339 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3340 * It will go in key context
2debd332 3341 */
8356ea51
HJ		 3342 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
3343 if (IS_ERR(cipher)) {
2debd332
HJ		 3344 aeadctx->enckey_len = 0;
3345 ret = -ENOMEM;
3346 goto out;
3347 }
8356ea51
HJ		 3348
3349 ret = crypto_cipher_setkey(cipher, key, keylen);
2debd332
HJ		 3350 if (ret) {
3351 aeadctx->enckey_len = 0;
3352 goto out1;
3353 }
3354 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
8356ea51 3355 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2debd332
HJ		 3356
3357out1:
8356ea51 3358 crypto_free_cipher(cipher);
2debd332
HJ		 3359out:
3360 return ret;
3361}
3362
3363static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3364 unsigned int keylen)
3365{
2f47d580 3366 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
2debd332
HJ		 3367 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3368 /* it contains auth and cipher key both*/
3369 struct crypto_authenc_keys keys;
3d64bd67 3370 unsigned int bs, subtype;
2debd332
HJ		 3371 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3372 int err = 0, i, key_ctx_len = 0;
3373 unsigned char ck_size = 0;
3374 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
ec1bca94 3375 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2debd332
HJ		 3376 struct algo_param param;
3377 int align;
3378 u8 *o_ptr = NULL;
3379
0e93708d
HJ		 3380 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3381 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3382 & CRYPTO_TFM_REQ_MASK);
3383 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3384 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3385 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3386 & CRYPTO_TFM_RES_MASK);
3387 if (err)
3388 goto out;
3389
2debd332
HJ		 3390 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3391 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3392 goto out;
3393 }
3394
3395 if (get_alg_config(&param, max_authsize)) {
3396 pr_err("chcr : Unsupported digest size\n");
3397 goto out;
3398 }
3d64bd67
HJ		 3399 subtype = get_aead_subtype(authenc);
3400 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3401 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3402 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3403 goto out;
3404 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3405 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3406 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3407 }
2debd332
HJ		 3408 if (keys.enckeylen == AES_KEYSIZE_128) {
3409 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3410 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3411 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3412 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3413 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3414 } else {
3415 pr_err("chcr : Unsupported cipher key\n");
3416 goto out;
3417 }
3418
3419 /* Copy only encryption key. We use authkey to generate h(ipad) and
3420 * h(opad) so authkey is not needed again. authkeylen size have the
3421 * size of the hash digest size.
3422 */
3423 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3424 aeadctx->enckey_len = keys.enckeylen;
3d64bd67
HJ		 3425 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3426 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
2debd332 3427
3d64bd67
HJ		 3428 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3429 aeadctx->enckey_len << 3);
3430 }
2debd332
HJ		 3431 base_hash = chcr_alloc_shash(max_authsize);
3432 if (IS_ERR(base_hash)) {
3433 pr_err("chcr : Base driver cannot be loaded\n");
0e93708d 3434 aeadctx->enckey_len = 0;
eb526531 3435 memzero_explicit(&keys, sizeof(keys));
0e93708d 3436 return -EINVAL;
324429d7 3437 }
2debd332
HJ		 3438 {
3439 SHASH_DESC_ON_STACK(shash, base_hash);
6faa0f57 3440
2debd332
HJ		 3441 shash->tfm = base_hash;
3442 shash->flags = crypto_shash_get_flags(base_hash);
3443 bs = crypto_shash_blocksize(base_hash);
3444 align = KEYCTX_ALIGN_PAD(max_authsize);
3445 o_ptr = actx->h_iopad + param.result_size + align;
3446
3447 if (keys.authkeylen > bs) {
3448 err = crypto_shash_digest(shash, keys.authkey,
3449 keys.authkeylen,
3450 o_ptr);
3451 if (err) {
3452 pr_err("chcr : Base driver cannot be loaded\n");
3453 goto out;
3454 }
3455 keys.authkeylen = max_authsize;
3456 } else
3457 memcpy(o_ptr, keys.authkey, keys.authkeylen);
3458
3459 /* Compute the ipad-digest*/
3460 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3461 memcpy(pad, o_ptr, keys.authkeylen);
3462 for (i = 0; i < bs >> 2; i++)
3463 *((unsigned int *)pad + i) ^= IPAD_DATA;
3464
3465 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3466 max_authsize))
3467 goto out;
3468 /* Compute the opad-digest */
3469 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3470 memcpy(pad, o_ptr, keys.authkeylen);
3471 for (i = 0; i < bs >> 2; i++)
3472 *((unsigned int *)pad + i) ^= OPAD_DATA;
3473
3474 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3475 goto out;
3476
3477 /* convert the ipad and opad digest to network order */
3478 chcr_change_order(actx->h_iopad, param.result_size);
3479 chcr_change_order(o_ptr, param.result_size);
3480 key_ctx_len = sizeof(struct _key_ctx) +
125d01ca 3481 roundup(keys.enckeylen, 16) +
2debd332
HJ		 3482 (param.result_size + align) * 2;
3483 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3484 0, 1, key_ctx_len >> 4);
3485 actx->auth_mode = param.auth_mode;
3486 chcr_free_shash(base_hash);
3487
eb526531 3488 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3489 return 0;
3490 }
3491out:
3492 aeadctx->enckey_len = 0;
eb526531 3493 memzero_explicit(&keys, sizeof(keys));
ec1bca94 3494 if (!IS_ERR(base_hash))
2debd332
HJ		 3495 chcr_free_shash(base_hash);
3496 return -EINVAL;
324429d7
HS		 3497}
3498
2debd332
HJ		 3499static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3500 const u8 *key, unsigned int keylen)
3501{
2f47d580 3502 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
2debd332
HJ		 3503 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3504 struct crypto_authenc_keys keys;
0e93708d 3505 int err;
2debd332 3506 /* it contains auth and cipher key both*/
3d64bd67 3507 unsigned int subtype;
2debd332
HJ		 3508 int key_ctx_len = 0;
3509 unsigned char ck_size = 0;
3510
0e93708d
HJ		 3511 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3512 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3513 & CRYPTO_TFM_REQ_MASK);
3514 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3515 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3516 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3517 & CRYPTO_TFM_RES_MASK);
3518 if (err)
3519 goto out;
3520
2debd332
HJ		 3521 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3522 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3523 goto out;
3524 }
3d64bd67
HJ		 3525 subtype = get_aead_subtype(authenc);
3526 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3527 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3528 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3529 goto out;
3530 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3531 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3532 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3533 }
2debd332
HJ		 3534 if (keys.enckeylen == AES_KEYSIZE_128) {
3535 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3536 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3537 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3538 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3539 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3540 } else {
3d64bd67 3541 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen);
2debd332
HJ		 3542 goto out;
3543 }
3544 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3545 aeadctx->enckey_len = keys.enckeylen;
3d64bd67
HJ		 3546 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3547 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3548 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3549 aeadctx->enckey_len << 3);
3550 }
125d01ca 3551 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
2debd332
HJ		 3552
3553 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3554 0, key_ctx_len >> 4);
3555 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
eb526531 3556 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3557 return 0;
3558out:
3559 aeadctx->enckey_len = 0;
eb526531 3560 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3561 return -EINVAL;
3562}
6dad4e8a
AG		 3563
3564static int chcr_aead_op(struct aead_request *req,
6dad4e8a
AG		 3565 int size,
3566 create_wr_t create_wr_fn)
3567{
3568 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3569 struct uld_ctx *u_ctx;
3570 struct sk_buff *skb;
6faa0f57 3571 int isfull = 0;
6dad4e8a
AG		 3572
3573 if (!a_ctx(tfm)->dev) {
3574 pr_err("chcr : %s : No crypto device.\n", __func__);
3575 return -ENXIO;
3576 }
3577 u_ctx = ULD_CTX(a_ctx(tfm));
3578 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3579 a_ctx(tfm)->tx_qidx)) {
6faa0f57 3580 isfull = 1;
6dad4e8a 3581 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
6faa0f57 3582 return -ENOSPC;
6dad4e8a
AG		 3583 }
3584
3585 /* Form a WR from req */
4262c98a 3586 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size);
6dad4e8a
AG		 3587
3588 if (IS_ERR(skb) || !skb)
3589 return PTR_ERR(skb);
3590
3591 skb->dev = u_ctx->lldi.ports[0];
3592 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
3593 chcr_send_wr(skb);
6faa0f57 3594 return isfull ? -EBUSY : -EINPROGRESS;
6dad4e8a
AG		 3595}
3596
2debd332
HJ		 3597static int chcr_aead_encrypt(struct aead_request *req)
3598{
3599 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3600 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3601
3602 reqctx->verify = VERIFY_HW;
4262c98a 3603 reqctx->op = CHCR_ENCRYPT_OP;
2debd332
HJ		 3604
3605 switch (get_aead_subtype(tfm)) {
3d64bd67
HJ		 3606 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3607 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3608 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3609 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
4262c98a 3610 return chcr_aead_op(req, 0, create_authenc_wr);
2debd332
HJ		 3611 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3612 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
4262c98a 3613 return chcr_aead_op(req, 0, create_aead_ccm_wr);
2debd332 3614 default:
4262c98a 3615 return chcr_aead_op(req, 0, create_gcm_wr);
2debd332
HJ		 3616 }
3617}
3618
3619static int chcr_aead_decrypt(struct aead_request *req)
3620{
3621 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 3622 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3623 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3624 int size;
3625
3626 if (aeadctx->mayverify == VERIFY_SW) {
3627 size = crypto_aead_maxauthsize(tfm);
3628 reqctx->verify = VERIFY_SW;
3629 } else {
3630 size = 0;
3631 reqctx->verify = VERIFY_HW;
3632 }
4262c98a 3633 reqctx->op = CHCR_DECRYPT_OP;
2debd332 3634 switch (get_aead_subtype(tfm)) {
3d64bd67
HJ		 3635 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3636 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3637 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3638 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
4262c98a 3639 return chcr_aead_op(req, size, create_authenc_wr);
2debd332
HJ		 3640 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3641 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
4262c98a 3642 return chcr_aead_op(req, size, create_aead_ccm_wr);
2debd332 3643 default:
4262c98a 3644 return chcr_aead_op(req, size, create_gcm_wr);
2debd332
HJ		 3645 }
3646}
3647
324429d7
HS		 3648static struct chcr_alg_template driver_algs[] = {
3649 /* AES-CBC */
3650 {
b8fd1f41 3651 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
324429d7
HS		 3652 .is_registered = 0,
3653 .alg.crypto = {
3654 .cra_name = "cbc(aes)",
2debd332 3655 .cra_driver_name = "cbc-aes-chcr",
324429d7 3656 .cra_blocksize = AES_BLOCK_SIZE,
324429d7 3657 .cra_init = chcr_cra_init,
b8fd1f41 3658 .cra_exit = chcr_cra_exit,
324429d7
HS		 3659 .cra_u.ablkcipher = {
3660 .min_keysize = AES_MIN_KEY_SIZE,
3661 .max_keysize = AES_MAX_KEY_SIZE,
3662 .ivsize = AES_BLOCK_SIZE,
3663 .setkey = chcr_aes_cbc_setkey,
3664 .encrypt = chcr_aes_encrypt,
3665 .decrypt = chcr_aes_decrypt,
3666 }
3667 }
3668 },
3669 {
b8fd1f41 3670 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
324429d7
HS		 3671 .is_registered = 0,
3672 .alg.crypto = {
3673 .cra_name = "xts(aes)",
2debd332 3674 .cra_driver_name = "xts-aes-chcr",
324429d7 3675 .cra_blocksize = AES_BLOCK_SIZE,
324429d7
HS		 3676 .cra_init = chcr_cra_init,
3677 .cra_exit = NULL,
b8fd1f41 3678 .cra_u .ablkcipher = {
324429d7
HS		 3679 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3680 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3681 .ivsize = AES_BLOCK_SIZE,
3682 .setkey = chcr_aes_xts_setkey,
3683 .encrypt = chcr_aes_encrypt,
3684 .decrypt = chcr_aes_decrypt,
3685 }
3686 }
b8fd1f41
HJ		 3687 },
3688 {
3689 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3690 .is_registered = 0,
3691 .alg.crypto = {
3692 .cra_name = "ctr(aes)",
3693 .cra_driver_name = "ctr-aes-chcr",
3694 .cra_blocksize = 1,
3695 .cra_init = chcr_cra_init,
3696 .cra_exit = chcr_cra_exit,
3697 .cra_u.ablkcipher = {
3698 .min_keysize = AES_MIN_KEY_SIZE,
3699 .max_keysize = AES_MAX_KEY_SIZE,
3700 .ivsize = AES_BLOCK_SIZE,
3701 .setkey = chcr_aes_ctr_setkey,
3702 .encrypt = chcr_aes_encrypt,
3703 .decrypt = chcr_aes_decrypt,
3704 }
3705 }
3706 },
3707 {
3708 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3709 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3710 .is_registered = 0,
3711 .alg.crypto = {
3712 .cra_name = "rfc3686(ctr(aes))",
3713 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3714 .cra_blocksize = 1,
3715 .cra_init = chcr_rfc3686_init,
3716 .cra_exit = chcr_cra_exit,
3717 .cra_u.ablkcipher = {
3718 .min_keysize = AES_MIN_KEY_SIZE +
3719 CTR_RFC3686_NONCE_SIZE,
3720 .max_keysize = AES_MAX_KEY_SIZE +
3721 CTR_RFC3686_NONCE_SIZE,
3722 .ivsize = CTR_RFC3686_IV_SIZE,
3723 .setkey = chcr_aes_rfc3686_setkey,
3724 .encrypt = chcr_aes_encrypt,
3725 .decrypt = chcr_aes_decrypt,
3726 .geniv = "seqiv",
3727 }
324429d7
HS		 3728 }
3729 },
3730 /* SHA */
3731 {
3732 .type = CRYPTO_ALG_TYPE_AHASH,
3733 .is_registered = 0,
3734 .alg.hash = {
3735 .halg.digestsize = SHA1_DIGEST_SIZE,
3736 .halg.base = {
3737 .cra_name = "sha1",
3738 .cra_driver_name = "sha1-chcr",
3739 .cra_blocksize = SHA1_BLOCK_SIZE,
3740 }
3741 }
3742 },
3743 {
3744 .type = CRYPTO_ALG_TYPE_AHASH,
3745 .is_registered = 0,
3746 .alg.hash = {
3747 .halg.digestsize = SHA256_DIGEST_SIZE,
3748 .halg.base = {
3749 .cra_name = "sha256",
3750 .cra_driver_name = "sha256-chcr",
3751 .cra_blocksize = SHA256_BLOCK_SIZE,
3752 }
3753 }
3754 },
3755 {
3756 .type = CRYPTO_ALG_TYPE_AHASH,
3757 .is_registered = 0,
3758 .alg.hash = {
3759 .halg.digestsize = SHA224_DIGEST_SIZE,
3760 .halg.base = {
3761 .cra_name = "sha224",
3762 .cra_driver_name = "sha224-chcr",
3763 .cra_blocksize = SHA224_BLOCK_SIZE,
3764 }
3765 }
3766 },
3767 {
3768 .type = CRYPTO_ALG_TYPE_AHASH,
3769 .is_registered = 0,
3770 .alg.hash = {
3771 .halg.digestsize = SHA384_DIGEST_SIZE,
3772 .halg.base = {
3773 .cra_name = "sha384",
3774 .cra_driver_name = "sha384-chcr",
3775 .cra_blocksize = SHA384_BLOCK_SIZE,
3776 }
3777 }
3778 },
3779 {
3780 .type = CRYPTO_ALG_TYPE_AHASH,
3781 .is_registered = 0,
3782 .alg.hash = {
3783 .halg.digestsize = SHA512_DIGEST_SIZE,
3784 .halg.base = {
3785 .cra_name = "sha512",
3786 .cra_driver_name = "sha512-chcr",
3787 .cra_blocksize = SHA512_BLOCK_SIZE,
3788 }
3789 }
3790 },
3791 /* HMAC */
3792 {
3793 .type = CRYPTO_ALG_TYPE_HMAC,
3794 .is_registered = 0,
3795 .alg.hash = {
3796 .halg.digestsize = SHA1_DIGEST_SIZE,
3797 .halg.base = {
3798 .cra_name = "hmac(sha1)",
2debd332 3799 .cra_driver_name = "hmac-sha1-chcr",
324429d7
HS		 3800 .cra_blocksize = SHA1_BLOCK_SIZE,
3801 }
3802 }
3803 },
3804 {
3805 .type = CRYPTO_ALG_TYPE_HMAC,
3806 .is_registered = 0,
3807 .alg.hash = {
3808 .halg.digestsize = SHA224_DIGEST_SIZE,
3809 .halg.base = {
3810 .cra_name = "hmac(sha224)",
2debd332 3811 .cra_driver_name = "hmac-sha224-chcr",
324429d7
HS		 3812 .cra_blocksize = SHA224_BLOCK_SIZE,
3813 }
3814 }
3815 },
3816 {
3817 .type = CRYPTO_ALG_TYPE_HMAC,
3818 .is_registered = 0,
3819 .alg.hash = {
3820 .halg.digestsize = SHA256_DIGEST_SIZE,
3821 .halg.base = {
3822 .cra_name = "hmac(sha256)",
2debd332 3823 .cra_driver_name = "hmac-sha256-chcr",
324429d7
HS		 3824 .cra_blocksize = SHA256_BLOCK_SIZE,
3825 }
3826 }
3827 },
3828 {
3829 .type = CRYPTO_ALG_TYPE_HMAC,
3830 .is_registered = 0,
3831 .alg.hash = {
3832 .halg.digestsize = SHA384_DIGEST_SIZE,
3833 .halg.base = {
3834 .cra_name = "hmac(sha384)",
2debd332 3835 .cra_driver_name = "hmac-sha384-chcr",
324429d7
HS		 3836 .cra_blocksize = SHA384_BLOCK_SIZE,
3837 }
3838 }
3839 },
3840 {
3841 .type = CRYPTO_ALG_TYPE_HMAC,
3842 .is_registered = 0,
3843 .alg.hash = {
3844 .halg.digestsize = SHA512_DIGEST_SIZE,
3845 .halg.base = {
3846 .cra_name = "hmac(sha512)",
2debd332 3847 .cra_driver_name = "hmac-sha512-chcr",
324429d7
HS		 3848 .cra_blocksize = SHA512_BLOCK_SIZE,
3849 }
3850 }
3851 },
2debd332
HJ		 3852 /* Add AEAD Algorithms */
3853 {
3854 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3855 .is_registered = 0,
3856 .alg.aead = {
3857 .base = {
3858 .cra_name = "gcm(aes)",
3859 .cra_driver_name = "gcm-aes-chcr",
3860 .cra_blocksize = 1,
e29abda5 3861 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3862 .cra_ctxsize = sizeof(struct chcr_context) +
3863 sizeof(struct chcr_aead_ctx) +
3864 sizeof(struct chcr_gcm_ctx),
3865 },
8f6acb7f 3866 .ivsize = GCM_AES_IV_SIZE,
2debd332
HJ		 3867 .maxauthsize = GHASH_DIGEST_SIZE,
3868 .setkey = chcr_gcm_setkey,
3869 .setauthsize = chcr_gcm_setauthsize,
3870 }
3871 },
3872 {
3873 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3874 .is_registered = 0,
3875 .alg.aead = {
3876 .base = {
3877 .cra_name = "rfc4106(gcm(aes))",
3878 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3879 .cra_blocksize = 1,
e29abda5 3880 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 3881 .cra_ctxsize = sizeof(struct chcr_context) +
3882 sizeof(struct chcr_aead_ctx) +
3883 sizeof(struct chcr_gcm_ctx),
3884
3885 },
8f6acb7f 3886 .ivsize = GCM_RFC4106_IV_SIZE,
2debd332
HJ		 3887 .maxauthsize = GHASH_DIGEST_SIZE,
3888 .setkey = chcr_gcm_setkey,
3889 .setauthsize = chcr_4106_4309_setauthsize,
3890 }
3891 },
3892 {
3893 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3894 .is_registered = 0,
3895 .alg.aead = {
3896 .base = {
3897 .cra_name = "ccm(aes)",
3898 .cra_driver_name = "ccm-aes-chcr",
3899 .cra_blocksize = 1,
e29abda5 3900 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3901 .cra_ctxsize = sizeof(struct chcr_context) +
3902 sizeof(struct chcr_aead_ctx),
3903
3904 },
3905 .ivsize = AES_BLOCK_SIZE,
3906 .maxauthsize = GHASH_DIGEST_SIZE,
3907 .setkey = chcr_aead_ccm_setkey,
3908 .setauthsize = chcr_ccm_setauthsize,
3909 }
3910 },
3911 {
3912 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3913 .is_registered = 0,
3914 .alg.aead = {
3915 .base = {
3916 .cra_name = "rfc4309(ccm(aes))",
3917 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3918 .cra_blocksize = 1,
e29abda5 3919 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 3920 .cra_ctxsize = sizeof(struct chcr_context) +
3921 sizeof(struct chcr_aead_ctx),
3922
3923 },
3924 .ivsize = 8,
3925 .maxauthsize = GHASH_DIGEST_SIZE,
3926 .setkey = chcr_aead_rfc4309_setkey,
3927 .setauthsize = chcr_4106_4309_setauthsize,
3928 }
3929 },
3930 {
3d64bd67 3931 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 3932 .is_registered = 0,
3933 .alg.aead = {
3934 .base = {
3935 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3936 .cra_driver_name =
3937 "authenc-hmac-sha1-cbc-aes-chcr",
3938 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 3939 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3940 .cra_ctxsize = sizeof(struct chcr_context) +
3941 sizeof(struct chcr_aead_ctx) +
3942 sizeof(struct chcr_authenc_ctx),
3943
3944 },
3945 .ivsize = AES_BLOCK_SIZE,
3946 .maxauthsize = SHA1_DIGEST_SIZE,
3947 .setkey = chcr_authenc_setkey,
3948 .setauthsize = chcr_authenc_setauthsize,
3949 }
3950 },
3951 {
3d64bd67 3952 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 3953 .is_registered = 0,
3954 .alg.aead = {
3955 .base = {
3956
3957 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3958 .cra_driver_name =
3959 "authenc-hmac-sha256-cbc-aes-chcr",
3960 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 3961 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3962 .cra_ctxsize = sizeof(struct chcr_context) +
3963 sizeof(struct chcr_aead_ctx) +
3964 sizeof(struct chcr_authenc_ctx),
3965
3966 },
3967 .ivsize = AES_BLOCK_SIZE,
3968 .maxauthsize = SHA256_DIGEST_SIZE,
3969 .setkey = chcr_authenc_setkey,
3970 .setauthsize = chcr_authenc_setauthsize,
3971 }
3972 },
3973 {
3d64bd67 3974 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 3975 .is_registered = 0,
3976 .alg.aead = {
3977 .base = {
3978 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3979 .cra_driver_name =
3980 "authenc-hmac-sha224-cbc-aes-chcr",
3981 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 3982 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3983 .cra_ctxsize = sizeof(struct chcr_context) +
3984 sizeof(struct chcr_aead_ctx) +
3985 sizeof(struct chcr_authenc_ctx),
3986 },
3987 .ivsize = AES_BLOCK_SIZE,
3988 .maxauthsize = SHA224_DIGEST_SIZE,
3989 .setkey = chcr_authenc_setkey,
3990 .setauthsize = chcr_authenc_setauthsize,
3991 }
3992 },
3993 {
3d64bd67 3994 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 3995 .is_registered = 0,
3996 .alg.aead = {
3997 .base = {
3998 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3999 .cra_driver_name =
4000 "authenc-hmac-sha384-cbc-aes-chcr",
4001 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4002 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4003 .cra_ctxsize = sizeof(struct chcr_context) +
4004 sizeof(struct chcr_aead_ctx) +
4005 sizeof(struct chcr_authenc_ctx),
4006
4007 },
4008 .ivsize = AES_BLOCK_SIZE,
4009 .maxauthsize = SHA384_DIGEST_SIZE,
4010 .setkey = chcr_authenc_setkey,
4011 .setauthsize = chcr_authenc_setauthsize,
4012 }
4013 },
4014 {
3d64bd67 4015 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4016 .is_registered = 0,
4017 .alg.aead = {
4018 .base = {
4019 .cra_name = "authenc(hmac(sha512),cbc(aes))",
4020 .cra_driver_name =
4021 "authenc-hmac-sha512-cbc-aes-chcr",
4022 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4023 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4024 .cra_ctxsize = sizeof(struct chcr_context) +
4025 sizeof(struct chcr_aead_ctx) +
4026 sizeof(struct chcr_authenc_ctx),
4027
4028 },
4029 .ivsize = AES_BLOCK_SIZE,
4030 .maxauthsize = SHA512_DIGEST_SIZE,
4031 .setkey = chcr_authenc_setkey,
4032 .setauthsize = chcr_authenc_setauthsize,
4033 }
4034 },
4035 {
3d64bd67 4036 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
2debd332
HJ		 4037 .is_registered = 0,
4038 .alg.aead = {
4039 .base = {
4040 .cra_name = "authenc(digest_null,cbc(aes))",
4041 .cra_driver_name =
4042 "authenc-digest_null-cbc-aes-chcr",
4043 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4044 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4045 .cra_ctxsize = sizeof(struct chcr_context) +
4046 sizeof(struct chcr_aead_ctx) +
4047 sizeof(struct chcr_authenc_ctx),
4048
4049 },
4050 .ivsize = AES_BLOCK_SIZE,
4051 .maxauthsize = 0,
4052 .setkey = chcr_aead_digest_null_setkey,
4053 .setauthsize = chcr_authenc_null_setauthsize,
4054 }
4055 },
3d64bd67
HJ		 4056 {
4057 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4058 .is_registered = 0,
4059 .alg.aead = {
4060 .base = {
4061 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4062 .cra_driver_name =
4063 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4064 .cra_blocksize = 1,
4065 .cra_priority = CHCR_AEAD_PRIORITY,
4066 .cra_ctxsize = sizeof(struct chcr_context) +
4067 sizeof(struct chcr_aead_ctx) +
4068 sizeof(struct chcr_authenc_ctx),
4069
4070 },
4071 .ivsize = CTR_RFC3686_IV_SIZE,
4072 .maxauthsize = SHA1_DIGEST_SIZE,
4073 .setkey = chcr_authenc_setkey,
4074 .setauthsize = chcr_authenc_setauthsize,
4075 }
4076 },
4077 {
4078 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4079 .is_registered = 0,
4080 .alg.aead = {
4081 .base = {
4082
4083 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4084 .cra_driver_name =
4085 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4086 .cra_blocksize = 1,
4087 .cra_priority = CHCR_AEAD_PRIORITY,
4088 .cra_ctxsize = sizeof(struct chcr_context) +
4089 sizeof(struct chcr_aead_ctx) +
4090 sizeof(struct chcr_authenc_ctx),
4091
4092 },
4093 .ivsize = CTR_RFC3686_IV_SIZE,
4094 .maxauthsize = SHA256_DIGEST_SIZE,
4095 .setkey = chcr_authenc_setkey,
4096 .setauthsize = chcr_authenc_setauthsize,
4097 }
4098 },
4099 {
4100 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4101 .is_registered = 0,
4102 .alg.aead = {
4103 .base = {
4104 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4105 .cra_driver_name =
4106 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4107 .cra_blocksize = 1,
4108 .cra_priority = CHCR_AEAD_PRIORITY,
4109 .cra_ctxsize = sizeof(struct chcr_context) +
4110 sizeof(struct chcr_aead_ctx) +
4111 sizeof(struct chcr_authenc_ctx),
4112 },
4113 .ivsize = CTR_RFC3686_IV_SIZE,
4114 .maxauthsize = SHA224_DIGEST_SIZE,
4115 .setkey = chcr_authenc_setkey,
4116 .setauthsize = chcr_authenc_setauthsize,
4117 }
4118 },
4119 {
4120 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4121 .is_registered = 0,
4122 .alg.aead = {
4123 .base = {
4124 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4125 .cra_driver_name =
4126 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4127 .cra_blocksize = 1,
4128 .cra_priority = CHCR_AEAD_PRIORITY,
4129 .cra_ctxsize = sizeof(struct chcr_context) +
4130 sizeof(struct chcr_aead_ctx) +
4131 sizeof(struct chcr_authenc_ctx),
4132
4133 },
4134 .ivsize = CTR_RFC3686_IV_SIZE,
4135 .maxauthsize = SHA384_DIGEST_SIZE,
4136 .setkey = chcr_authenc_setkey,
4137 .setauthsize = chcr_authenc_setauthsize,
4138 }
4139 },
4140 {
4141 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4142 .is_registered = 0,
4143 .alg.aead = {
4144 .base = {
4145 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4146 .cra_driver_name =
4147 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4148 .cra_blocksize = 1,
4149 .cra_priority = CHCR_AEAD_PRIORITY,
4150 .cra_ctxsize = sizeof(struct chcr_context) +
4151 sizeof(struct chcr_aead_ctx) +
4152 sizeof(struct chcr_authenc_ctx),
4153
4154 },
4155 .ivsize = CTR_RFC3686_IV_SIZE,
4156 .maxauthsize = SHA512_DIGEST_SIZE,
4157 .setkey = chcr_authenc_setkey,
4158 .setauthsize = chcr_authenc_setauthsize,
4159 }
4160 },
4161 {
4162 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4163 .is_registered = 0,
4164 .alg.aead = {
4165 .base = {
4166 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4167 .cra_driver_name =
4168 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4169 .cra_blocksize = 1,
4170 .cra_priority = CHCR_AEAD_PRIORITY,
4171 .cra_ctxsize = sizeof(struct chcr_context) +
4172 sizeof(struct chcr_aead_ctx) +
4173 sizeof(struct chcr_authenc_ctx),
4174
4175 },
4176 .ivsize = CTR_RFC3686_IV_SIZE,
4177 .maxauthsize = 0,
4178 .setkey = chcr_aead_digest_null_setkey,
4179 .setauthsize = chcr_authenc_null_setauthsize,
4180 }
4181 },
4182
324429d7
HS		 4183};
4184
4185/*
4186 * chcr_unregister_alg - Deregister crypto algorithms with
4187 * kernel framework.
4188 */
4189static int chcr_unregister_alg(void)
4190{
4191 int i;
4192
4193 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4194 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4195 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4196 if (driver_algs[i].is_registered)
4197 crypto_unregister_alg(
4198 &driver_algs[i].alg.crypto);
4199 break;
2debd332
HJ		 4200 case CRYPTO_ALG_TYPE_AEAD:
4201 if (driver_algs[i].is_registered)
4202 crypto_unregister_aead(
4203 &driver_algs[i].alg.aead);
4204 break;
324429d7
HS		 4205 case CRYPTO_ALG_TYPE_AHASH:
4206 if (driver_algs[i].is_registered)
4207 crypto_unregister_ahash(
4208 &driver_algs[i].alg.hash);
4209 break;
4210 }
4211 driver_algs[i].is_registered = 0;
4212 }
4213 return 0;
4214}
4215
4216#define SZ_AHASH_CTX sizeof(struct chcr_context)
4217#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4218#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
324429d7
HS		 4219
4220/*
4221 * chcr_register_alg - Register crypto algorithms with kernel framework.
4222 */
4223static int chcr_register_alg(void)
4224{
4225 struct crypto_alg ai;
4226 struct ahash_alg *a_hash;
4227 int err = 0, i;
4228 char *name = NULL;
4229
4230 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4231 if (driver_algs[i].is_registered)
4232 continue;
4233 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4234 case CRYPTO_ALG_TYPE_ABLKCIPHER:
b8fd1f41
HJ		 4235 driver_algs[i].alg.crypto.cra_priority =
4236 CHCR_CRA_PRIORITY;
4237 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
4238 driver_algs[i].alg.crypto.cra_flags =
4239 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
4240 CRYPTO_ALG_NEED_FALLBACK;
4241 driver_algs[i].alg.crypto.cra_ctxsize =
4242 sizeof(struct chcr_context) +
4243 sizeof(struct ablk_ctx);
4244 driver_algs[i].alg.crypto.cra_alignmask = 0;
4245 driver_algs[i].alg.crypto.cra_type =
4246 &crypto_ablkcipher_type;
324429d7
HS		 4247 err = crypto_register_alg(&driver_algs[i].alg.crypto);
4248 name = driver_algs[i].alg.crypto.cra_driver_name;
4249 break;
2debd332 4250 case CRYPTO_ALG_TYPE_AEAD:
2debd332 4251 driver_algs[i].alg.aead.base.cra_flags =
3f4a537a 4252 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
2debd332
HJ		 4253 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4254 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4255 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4256 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4257 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4258 err = crypto_register_aead(&driver_algs[i].alg.aead);
4259 name = driver_algs[i].alg.aead.base.cra_driver_name;
4260 break;
324429d7
HS		 4261 case CRYPTO_ALG_TYPE_AHASH:
4262 a_hash = &driver_algs[i].alg.hash;
4263 a_hash->update = chcr_ahash_update;
4264 a_hash->final = chcr_ahash_final;
4265 a_hash->finup = chcr_ahash_finup;
4266 a_hash->digest = chcr_ahash_digest;
4267 a_hash->export = chcr_ahash_export;
4268 a_hash->import = chcr_ahash_import;
4269 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4270 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4271 a_hash->halg.base.cra_module = THIS_MODULE;
6a38f622 4272 a_hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC;
324429d7
HS		 4273 a_hash->halg.base.cra_alignmask = 0;
4274 a_hash->halg.base.cra_exit = NULL;
324429d7
HS		 4275
4276 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4277 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4278 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
4279 a_hash->init = chcr_hmac_init;
4280 a_hash->setkey = chcr_ahash_setkey;
4281 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4282 } else {
4283 a_hash->init = chcr_sha_init;
4284 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4285 a_hash->halg.base.cra_init = chcr_sha_cra_init;
4286 }
4287 err = crypto_register_ahash(&driver_algs[i].alg.hash);
4288 ai = driver_algs[i].alg.hash.halg.base;
4289 name = ai.cra_driver_name;
4290 break;
4291 }
4292 if (err) {
4293 pr_err("chcr : %s : Algorithm registration failed\n",
4294 name);
4295 goto register_err;
4296 } else {
4297 driver_algs[i].is_registered = 1;
4298 }
4299 }
4300 return 0;
4301
4302register_err:
4303 chcr_unregister_alg();
4304 return err;
4305}
4306
4307/*
4308 * start_crypto - Register the crypto algorithms.
4309 * This should called once when the first device comesup. After this
4310 * kernel will start calling driver APIs for crypto operations.
4311 */
4312int start_crypto(void)
4313{
4314 return chcr_register_alg();
4315}
4316
4317/*
4318 * stop_crypto - Deregister all the crypto algorithms with kernel.
4319 * This should be called once when the last device goes down. After this
4320 * kernel will not call the driver API for crypto operations.
4321 */
4322int stop_crypto(void)
4323{
4324 chcr_unregister_alg();
4325 return 0;
4326}
Linux 6.x block layer and io_uring tree(s)