projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: chcr - Fix fallback key setting
[linux-2.6-block.git] / drivers / crypto / chelsio / chcr_algo.c
CommitLineData
324429d7
HS		 1/*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42#define pr_fmt(fmt) "chcr:" fmt
43
44#include <linux/kernel.h>
45#include <linux/module.h>
46#include <linux/crypto.h>
47#include <linux/cryptohash.h>
48#include <linux/skbuff.h>
49#include <linux/rtnetlink.h>
50#include <linux/highmem.h>
51#include <linux/scatterlist.h>
52
53#include <crypto/aes.h>
54#include <crypto/algapi.h>
55#include <crypto/hash.h>
56#include <crypto/sha.h>
2debd332
HJ		 57#include <crypto/authenc.h>
58#include <crypto/internal/aead.h>
59#include <crypto/null.h>
60#include <crypto/internal/skcipher.h>
61#include <crypto/aead.h>
62#include <crypto/scatterwalk.h>
324429d7
HS		 63#include <crypto/internal/hash.h>
64
65#include "t4fw_api.h"
66#include "t4_msg.h"
67#include "chcr_core.h"
68#include "chcr_algo.h"
69#include "chcr_crypto.h"
70
2debd332
HJ		 71static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
72{
73 return ctx->crypto_ctx->aeadctx;
74}
75
324429d7
HS		 76static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
77{
78 return ctx->crypto_ctx->ablkctx;
79}
80
81static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
82{
83 return ctx->crypto_ctx->hmacctx;
84}
85
2debd332
HJ		 86static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
87{
88 return gctx->ctx->gcm;
89}
90
91static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
92{
93 return gctx->ctx->authenc;
94}
95
324429d7
HS		 96static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
97{
98 return ctx->dev->u_ctx;
99}
100
101static inline int is_ofld_imm(const struct sk_buff *skb)
102{
103 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
104}
105
106/*
107 * sgl_len - calculates the size of an SGL of the given capacity
108 * @n: the number of SGL entries
109 * Calculates the number of flits needed for a scatter/gather list that
110 * can hold the given number of entries.
111 */
112static inline unsigned int sgl_len(unsigned int n)
113{
114 n--;
115 return (3 * n) / 2 + (n & 1) + 2;
116}
117
2debd332
HJ		 118static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
119{
120 u8 temp[SHA512_DIGEST_SIZE];
121 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
122 int authsize = crypto_aead_authsize(tfm);
123 struct cpl_fw6_pld *fw6_pld;
124 int cmp = 0;
125
126 fw6_pld = (struct cpl_fw6_pld *)input;
127 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
128 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
129 cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize);
130 } else {
131
132 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
133 authsize, req->assoclen +
134 req->cryptlen - authsize);
135 cmp = memcmp(temp, (fw6_pld + 1), authsize);
136 }
137 if (cmp)
138 *err = -EBADMSG;
139 else
140 *err = 0;
141}
142
324429d7
HS		 143/*
144 * chcr_handle_resp - Unmap the DMA buffers associated with the request
145 * @req: crypto request
146 */
147int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2debd332 148 int err)
324429d7
HS		 149{
150 struct crypto_tfm *tfm = req->tfm;
151 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
152 struct uld_ctx *u_ctx = ULD_CTX(ctx);
153 struct chcr_req_ctx ctx_req;
154 struct cpl_fw6_pld *fw6_pld;
155 unsigned int digestsize, updated_digestsize;
156
157 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2debd332
HJ		 158 case CRYPTO_ALG_TYPE_AEAD:
159 ctx_req.req.aead_req = (struct aead_request *)req;
160 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
94e1dab1 161 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
2debd332
HJ		 162 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
163 if (ctx_req.ctx.reqctx->skb) {
164 kfree_skb(ctx_req.ctx.reqctx->skb);
165 ctx_req.ctx.reqctx->skb = NULL;
166 }
167 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
168 chcr_verify_tag(ctx_req.req.aead_req, input,
169 &err);
170 ctx_req.ctx.reqctx->verify = VERIFY_HW;
171 }
172 break;
173
44e9f799 174 case CRYPTO_ALG_TYPE_ABLKCIPHER:
324429d7
HS		 175 ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
176 ctx_req.ctx.ablk_ctx =
177 ablkcipher_request_ctx(ctx_req.req.ablk_req);
2debd332 178 if (!err) {
324429d7
HS		 179 fw6_pld = (struct cpl_fw6_pld *)input;
180 memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
181 AES_BLOCK_SIZE);
182 }
183 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
5c86a8ff 184 ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE);
324429d7
HS		 185 if (ctx_req.ctx.ablk_ctx->skb) {
186 kfree_skb(ctx_req.ctx.ablk_ctx->skb);
187 ctx_req.ctx.ablk_ctx->skb = NULL;
188 }
189 break;
190
191 case CRYPTO_ALG_TYPE_AHASH:
192 ctx_req.req.ahash_req = (struct ahash_request *)req;
193 ctx_req.ctx.ahash_ctx =
194 ahash_request_ctx(ctx_req.req.ahash_req);
195 digestsize =
196 crypto_ahash_digestsize(crypto_ahash_reqtfm(
197 ctx_req.req.ahash_req));
198 updated_digestsize = digestsize;
199 if (digestsize == SHA224_DIGEST_SIZE)
200 updated_digestsize = SHA256_DIGEST_SIZE;
201 else if (digestsize == SHA384_DIGEST_SIZE)
202 updated_digestsize = SHA512_DIGEST_SIZE;
5c86a8ff
HJ		 203 if (ctx_req.ctx.ahash_ctx->skb) {
204 kfree_skb(ctx_req.ctx.ahash_ctx->skb);
324429d7 205 ctx_req.ctx.ahash_ctx->skb = NULL;
5c86a8ff 206 }
324429d7
HS		 207 if (ctx_req.ctx.ahash_ctx->result == 1) {
208 ctx_req.ctx.ahash_ctx->result = 0;
209 memcpy(ctx_req.req.ahash_req->result, input +
210 sizeof(struct cpl_fw6_pld),
211 digestsize);
212 } else {
213 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
214 sizeof(struct cpl_fw6_pld),
215 updated_digestsize);
216 }
324429d7
HS		 217 break;
218 }
2debd332 219 return err;
324429d7
HS		 220}
221
222/*
223 * calc_tx_flits_ofld - calculate # of flits for an offload packet
224 * @skb: the packet
225 * Returns the number of flits needed for the given offload packet.
226 * These packets are already fully constructed and no additional headers
227 * will be added.
228 */
229static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
230{
231 unsigned int flits, cnt;
232
233 if (is_ofld_imm(skb))
234 return DIV_ROUND_UP(skb->len, 8);
235
236 flits = skb_transport_offset(skb) / 8; /* headers */
237 cnt = skb_shinfo(skb)->nr_frags;
238 if (skb_tail_pointer(skb) != skb_transport_header(skb))
239 cnt++;
240 return flits + sgl_len(cnt);
241}
242
39f91a34
HJ		 243static inline void get_aes_decrypt_key(unsigned char *dec_key,
244 const unsigned char *key,
245 unsigned int keylength)
246{
247 u32 temp;
248 u32 w_ring[MAX_NK];
249 int i, j, k;
250 u8 nr, nk;
251
252 switch (keylength) {
253 case AES_KEYLENGTH_128BIT:
254 nk = KEYLENGTH_4BYTES;
255 nr = NUMBER_OF_ROUNDS_10;
256 break;
257 case AES_KEYLENGTH_192BIT:
258 nk = KEYLENGTH_6BYTES;
259 nr = NUMBER_OF_ROUNDS_12;
260 break;
261 case AES_KEYLENGTH_256BIT:
262 nk = KEYLENGTH_8BYTES;
263 nr = NUMBER_OF_ROUNDS_14;
264 break;
265 default:
266 return;
267 }
268 for (i = 0; i < nk; i++)
269 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
270
271 i = 0;
272 temp = w_ring[nk - 1];
273 while (i + nk < (nr + 1) * 4) {
274 if (!(i % nk)) {
275 /* RotWord(temp) */
276 temp = (temp << 8) | (temp >> 24);
277 temp = aes_ks_subword(temp);
278 temp ^= round_constant[i / nk];
279 } else if (nk == 8 && (i % 4 == 0)) {
280 temp = aes_ks_subword(temp);
281 }
282 w_ring[i % nk] ^= temp;
283 temp = w_ring[i % nk];
284 i++;
285 }
286 i--;
287 for (k = 0, j = i % nk; k < nk; k++) {
288 *((u32 *)dec_key + k) = htonl(w_ring[j]);
289 j--;
290 if (j < 0)
291 j += nk;
292 }
293}
294
e7922729 295static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
324429d7 296{
ec1bca94 297 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
324429d7
HS		 298
299 switch (ds) {
300 case SHA1_DIGEST_SIZE:
e7922729 301 base_hash = crypto_alloc_shash("sha1", 0, 0);
324429d7
HS		 302 break;
303 case SHA224_DIGEST_SIZE:
e7922729 304 base_hash = crypto_alloc_shash("sha224", 0, 0);
324429d7
HS		 305 break;
306 case SHA256_DIGEST_SIZE:
e7922729 307 base_hash = crypto_alloc_shash("sha256", 0, 0);
324429d7
HS		 308 break;
309 case SHA384_DIGEST_SIZE:
e7922729 310 base_hash = crypto_alloc_shash("sha384", 0, 0);
324429d7
HS		 311 break;
312 case SHA512_DIGEST_SIZE:
e7922729 313 base_hash = crypto_alloc_shash("sha512", 0, 0);
324429d7
HS		 314 break;
315 }
324429d7 316
e7922729 317 return base_hash;
324429d7
HS		 318}
319
320static int chcr_compute_partial_hash(struct shash_desc *desc,
321 char *iopad, char *result_hash,
322 int digest_size)
323{
324 struct sha1_state sha1_st;
325 struct sha256_state sha256_st;
326 struct sha512_state sha512_st;
327 int error;
328
329 if (digest_size == SHA1_DIGEST_SIZE) {
330 error = crypto_shash_init(desc) ?:
331 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
332 crypto_shash_export(desc, (void *)&sha1_st);
333 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
334 } else if (digest_size == SHA224_DIGEST_SIZE) {
335 error = crypto_shash_init(desc) ?:
336 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
337 crypto_shash_export(desc, (void *)&sha256_st);
338 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
339
340 } else if (digest_size == SHA256_DIGEST_SIZE) {
341 error = crypto_shash_init(desc) ?:
342 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
343 crypto_shash_export(desc, (void *)&sha256_st);
344 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
345
346 } else if (digest_size == SHA384_DIGEST_SIZE) {
347 error = crypto_shash_init(desc) ?:
348 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
349 crypto_shash_export(desc, (void *)&sha512_st);
350 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
351
352 } else if (digest_size == SHA512_DIGEST_SIZE) {
353 error = crypto_shash_init(desc) ?:
354 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
355 crypto_shash_export(desc, (void *)&sha512_st);
356 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
357 } else {
358 error = -EINVAL;
359 pr_err("Unknown digest size %d\n", digest_size);
360 }
361 return error;
362}
363
364static void chcr_change_order(char *buf, int ds)
365{
366 int i;
367
368 if (ds == SHA512_DIGEST_SIZE) {
369 for (i = 0; i < (ds / sizeof(u64)); i++)
370 *((__be64 *)buf + i) =
371 cpu_to_be64(*((u64 *)buf + i));
372 } else {
373 for (i = 0; i < (ds / sizeof(u32)); i++)
374 *((__be32 *)buf + i) =
375 cpu_to_be32(*((u32 *)buf + i));
376 }
377}
378
379static inline int is_hmac(struct crypto_tfm *tfm)
380{
381 struct crypto_alg *alg = tfm->__crt_alg;
382 struct chcr_alg_template *chcr_crypto_alg =
383 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
384 alg.hash);
5c86a8ff 385 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
324429d7
HS		 386 return 1;
387 return 0;
388}
389
324429d7
HS		 390static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
391 struct scatterlist *sg,
392 struct phys_sge_parm *sg_param)
393{
394 struct phys_sge_pairs *to;
adf1ca61
HJ		 395 int out_buf_size = sg_param->obsize;
396 unsigned int nents = sg_param->nents, i, j = 0;
324429d7
HS		 397
398 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
399 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
400 phys_cpl->pcirlxorder_to_noofsgentr =
401 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
402 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
403 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
404 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
405 CPL_RX_PHYS_DSGL_DCAID_V(0) |
406 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
407 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
408 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
409 phys_cpl->rss_hdr_int.hash_val = 0;
410 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
411 sizeof(struct cpl_rx_phys_dsgl));
412
413 for (i = 0; nents; to++) {
adf1ca61
HJ		 414 for (j = 0; j < 8 && nents; j++, nents--) {
415 out_buf_size -= sg_dma_len(sg);
416 to->len[j] = htons(sg_dma_len(sg));
324429d7 417 to->addr[j] = cpu_to_be64(sg_dma_address(sg));
324429d7
HS		 418 sg = sg_next(sg);
419 }
420 }
adf1ca61
HJ		 421 if (out_buf_size) {
422 j--;
423 to--;
424 to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size));
425 }
324429d7
HS		 426}
427
adf1ca61
HJ		 428static inline int map_writesg_phys_cpl(struct device *dev,
429 struct cpl_rx_phys_dsgl *phys_cpl,
430 struct scatterlist *sg,
431 struct phys_sge_parm *sg_param)
324429d7
HS		 432{
433 if (!sg || !sg_param->nents)
434 return 0;
435
436 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
437 if (sg_param->nents == 0) {
438 pr_err("CHCR : DMA mapping failed\n");
439 return -EINVAL;
440 }
441 write_phys_cpl(phys_cpl, sg, sg_param);
442 return 0;
443}
444
2debd332
HJ		 445static inline int get_aead_subtype(struct crypto_aead *aead)
446{
447 struct aead_alg *alg = crypto_aead_alg(aead);
448 struct chcr_alg_template *chcr_crypto_alg =
449 container_of(alg, struct chcr_alg_template, alg.aead);
450 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
451}
452
324429d7
HS		 453static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
454{
455 struct crypto_alg *alg = tfm->__crt_alg;
456 struct chcr_alg_template *chcr_crypto_alg =
457 container_of(alg, struct chcr_alg_template, alg.crypto);
458
459 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
460}
461
358961d1
HJ		 462static inline void write_buffer_to_skb(struct sk_buff *skb,
463 unsigned int *frags,
464 char *bfr,
465 u8 bfr_len)
466{
467 skb->len += bfr_len;
468 skb->data_len += bfr_len;
469 skb->truesize += bfr_len;
470 get_page(virt_to_page(bfr));
471 skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
472 offset_in_page(bfr), bfr_len);
473 (*frags)++;
474}
475
476
324429d7 477static inline void
358961d1 478write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
324429d7
HS		 479 struct scatterlist *sg, unsigned int count)
480{
481 struct page *spage;
482 unsigned int page_len;
483
484 skb->len += count;
485 skb->data_len += count;
486 skb->truesize += count;
18f0aa06 487
324429d7 488 while (count > 0) {
18f0aa06 489 if (!sg || (!(sg->length)))
324429d7
HS		 490 break;
491 spage = sg_page(sg);
492 get_page(spage);
493 page_len = min(sg->length, count);
494 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
495 (*frags)++;
496 count -= page_len;
497 sg = sg_next(sg);
498 }
499}
500
501static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
502 struct _key_ctx *key_ctx)
503{
504 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
cc1b156d 505 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
324429d7
HS		 506 } else {
507 memcpy(key_ctx->key,
508 ablkctx->key + (ablkctx->enckey_len >> 1),
509 ablkctx->enckey_len >> 1);
cc1b156d
HJ		 510 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
511 ablkctx->rrkey, ablkctx->enckey_len >> 1);
324429d7
HS		 512 }
513 return 0;
514}
515
516static inline void create_wreq(struct chcr_context *ctx,
358961d1 517 struct chcr_wr *chcr_req,
324429d7
HS		 518 void *req, struct sk_buff *skb,
519 int kctx_len, int hash_sz,
2debd332 520 int is_iv,
2512a624
HJ		 521 unsigned int sc_len,
522 unsigned int lcb)
324429d7
HS		 523{
524 struct uld_ctx *u_ctx = ULD_CTX(ctx);
324429d7 525 int iv_loc = IV_DSGL;
72a56ca9 526 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
324429d7
HS		 527 unsigned int immdatalen = 0, nr_frags = 0;
528
529 if (is_ofld_imm(skb)) {
530 immdatalen = skb->data_len;
531 iv_loc = IV_IMMEDIATE;
532 } else {
533 nr_frags = skb_shinfo(skb)->nr_frags;
534 }
535
358961d1
HJ		 536 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
537 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
538 chcr_req->wreq.pld_size_hash_size =
324429d7
HS		 539 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
540 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
358961d1
HJ		 541 chcr_req->wreq.len16_pkd =
542 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
324429d7 543 (calc_tx_flits_ofld(skb) * 8), 16)));
358961d1
HJ		 544 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
545 chcr_req->wreq.rx_chid_to_rx_q_id =
8a13449f 546 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
2512a624
HJ		 547 is_iv ? iv_loc : IV_NOP, !!lcb,
548 ctx->tx_qidx);
324429d7 549
8a13449f
HJ		 550 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
551 qid);
358961d1
HJ		 552 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
553 16) - ((sizeof(chcr_req->wreq)) >> 4)));
324429d7 554
358961d1
HJ		 555 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
556 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
557 sizeof(chcr_req->key_ctx) +
2debd332 558 kctx_len + sc_len + immdatalen);
324429d7
HS		 559}
560
561/**
562 * create_cipher_wr - form the WR for cipher operations
563 * @req: cipher req.
564 * @ctx: crypto driver context of the request.
565 * @qid: ingress qid where response of this WR should be received.
566 * @op_type: encryption or decryption
567 */
568static struct sk_buff
358961d1
HJ		 569*create_cipher_wr(struct ablkcipher_request *req,
570 unsigned short qid,
324429d7
HS		 571 unsigned short op_type)
572{
324429d7 573 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
358961d1 574 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
324429d7
HS		 575 struct uld_ctx *u_ctx = ULD_CTX(ctx);
576 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
577 struct sk_buff *skb = NULL;
358961d1 578 struct chcr_wr *chcr_req;
324429d7 579 struct cpl_rx_phys_dsgl *phys_cpl;
5c86a8ff 580 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
324429d7 581 struct phys_sge_parm sg_param;
adf1ca61 582 unsigned int frags = 0, transhdr_len, phys_dsgl;
324429d7 583 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
358961d1
HJ		 584 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
585 GFP_ATOMIC;
324429d7
HS		 586
587 if (!req->info)
588 return ERR_PTR(-EINVAL);
5c86a8ff
HJ		 589 reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
590 if (reqctx->dst_nents <= 0) {
adf1ca61
HJ		 591 pr_err("AES:Invalid Destination sg lists\n");
592 return ERR_PTR(-EINVAL);
593 }
324429d7 594 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
358961d1
HJ		 595 (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) {
596 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
597 ablkctx->enckey_len, req->nbytes, ivsize);
324429d7 598 return ERR_PTR(-EINVAL);
358961d1 599 }
324429d7 600
5c86a8ff 601 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
324429d7 602
358961d1 603 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
324429d7 604 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
358961d1 605 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
324429d7
HS		 606 if (!skb)
607 return ERR_PTR(-ENOMEM);
608 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
358961d1
HJ		 609 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
610 memset(chcr_req, 0, transhdr_len);
611 chcr_req->sec_cpl.op_ivinsrtofst =
8a13449f 612 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
358961d1
HJ		 613
614 chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes);
615 chcr_req->sec_cpl.aadstart_cipherstop_hi =
616 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
617
618 chcr_req->sec_cpl.cipherstop_lo_authinsert =
619 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
620 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
324429d7 621 ablkctx->ciph_mode,
358961d1
HJ		 622 0, 0, ivsize >> 1);
623 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
324429d7
HS		 624 0, 1, phys_dsgl);
625
358961d1 626 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
324429d7 627 if (op_type == CHCR_DECRYPT_OP) {
358961d1 628 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
324429d7
HS		 629 } else {
630 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
358961d1
HJ		 631 memcpy(chcr_req->key_ctx.key, ablkctx->key,
632 ablkctx->enckey_len);
324429d7 633 } else {
358961d1 634 memcpy(chcr_req->key_ctx.key, ablkctx->key +
324429d7
HS		 635 (ablkctx->enckey_len >> 1),
636 ablkctx->enckey_len >> 1);
358961d1 637 memcpy(chcr_req->key_ctx.key +
324429d7
HS		 638 (ablkctx->enckey_len >> 1),
639 ablkctx->key,
640 ablkctx->enckey_len >> 1);
641 }
642 }
358961d1 643 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
5c86a8ff 644 sg_param.nents = reqctx->dst_nents;
358961d1 645 sg_param.obsize = req->nbytes;
324429d7
HS		 646 sg_param.qid = qid;
647 sg_param.align = 1;
648 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
649 &sg_param))
650 goto map_fail1;
651
652 skb_set_transport_header(skb, transhdr_len);
5c86a8ff
HJ		 653 memcpy(reqctx->iv, req->info, ivsize);
654 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
358961d1 655 write_sg_to_skb(skb, &frags, req->src, req->nbytes);
2debd332 656 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
2512a624
HJ		 657 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
658 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
5c86a8ff 659 reqctx->skb = skb;
324429d7
HS		 660 skb_get(skb);
661 return skb;
662map_fail1:
663 kfree_skb(skb);
664 return ERR_PTR(-ENOMEM);
665}
666
667static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
668 unsigned int keylen)
669{
670 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
671 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
324429d7
HS		 672 unsigned int ck_size, context_size;
673 u16 alignment = 0;
674
324429d7
HS		 675 if (keylen == AES_KEYSIZE_128) {
676 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
677 } else if (keylen == AES_KEYSIZE_192) {
678 alignment = 8;
679 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
680 } else if (keylen == AES_KEYSIZE_256) {
681 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
682 } else {
683 goto badkey_err;
684 }
cc1b156d
HJ		 685 memcpy(ablkctx->key, key, keylen);
686 ablkctx->enckey_len = keylen;
687 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
324429d7
HS		 688 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
689 keylen + alignment) >> 4;
690
691 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
692 0, 0, context_size);
693 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
694 return 0;
695badkey_err:
696 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
697 ablkctx->enckey_len = 0;
698 return -EINVAL;
699}
700
73b86bb7 701static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
324429d7 702{
324429d7 703 struct adapter *adap = netdev2adap(dev);
ab677ff4
HS		 704 struct sge_uld_txq_info *txq_info =
705 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
706 struct sge_uld_txq *txq;
707 int ret = 0;
324429d7
HS		 708
709 local_bh_disable();
ab677ff4
HS		 710 txq = &txq_info->uldtxq[idx];
711 spin_lock(&txq->sendq.lock);
712 if (txq->full)
324429d7 713 ret = -1;
ab677ff4 714 spin_unlock(&txq->sendq.lock);
324429d7
HS		 715 local_bh_enable();
716 return ret;
717}
718
719static int chcr_aes_encrypt(struct ablkcipher_request *req)
720{
721 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
722 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
324429d7
HS		 723 struct uld_ctx *u_ctx = ULD_CTX(ctx);
724 struct sk_buff *skb;
725
726 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 727 ctx->tx_qidx))) {
324429d7
HS		 728 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
729 return -EBUSY;
730 }
731
72a56ca9 732 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx],
324429d7
HS		 733 CHCR_ENCRYPT_OP);
734 if (IS_ERR(skb)) {
735 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
736 return PTR_ERR(skb);
737 }
738 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 739 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 740 chcr_send_wr(skb);
741 return -EINPROGRESS;
742}
743
744static int chcr_aes_decrypt(struct ablkcipher_request *req)
745{
746 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
747 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
324429d7
HS		 748 struct uld_ctx *u_ctx = ULD_CTX(ctx);
749 struct sk_buff *skb;
750
751 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 752 ctx->tx_qidx))) {
324429d7
HS		 753 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
754 return -EBUSY;
755 }
756
72a56ca9 757 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx],
324429d7
HS		 758 CHCR_DECRYPT_OP);
759 if (IS_ERR(skb)) {
760 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
761 return PTR_ERR(skb);
762 }
763 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 764 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 765 chcr_send_wr(skb);
766 return -EINPROGRESS;
767}
768
769static int chcr_device_init(struct chcr_context *ctx)
770{
771 struct uld_ctx *u_ctx;
72a56ca9 772 struct adapter *adap;
324429d7 773 unsigned int id;
72a56ca9 774 int txq_perchan, txq_idx, ntxq;
324429d7
HS		 775 int err = 0, rxq_perchan, rxq_idx;
776
777 id = smp_processor_id();
778 if (!ctx->dev) {
779 err = assign_chcr_device(&ctx->dev);
780 if (err) {
781 pr_err("chcr device assignment fails\n");
782 goto out;
783 }
784 u_ctx = ULD_CTX(ctx);
72a56ca9
HJ		 785 adap = padap(ctx->dev);
786 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
787 adap->vres.ncrypto_fc);
324429d7 788 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
72a56ca9 789 txq_perchan = ntxq / u_ctx->lldi.nchan;
324429d7
HS		 790 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
791 rxq_idx += id % rxq_perchan;
72a56ca9
HJ		 792 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
793 txq_idx += id % txq_perchan;
324429d7 794 spin_lock(&ctx->dev->lock_chcr_dev);
72a56ca9
HJ		 795 ctx->rx_qidx = rxq_idx;
796 ctx->tx_qidx = txq_idx;
ab677ff4 797 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
8a13449f 798 ctx->dev->rx_channel_id = 0;
324429d7
HS		 799 spin_unlock(&ctx->dev->lock_chcr_dev);
800 }
801out:
802 return err;
803}
804
805static int chcr_cra_init(struct crypto_tfm *tfm)
806{
807 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
808 return chcr_device_init(crypto_tfm_ctx(tfm));
809}
810
811static int get_alg_config(struct algo_param *params,
812 unsigned int auth_size)
813{
814 switch (auth_size) {
815 case SHA1_DIGEST_SIZE:
816 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
817 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
818 params->result_size = SHA1_DIGEST_SIZE;
819 break;
820 case SHA224_DIGEST_SIZE:
821 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
822 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
823 params->result_size = SHA256_DIGEST_SIZE;
824 break;
825 case SHA256_DIGEST_SIZE:
826 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
827 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
828 params->result_size = SHA256_DIGEST_SIZE;
829 break;
830 case SHA384_DIGEST_SIZE:
831 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
832 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
833 params->result_size = SHA512_DIGEST_SIZE;
834 break;
835 case SHA512_DIGEST_SIZE:
836 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
837 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
838 params->result_size = SHA512_DIGEST_SIZE;
839 break;
840 default:
841 pr_err("chcr : ERROR, unsupported digest size\n");
842 return -EINVAL;
843 }
844 return 0;
845}
846
e7922729 847static inline void chcr_free_shash(struct crypto_shash *base_hash)
324429d7 848{
e7922729 849 crypto_free_shash(base_hash);
324429d7
HS		 850}
851
852/**
358961d1 853 * create_hash_wr - Create hash work request
324429d7
HS		 854 * @req - Cipher req base
855 */
358961d1 856static struct sk_buff *create_hash_wr(struct ahash_request *req,
2debd332 857 struct hash_wr_param *param)
324429d7
HS		 858{
859 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
860 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
861 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
862 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
863 struct sk_buff *skb = NULL;
358961d1 864 struct chcr_wr *chcr_req;
324429d7
HS		 865 unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
866 unsigned int digestsize = crypto_ahash_digestsize(tfm);
358961d1 867 unsigned int kctx_len = 0;
324429d7 868 u8 hash_size_in_response = 0;
358961d1
HJ		 869 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
870 GFP_ATOMIC;
324429d7
HS		 871
872 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
358961d1 873 kctx_len = param->alg_prm.result_size + iopad_alignment;
324429d7
HS		 874 if (param->opad_needed)
875 kctx_len += param->alg_prm.result_size + iopad_alignment;
876
877 if (req_ctx->result)
878 hash_size_in_response = digestsize;
879 else
880 hash_size_in_response = param->alg_prm.result_size;
881 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
358961d1 882 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
324429d7
HS		 883 if (!skb)
884 return skb;
885
886 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
358961d1
HJ		 887 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
888 memset(chcr_req, 0, transhdr_len);
324429d7 889
358961d1 890 chcr_req->sec_cpl.op_ivinsrtofst =
8a13449f 891 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
358961d1 892 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
324429d7 893
358961d1 894 chcr_req->sec_cpl.aadstart_cipherstop_hi =
324429d7 895 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
358961d1 896 chcr_req->sec_cpl.cipherstop_lo_authinsert =
324429d7 897 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
358961d1 898 chcr_req->sec_cpl.seqno_numivs =
324429d7 899 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
358961d1 900 param->opad_needed, 0);
324429d7 901
358961d1 902 chcr_req->sec_cpl.ivgen_hdrlen =
324429d7
HS		 903 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
904
358961d1
HJ		 905 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
906 param->alg_prm.result_size);
324429d7
HS		 907
908 if (param->opad_needed)
358961d1
HJ		 909 memcpy(chcr_req->key_ctx.key +
910 ((param->alg_prm.result_size <= 32) ? 32 :
911 CHCR_HASH_MAX_DIGEST_SIZE),
324429d7
HS		 912 hmacctx->opad, param->alg_prm.result_size);
913
358961d1 914 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
324429d7
HS		 915 param->alg_prm.mk_size, 0,
916 param->opad_needed,
358961d1
HJ		 917 ((kctx_len +
918 sizeof(chcr_req->key_ctx)) >> 4));
919 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
324429d7
HS		 920
921 skb_set_transport_header(skb, transhdr_len);
922 if (param->bfr_len != 0)
44fce12a
HJ		 923 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
924 param->bfr_len);
324429d7 925 if (param->sg_len != 0)
358961d1 926 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
324429d7 927
2debd332 928 create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0,
2512a624 929 DUMMY_BYTES, 0);
324429d7
HS		 930 req_ctx->skb = skb;
931 skb_get(skb);
932 return skb;
933}
934
935static int chcr_ahash_update(struct ahash_request *req)
936{
937 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
938 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
939 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
940 struct uld_ctx *u_ctx = NULL;
941 struct sk_buff *skb;
942 u8 remainder = 0, bs;
943 unsigned int nbytes = req->nbytes;
944 struct hash_wr_param params;
945
946 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
947
948 u_ctx = ULD_CTX(ctx);
949 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 950 ctx->tx_qidx))) {
324429d7
HS		 951 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
952 return -EBUSY;
953 }
954
44fce12a
HJ		 955 if (nbytes + req_ctx->reqlen >= bs) {
956 remainder = (nbytes + req_ctx->reqlen) % bs;
957 nbytes = nbytes + req_ctx->reqlen - remainder;
324429d7 958 } else {
44fce12a
HJ		 959 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
960 + req_ctx->reqlen, nbytes, 0);
961 req_ctx->reqlen += nbytes;
324429d7
HS		 962 return 0;
963 }
964
965 params.opad_needed = 0;
966 params.more = 1;
967 params.last = 0;
44fce12a
HJ		 968 params.sg_len = nbytes - req_ctx->reqlen;
969 params.bfr_len = req_ctx->reqlen;
324429d7
HS		 970 params.scmd1 = 0;
971 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
972 req_ctx->result = 0;
973 req_ctx->data_len += params.sg_len + params.bfr_len;
358961d1 974 skb = create_hash_wr(req, &params);
324429d7
HS		 975 if (!skb)
976 return -ENOMEM;
977
44fce12a
HJ		 978 if (remainder) {
979 u8 *temp;
980 /* Swap buffers */
981 temp = req_ctx->reqbfr;
982 req_ctx->reqbfr = req_ctx->skbfr;
983 req_ctx->skbfr = temp;
324429d7 984 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
44fce12a 985 req_ctx->reqbfr, remainder, req->nbytes -
324429d7 986 remainder);
44fce12a
HJ		 987 }
988 req_ctx->reqlen = remainder;
324429d7 989 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 990 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 991 chcr_send_wr(skb);
992
993 return -EINPROGRESS;
994}
995
996static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
997{
998 memset(bfr_ptr, 0, bs);
999 *bfr_ptr = 0x80;
1000 if (bs == 64)
1001 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1002 else
1003 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1004}
1005
1006static int chcr_ahash_final(struct ahash_request *req)
1007{
1008 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1009 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1010 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1011 struct hash_wr_param params;
1012 struct sk_buff *skb;
1013 struct uld_ctx *u_ctx = NULL;
1014 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1015
1016 u_ctx = ULD_CTX(ctx);
1017 if (is_hmac(crypto_ahash_tfm(rtfm)))
1018 params.opad_needed = 1;
1019 else
1020 params.opad_needed = 0;
1021 params.sg_len = 0;
1022 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1023 req_ctx->result = 1;
44fce12a 1024 params.bfr_len = req_ctx->reqlen;
324429d7 1025 req_ctx->data_len += params.bfr_len + params.sg_len;
44fce12a
HJ		 1026 if (req_ctx->reqlen == 0) {
1027 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1028 params.last = 0;
1029 params.more = 1;
1030 params.scmd1 = 0;
1031 params.bfr_len = bs;
1032
1033 } else {
1034 params.scmd1 = req_ctx->data_len;
1035 params.last = 1;
1036 params.more = 0;
1037 }
358961d1 1038 skb = create_hash_wr(req, &params);
9a97ffd4
DC		 1039 if (!skb)
1040 return -ENOMEM;
358961d1 1041
324429d7 1042 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 1043 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 1044 chcr_send_wr(skb);
1045 return -EINPROGRESS;
1046}
1047
1048static int chcr_ahash_finup(struct ahash_request *req)
1049{
1050 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1051 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1052 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1053 struct uld_ctx *u_ctx = NULL;
1054 struct sk_buff *skb;
1055 struct hash_wr_param params;
1056 u8 bs;
1057
1058 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1059 u_ctx = ULD_CTX(ctx);
1060
1061 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 1062 ctx->tx_qidx))) {
324429d7
HS		 1063 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1064 return -EBUSY;
1065 }
1066
1067 if (is_hmac(crypto_ahash_tfm(rtfm)))
1068 params.opad_needed = 1;
1069 else
1070 params.opad_needed = 0;
1071
1072 params.sg_len = req->nbytes;
44fce12a 1073 params.bfr_len = req_ctx->reqlen;
324429d7
HS		 1074 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1075 req_ctx->data_len += params.bfr_len + params.sg_len;
1076 req_ctx->result = 1;
44fce12a
HJ		 1077 if ((req_ctx->reqlen + req->nbytes) == 0) {
1078 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1079 params.last = 0;
1080 params.more = 1;
1081 params.scmd1 = 0;
1082 params.bfr_len = bs;
1083 } else {
1084 params.scmd1 = req_ctx->data_len;
1085 params.last = 1;
1086 params.more = 0;
1087 }
1088
358961d1 1089 skb = create_hash_wr(req, &params);
324429d7
HS		 1090 if (!skb)
1091 return -ENOMEM;
358961d1 1092
324429d7 1093 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 1094 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 1095 chcr_send_wr(skb);
1096
1097 return -EINPROGRESS;
1098}
1099
1100static int chcr_ahash_digest(struct ahash_request *req)
1101{
1102 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1103 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1104 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1105 struct uld_ctx *u_ctx = NULL;
1106 struct sk_buff *skb;
1107 struct hash_wr_param params;
1108 u8 bs;
1109
1110 rtfm->init(req);
1111 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1112
1113 u_ctx = ULD_CTX(ctx);
1114 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 1115 ctx->tx_qidx))) {
324429d7
HS		 1116 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1117 return -EBUSY;
1118 }
1119
1120 if (is_hmac(crypto_ahash_tfm(rtfm)))
1121 params.opad_needed = 1;
1122 else
1123 params.opad_needed = 0;
1124
1125 params.last = 0;
1126 params.more = 0;
1127 params.sg_len = req->nbytes;
1128 params.bfr_len = 0;
1129 params.scmd1 = 0;
1130 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1131 req_ctx->result = 1;
1132 req_ctx->data_len += params.bfr_len + params.sg_len;
1133
44fce12a
HJ		 1134 if (req->nbytes == 0) {
1135 create_last_hash_block(req_ctx->reqbfr, bs, 0);
324429d7
HS		 1136 params.more = 1;
1137 params.bfr_len = bs;
1138 }
1139
358961d1 1140 skb = create_hash_wr(req, &params);
324429d7
HS		 1141 if (!skb)
1142 return -ENOMEM;
1143
1144 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 1145 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
324429d7
HS		 1146 chcr_send_wr(skb);
1147 return -EINPROGRESS;
1148}
1149
1150static int chcr_ahash_export(struct ahash_request *areq, void *out)
1151{
1152 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1153 struct chcr_ahash_req_ctx *state = out;
1154
44fce12a 1155 state->reqlen = req_ctx->reqlen;
324429d7 1156 state->data_len = req_ctx->data_len;
44fce12a 1157 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
324429d7
HS		 1158 memcpy(state->partial_hash, req_ctx->partial_hash,
1159 CHCR_HASH_MAX_DIGEST_SIZE);
44fce12a 1160 return 0;
324429d7
HS		 1161}
1162
1163static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1164{
1165 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1166 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1167
44fce12a 1168 req_ctx->reqlen = state->reqlen;
324429d7 1169 req_ctx->data_len = state->data_len;
44fce12a
HJ		 1170 req_ctx->reqbfr = req_ctx->bfr1;
1171 req_ctx->skbfr = req_ctx->bfr2;
1172 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
324429d7
HS		 1173 memcpy(req_ctx->partial_hash, state->partial_hash,
1174 CHCR_HASH_MAX_DIGEST_SIZE);
1175 return 0;
1176}
1177
1178static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1179 unsigned int keylen)
1180{
1181 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1182 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1183 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1184 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1185 unsigned int i, err = 0, updated_digestsize;
1186
e7922729
HJ		 1187 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1188
1189 /* use the key to calculate the ipad and opad. ipad will sent with the
324429d7
HS		 1190 * first request's data. opad will be sent with the final hash result
1191 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1192 */
e7922729
HJ		 1193 shash->tfm = hmacctx->base_hash;
1194 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
324429d7 1195 if (keylen > bs) {
e7922729 1196 err = crypto_shash_digest(shash, key, keylen,
324429d7
HS		 1197 hmacctx->ipad);
1198 if (err)
1199 goto out;
1200 keylen = digestsize;
1201 } else {
1202 memcpy(hmacctx->ipad, key, keylen);
1203 }
1204 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1205 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1206
1207 for (i = 0; i < bs / sizeof(int); i++) {
1208 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1209 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1210 }
1211
1212 updated_digestsize = digestsize;
1213 if (digestsize == SHA224_DIGEST_SIZE)
1214 updated_digestsize = SHA256_DIGEST_SIZE;
1215 else if (digestsize == SHA384_DIGEST_SIZE)
1216 updated_digestsize = SHA512_DIGEST_SIZE;
e7922729 1217 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
324429d7
HS		 1218 hmacctx->ipad, digestsize);
1219 if (err)
1220 goto out;
1221 chcr_change_order(hmacctx->ipad, updated_digestsize);
1222
e7922729 1223 err = chcr_compute_partial_hash(shash, hmacctx->opad,
324429d7
HS		 1224 hmacctx->opad, digestsize);
1225 if (err)
1226 goto out;
1227 chcr_change_order(hmacctx->opad, updated_digestsize);
1228out:
1229 return err;
1230}
1231
1232static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
1233 unsigned int key_len)
1234{
1235 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1236 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
324429d7
HS		 1237 unsigned short context_size = 0;
1238
cc1b156d
HJ		 1239 if ((key_len != (AES_KEYSIZE_128 << 1)) &&
1240 (key_len != (AES_KEYSIZE_256 << 1))) {
324429d7
HS		 1241 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
1242 CRYPTO_TFM_RES_BAD_KEY_LEN);
1243 ablkctx->enckey_len = 0;
cc1b156d
HJ		 1244 return -EINVAL;
1245
324429d7 1246 }
cc1b156d
HJ		 1247
1248 memcpy(ablkctx->key, key, key_len);
1249 ablkctx->enckey_len = key_len;
1250 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1251 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1252 ablkctx->key_ctx_hdr =
1253 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1254 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1255 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1256 CHCR_KEYCTX_NO_KEY, 1,
1257 0, context_size);
1258 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1259 return 0;
324429d7
HS		 1260}
1261
1262static int chcr_sha_init(struct ahash_request *areq)
1263{
1264 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1265 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1266 int digestsize = crypto_ahash_digestsize(tfm);
1267
1268 req_ctx->data_len = 0;
44fce12a
HJ		 1269 req_ctx->reqlen = 0;
1270 req_ctx->reqbfr = req_ctx->bfr1;
1271 req_ctx->skbfr = req_ctx->bfr2;
324429d7
HS		 1272 req_ctx->skb = NULL;
1273 req_ctx->result = 0;
1274 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1275 return 0;
1276}
1277
1278static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1279{
1280 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1281 sizeof(struct chcr_ahash_req_ctx));
1282 return chcr_device_init(crypto_tfm_ctx(tfm));
1283}
1284
1285static int chcr_hmac_init(struct ahash_request *areq)
1286{
1287 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1288 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1289 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1290 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1291 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1292 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1293
1294 chcr_sha_init(areq);
1295 req_ctx->data_len = bs;
1296 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1297 if (digestsize == SHA224_DIGEST_SIZE)
1298 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1299 SHA256_DIGEST_SIZE);
1300 else if (digestsize == SHA384_DIGEST_SIZE)
1301 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1302 SHA512_DIGEST_SIZE);
1303 else
1304 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1305 digestsize);
1306 }
1307 return 0;
1308}
1309
1310static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1311{
1312 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1313 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1314 unsigned int digestsize =
1315 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1316
1317 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1318 sizeof(struct chcr_ahash_req_ctx));
e7922729
HJ		 1319 hmacctx->base_hash = chcr_alloc_shash(digestsize);
1320 if (IS_ERR(hmacctx->base_hash))
1321 return PTR_ERR(hmacctx->base_hash);
324429d7
HS		 1322 return chcr_device_init(crypto_tfm_ctx(tfm));
1323}
1324
324429d7
HS		 1325static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
1326{
1327 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1328 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1329
e7922729
HJ		 1330 if (hmacctx->base_hash) {
1331 chcr_free_shash(hmacctx->base_hash);
1332 hmacctx->base_hash = NULL;
324429d7
HS		 1333 }
1334}
1335
2debd332
HJ		 1336static int chcr_copy_assoc(struct aead_request *req,
1337 struct chcr_aead_ctx *ctx)
1338{
1339 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
1340
1341 skcipher_request_set_tfm(skreq, ctx->null);
1342 skcipher_request_set_callback(skreq, aead_request_flags(req),
1343 NULL, NULL);
1344 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
1345 NULL);
1346
1347 return crypto_skcipher_encrypt(skreq);
1348}
0e93708d
HJ		 1349static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
1350 int aadmax, int wrlen,
1351 unsigned short op_type)
1352{
1353 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
1354
1355 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
1356 (req->assoclen > aadmax) ||
1357 (src_nent > MAX_SKB_FRAGS) ||
1358 (wrlen > MAX_WR_SIZE))
1359 return 1;
1360 return 0;
1361}
2debd332 1362
0e93708d
HJ		 1363static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
1364{
1365 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1366 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1367 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1368 struct aead_request *subreq = aead_request_ctx(req);
1369
1370 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
1371 aead_request_set_callback(subreq, req->base.flags,
1372 req->base.complete, req->base.data);
1373 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
1374 req->iv);
1375 aead_request_set_ad(subreq, req->assoclen);
1376 return op_type ? crypto_aead_decrypt(subreq) :
1377 crypto_aead_encrypt(subreq);
1378}
2debd332
HJ		 1379
1380static struct sk_buff *create_authenc_wr(struct aead_request *req,
1381 unsigned short qid,
1382 int size,
1383 unsigned short op_type)
1384{
1385 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1386 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1387 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1388 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1389 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
1390 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1391 struct sk_buff *skb = NULL;
1392 struct chcr_wr *chcr_req;
1393 struct cpl_rx_phys_dsgl *phys_cpl;
1394 struct phys_sge_parm sg_param;
94e1dab1 1395 struct scatterlist *src;
2debd332
HJ		 1396 unsigned int frags = 0, transhdr_len;
1397 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
1398 unsigned int kctx_len = 0;
1399 unsigned short stop_offset = 0;
1400 unsigned int assoclen = req->assoclen;
1401 unsigned int authsize = crypto_aead_authsize(tfm);
0e93708d 1402 int err = -EINVAL, src_nent;
2debd332
HJ		 1403 int null = 0;
1404 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1405 GFP_ATOMIC;
1406
1407 if (aeadctx->enckey_len == 0 || (req->cryptlen == 0))
1408 goto err;
1409
1410 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1411 goto err;
0e93708d
HJ		 1412 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1413 if (src_nent < 0)
2debd332 1414 goto err;
94e1dab1
HJ		 1415 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1416 reqctx->dst = src;
1417
2debd332
HJ		 1418 if (req->src != req->dst) {
1419 err = chcr_copy_assoc(req, aeadctx);
1420 if (err)
1421 return ERR_PTR(err);
94e1dab1
HJ		 1422 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1423 req->assoclen);
2debd332
HJ		 1424 }
1425 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
1426 null = 1;
1427 assoclen = 0;
1428 }
94e1dab1 1429 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2debd332 1430 (op_type ? -authsize : authsize));
0e93708d 1431 if (reqctx->dst_nents < 0) {
2debd332
HJ		 1432 pr_err("AUTHENC:Invalid Destination sg entries\n");
1433 goto err;
1434 }
1435 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1436 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
1437 - sizeof(chcr_req->key_ctx);
1438 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
0e93708d
HJ		 1439 if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG,
1440 T6_MAX_AAD_SIZE,
1441 transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
1442 op_type)) {
1443 return ERR_PTR(chcr_aead_fallback(req, op_type));
1444 }
2debd332
HJ		 1445 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1446 if (!skb)
1447 goto err;
1448
1449 /* LLD is going to write the sge hdr. */
1450 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1451
1452 /* Write WR */
1453 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
1454 memset(chcr_req, 0, transhdr_len);
1455
1456 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
1457
1458 /*
1459 * Input order is AAD,IV and Payload. where IV should be included as
1460 * the part of authdata. All other fields should be filled according
1461 * to the hardware spec
1462 */
1463 chcr_req->sec_cpl.op_ivinsrtofst =
8a13449f 1464 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
2debd332
HJ		 1465 (ivsize ? (assoclen + 1) : 0));
1466 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
1467 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1468 assoclen ? 1 : 0, assoclen,
1469 assoclen + ivsize + 1,
1470 (stop_offset & 0x1F0) >> 4);
1471 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
1472 stop_offset & 0xF,
1473 null ? 0 : assoclen + ivsize + 1,
1474 stop_offset, stop_offset);
1475 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
1476 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
1477 CHCR_SCMD_CIPHER_MODE_AES_CBC,
1478 actx->auth_mode, aeadctx->hmac_ctrl,
1479 ivsize >> 1);
1480 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
1481 0, 1, dst_size);
1482
1483 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1484 if (op_type == CHCR_ENCRYPT_OP)
1485 memcpy(chcr_req->key_ctx.key, aeadctx->key,
1486 aeadctx->enckey_len);
1487 else
1488 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
1489 aeadctx->enckey_len);
1490
1491 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
1492 4), actx->h_iopad, kctx_len -
1493 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
1494
1495 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1496 sg_param.nents = reqctx->dst_nents;
1497 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1498 sg_param.qid = qid;
1499 sg_param.align = 0;
94e1dab1 1500 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
2debd332
HJ		 1501 &sg_param))
1502 goto dstmap_fail;
1503
1504 skb_set_transport_header(skb, transhdr_len);
1505
1506 if (assoclen) {
1507 /* AAD buffer in */
1508 write_sg_to_skb(skb, &frags, req->src, assoclen);
1509
1510 }
1511 write_buffer_to_skb(skb, &frags, req->iv, ivsize);
1512 write_sg_to_skb(skb, &frags, src, req->cryptlen);
1513 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
2512a624 1514 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2debd332
HJ		 1515 reqctx->skb = skb;
1516 skb_get(skb);
1517
1518 return skb;
1519dstmap_fail:
1520 /* ivmap_fail: */
1521 kfree_skb(skb);
1522err:
1523 return ERR_PTR(-EINVAL);
1524}
1525
2debd332
HJ		 1526static int set_msg_len(u8 *block, unsigned int msglen, int csize)
1527{
1528 __be32 data;
1529
1530 memset(block, 0, csize);
1531 block += csize;
1532
1533 if (csize >= 4)
1534 csize = 4;
1535 else if (msglen > (unsigned int)(1 << (8 * csize)))
1536 return -EOVERFLOW;
1537
1538 data = cpu_to_be32(msglen);
1539 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1540
1541 return 0;
1542}
1543
1544static void generate_b0(struct aead_request *req,
1545 struct chcr_aead_ctx *aeadctx,
1546 unsigned short op_type)
1547{
1548 unsigned int l, lp, m;
1549 int rc;
1550 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1551 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1552 u8 *b0 = reqctx->scratch_pad;
1553
1554 m = crypto_aead_authsize(aead);
1555
1556 memcpy(b0, reqctx->iv, 16);
1557
1558 lp = b0[0];
1559 l = lp + 1;
1560
1561 /* set m, bits 3-5 */
1562 *b0 |= (8 * ((m - 2) / 2));
1563
1564 /* set adata, bit 6, if associated data is used */
1565 if (req->assoclen)
1566 *b0 |= 64;
1567 rc = set_msg_len(b0 + 16 - l,
1568 (op_type == CHCR_DECRYPT_OP) ?
1569 req->cryptlen - m : req->cryptlen, l);
1570}
1571
1572static inline int crypto_ccm_check_iv(const u8 *iv)
1573{
1574 /* 2 <= L <= 8, so 1 <= L' <= 7. */
1575 if (iv[0] < 1 || iv[0] > 7)
1576 return -EINVAL;
1577
1578 return 0;
1579}
1580
1581static int ccm_format_packet(struct aead_request *req,
1582 struct chcr_aead_ctx *aeadctx,
1583 unsigned int sub_type,
1584 unsigned short op_type)
1585{
1586 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1587 int rc = 0;
1588
2debd332
HJ		 1589 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
1590 reqctx->iv[0] = 3;
1591 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
1592 memcpy(reqctx->iv + 4, req->iv, 8);
1593 memset(reqctx->iv + 12, 0, 4);
1594 *((unsigned short *)(reqctx->scratch_pad + 16)) =
1595 htons(req->assoclen - 8);
1596 } else {
1597 memcpy(reqctx->iv, req->iv, 16);
1598 *((unsigned short *)(reqctx->scratch_pad + 16)) =
1599 htons(req->assoclen);
1600 }
1601 generate_b0(req, aeadctx, op_type);
1602 /* zero the ctr value */
1603 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
1604 return rc;
1605}
1606
1607static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
1608 unsigned int dst_size,
1609 struct aead_request *req,
1610 unsigned short op_type,
1611 struct chcr_context *chcrctx)
1612{
1613 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
0a7bd30c 1614 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2debd332
HJ		 1615 unsigned int ivsize = AES_BLOCK_SIZE;
1616 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
1617 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
8a13449f 1618 unsigned int c_id = chcrctx->dev->rx_channel_id;
2debd332
HJ		 1619 unsigned int ccm_xtra;
1620 unsigned char tag_offset = 0, auth_offset = 0;
2debd332
HJ		 1621 unsigned int assoclen;
1622
1623 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
1624 assoclen = req->assoclen - 8;
1625 else
1626 assoclen = req->assoclen;
1627 ccm_xtra = CCM_B0_SIZE +
1628 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
1629
1630 auth_offset = req->cryptlen ?
1631 (assoclen + ivsize + 1 + ccm_xtra) : 0;
1632 if (op_type == CHCR_DECRYPT_OP) {
1633 if (crypto_aead_authsize(tfm) != req->cryptlen)
1634 tag_offset = crypto_aead_authsize(tfm);
1635 else
1636 auth_offset = 0;
1637 }
1638
1639
1640 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
1641 2, (ivsize ? (assoclen + 1) : 0) +
1642 ccm_xtra);
1643 sec_cpl->pldlen =
1644 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
1645 /* For CCM there wil be b0 always. So AAD start will be 1 always */
1646 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1647 1, assoclen + ccm_xtra, assoclen
1648 + ivsize + 1 + ccm_xtra, 0);
1649
1650 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
1651 auth_offset, tag_offset,
1652 (op_type == CHCR_ENCRYPT_OP) ? 0 :
1653 crypto_aead_authsize(tfm));
1654 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
1655 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
0a7bd30c
HJ		 1656 cipher_mode, mac_mode,
1657 aeadctx->hmac_ctrl, ivsize >> 1);
2debd332
HJ		 1658
1659 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
1660 1, dst_size);
1661}
1662
1663int aead_ccm_validate_input(unsigned short op_type,
1664 struct aead_request *req,
1665 struct chcr_aead_ctx *aeadctx,
1666 unsigned int sub_type)
1667{
1668 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
1669 if (crypto_ccm_check_iv(req->iv)) {
1670 pr_err("CCM: IV check fails\n");
1671 return -EINVAL;
1672 }
1673 } else {
1674 if (req->assoclen != 16 && req->assoclen != 20) {
1675 pr_err("RFC4309: Invalid AAD length %d\n",
1676 req->assoclen);
1677 return -EINVAL;
1678 }
1679 }
1680 if (aeadctx->enckey_len == 0) {
1681 pr_err("CCM: Encryption key not set\n");
1682 return -EINVAL;
1683 }
1684 return 0;
1685}
1686
1687unsigned int fill_aead_req_fields(struct sk_buff *skb,
1688 struct aead_request *req,
1689 struct scatterlist *src,
1690 unsigned int ivsize,
1691 struct chcr_aead_ctx *aeadctx)
1692{
1693 unsigned int frags = 0;
1694 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1695 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1696 /* b0 and aad length(if available) */
1697
1698 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
1699 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0));
1700 if (req->assoclen) {
1701 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
1702 write_sg_to_skb(skb, &frags, req->src,
1703 req->assoclen - 8);
1704 else
1705 write_sg_to_skb(skb, &frags, req->src, req->assoclen);
1706 }
1707 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
1708 if (req->cryptlen)
1709 write_sg_to_skb(skb, &frags, src, req->cryptlen);
1710
1711 return frags;
1712}
1713
1714static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
1715 unsigned short qid,
1716 int size,
1717 unsigned short op_type)
1718{
1719 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1720 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1721 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1722 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1723 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1724 struct sk_buff *skb = NULL;
1725 struct chcr_wr *chcr_req;
1726 struct cpl_rx_phys_dsgl *phys_cpl;
1727 struct phys_sge_parm sg_param;
94e1dab1 1728 struct scatterlist *src;
2debd332
HJ		 1729 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
1730 unsigned int dst_size = 0, kctx_len;
1731 unsigned int sub_type;
1732 unsigned int authsize = crypto_aead_authsize(tfm);
0e93708d 1733 int err = -EINVAL, src_nent;
2debd332
HJ		 1734 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1735 GFP_ATOMIC;
1736
1737
1738 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1739 goto err;
0e93708d
HJ		 1740 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1741 if (src_nent < 0)
2debd332 1742 goto err;
0e93708d 1743
2debd332 1744 sub_type = get_aead_subtype(tfm);
94e1dab1
HJ		 1745 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1746 reqctx->dst = src;
1747
2debd332
HJ		 1748 if (req->src != req->dst) {
1749 err = chcr_copy_assoc(req, aeadctx);
1750 if (err) {
1751 pr_err("AAD copy to destination buffer fails\n");
1752 return ERR_PTR(err);
1753 }
94e1dab1
HJ		 1754 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1755 req->assoclen);
2debd332 1756 }
94e1dab1 1757 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2debd332 1758 (op_type ? -authsize : authsize));
0e93708d 1759 if (reqctx->dst_nents < 0) {
2debd332
HJ		 1760 pr_err("CCM:Invalid Destination sg entries\n");
1761 goto err;
1762 }
1763
1764
1765 if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type))
1766 goto err;
1767
1768 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1769 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
1770 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
0e93708d
HJ		 1771 if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
1772 T6_MAX_AAD_SIZE - 18,
1773 transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
1774 op_type)) {
1775 return ERR_PTR(chcr_aead_fallback(req, op_type));
1776 }
1777
2debd332
HJ		 1778 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1779
1780 if (!skb)
1781 goto err;
1782
1783 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1784
1785 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
1786 memset(chcr_req, 0, transhdr_len);
1787
1788 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
1789
1790 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1791 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
1792 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
1793 16), aeadctx->key, aeadctx->enckey_len);
1794
1795 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1796 if (ccm_format_packet(req, aeadctx, sub_type, op_type))
1797 goto dstmap_fail;
1798
1799 sg_param.nents = reqctx->dst_nents;
1800 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1801 sg_param.qid = qid;
1802 sg_param.align = 0;
94e1dab1 1803 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
2debd332
HJ		 1804 &sg_param))
1805 goto dstmap_fail;
1806
1807 skb_set_transport_header(skb, transhdr_len);
1808 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
1809 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
2512a624 1810 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2debd332
HJ		 1811 reqctx->skb = skb;
1812 skb_get(skb);
1813 return skb;
1814dstmap_fail:
1815 kfree_skb(skb);
1816 skb = NULL;
1817err:
1818 return ERR_PTR(-EINVAL);
1819}
1820
1821static struct sk_buff *create_gcm_wr(struct aead_request *req,
1822 unsigned short qid,
1823 int size,
1824 unsigned short op_type)
1825{
1826 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1827 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1828 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1829 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1830 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1831 struct sk_buff *skb = NULL;
1832 struct chcr_wr *chcr_req;
1833 struct cpl_rx_phys_dsgl *phys_cpl;
1834 struct phys_sge_parm sg_param;
94e1dab1 1835 struct scatterlist *src;
2debd332
HJ		 1836 unsigned int frags = 0, transhdr_len;
1837 unsigned int ivsize = AES_BLOCK_SIZE;
1838 unsigned int dst_size = 0, kctx_len;
1839 unsigned char tag_offset = 0;
1840 unsigned int crypt_len = 0;
1841 unsigned int authsize = crypto_aead_authsize(tfm);
0e93708d 1842 int err = -EINVAL, src_nent;
2debd332
HJ		 1843 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1844 GFP_ATOMIC;
1845
1846 /* validate key size */
1847 if (aeadctx->enckey_len == 0)
1848 goto err;
1849
1850 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1851 goto err;
0e93708d
HJ		 1852 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1853 if (src_nent < 0)
2debd332
HJ		 1854 goto err;
1855
94e1dab1
HJ		 1856 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1857 reqctx->dst = src;
2debd332
HJ		 1858 if (req->src != req->dst) {
1859 err = chcr_copy_assoc(req, aeadctx);
1860 if (err)
1861 return ERR_PTR(err);
94e1dab1
HJ		 1862 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1863 req->assoclen);
2debd332
HJ		 1864 }
1865
1866 if (!req->cryptlen)
1867 /* null-payload is not supported in the hardware.
1868 * software is sending block size
1869 */
1870 crypt_len = AES_BLOCK_SIZE;
1871 else
1872 crypt_len = req->cryptlen;
94e1dab1 1873 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2debd332 1874 (op_type ? -authsize : authsize));
0e93708d 1875 if (reqctx->dst_nents < 0) {
2debd332
HJ		 1876 pr_err("GCM:Invalid Destination sg entries\n");
1877 goto err;
1878 }
1879
1880
1881 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1882 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
1883 AEAD_H_SIZE;
1884 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
0e93708d
HJ		 1885 if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG,
1886 T6_MAX_AAD_SIZE,
1887 transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
1888 op_type)) {
1889 return ERR_PTR(chcr_aead_fallback(req, op_type));
1890 }
2debd332
HJ		 1891 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1892 if (!skb)
1893 goto err;
1894
1895 /* NIC driver is going to write the sge hdr. */
1896 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1897
1898 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
1899 memset(chcr_req, 0, transhdr_len);
1900
1901 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
1902 req->assoclen -= 8;
1903
1904 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
1905 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
8a13449f 1906 ctx->dev->rx_channel_id, 2, (ivsize ?
2debd332 1907 (req->assoclen + 1) : 0));
0e93708d
HJ		 1908 chcr_req->sec_cpl.pldlen =
1909 htonl(req->assoclen + ivsize + req->cryptlen);
2debd332
HJ		 1910 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1911 req->assoclen ? 1 : 0, req->assoclen,
1912 req->assoclen + ivsize + 1, 0);
2debd332
HJ		 1913 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1914 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1,
1915 tag_offset, tag_offset);
1916 chcr_req->sec_cpl.seqno_numivs =
1917 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
1918 CHCR_ENCRYPT_OP) ? 1 : 0,
1919 CHCR_SCMD_CIPHER_MODE_AES_GCM,
0a7bd30c
HJ		 1920 CHCR_SCMD_AUTH_MODE_GHASH,
1921 aeadctx->hmac_ctrl, ivsize >> 1);
2debd332
HJ		 1922 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
1923 0, 1, dst_size);
1924 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1925 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
1926 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
1927 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
1928
1929 /* prepare a 16 byte iv */
1930 /* S A L T | IV | 0x00000001 */
1931 if (get_aead_subtype(tfm) ==
1932 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
1933 memcpy(reqctx->iv, aeadctx->salt, 4);
1934 memcpy(reqctx->iv + 4, req->iv, 8);
1935 } else {
1936 memcpy(reqctx->iv, req->iv, 12);
1937 }
1938 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
1939
1940 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1941 sg_param.nents = reqctx->dst_nents;
1942 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1943 sg_param.qid = qid;
1944 sg_param.align = 0;
94e1dab1 1945 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
2debd332
HJ		 1946 &sg_param))
1947 goto dstmap_fail;
1948
1949 skb_set_transport_header(skb, transhdr_len);
1950
1951 write_sg_to_skb(skb, &frags, req->src, req->assoclen);
1952
1953 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
0e93708d 1954 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2debd332 1955 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
2512a624
HJ		 1956 sizeof(struct cpl_rx_phys_dsgl) + dst_size,
1957 reqctx->verify);
2debd332
HJ		 1958 reqctx->skb = skb;
1959 skb_get(skb);
1960 return skb;
1961
1962dstmap_fail:
1963 /* ivmap_fail: */
1964 kfree_skb(skb);
1965 skb = NULL;
1966err:
1967 return skb;
1968}
1969
1970
1971
1972static int chcr_aead_cra_init(struct crypto_aead *tfm)
1973{
1974 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1975 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
0e93708d
HJ		 1976 struct aead_alg *alg = crypto_aead_alg(tfm);
1977
1978 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
1979 CRYPTO_ALG_NEED_FALLBACK);
1980 if (IS_ERR(aeadctx->sw_cipher))
1981 return PTR_ERR(aeadctx->sw_cipher);
1982 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
1983 sizeof(struct aead_request) +
1984 crypto_aead_reqsize(aeadctx->sw_cipher)));
2debd332
HJ		 1985 aeadctx->null = crypto_get_default_null_skcipher();
1986 if (IS_ERR(aeadctx->null))
1987 return PTR_ERR(aeadctx->null);
1988 return chcr_device_init(ctx);
1989}
1990
1991static void chcr_aead_cra_exit(struct crypto_aead *tfm)
1992{
0e93708d
HJ		 1993 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1994 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1995
2debd332 1996 crypto_put_default_null_skcipher();
0e93708d 1997 crypto_free_aead(aeadctx->sw_cipher);
2debd332
HJ		 1998}
1999
2000static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2001 unsigned int authsize)
2002{
2003 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2004
2005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2006 aeadctx->mayverify = VERIFY_HW;
0e93708d 2007 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 2008}
2009static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2010 unsigned int authsize)
2011{
2012 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2013 u32 maxauth = crypto_aead_maxauthsize(tfm);
2014
2015 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2016 * true for sha1. authsize == 12 condition should be before
2017 * authsize == (maxauth >> 1)
2018 */
2019 if (authsize == ICV_4) {
2020 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2021 aeadctx->mayverify = VERIFY_HW;
2022 } else if (authsize == ICV_6) {
2023 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2024 aeadctx->mayverify = VERIFY_HW;
2025 } else if (authsize == ICV_10) {
2026 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2027 aeadctx->mayverify = VERIFY_HW;
2028 } else if (authsize == ICV_12) {
2029 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2030 aeadctx->mayverify = VERIFY_HW;
2031 } else if (authsize == ICV_14) {
2032 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2033 aeadctx->mayverify = VERIFY_HW;
2034 } else if (authsize == (maxauth >> 1)) {
2035 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2036 aeadctx->mayverify = VERIFY_HW;
2037 } else if (authsize == maxauth) {
2038 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2039 aeadctx->mayverify = VERIFY_HW;
2040 } else {
2041 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2042 aeadctx->mayverify = VERIFY_SW;
2043 }
0e93708d 2044 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 2045}
2046
2047
2048static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2049{
2050 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2051
2052 switch (authsize) {
2053 case ICV_4:
2054 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2055 aeadctx->mayverify = VERIFY_HW;
2056 break;
2057 case ICV_8:
2058 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2059 aeadctx->mayverify = VERIFY_HW;
2060 break;
2061 case ICV_12:
2062 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2063 aeadctx->mayverify = VERIFY_HW;
2064 break;
2065 case ICV_14:
2066 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2067 aeadctx->mayverify = VERIFY_HW;
2068 break;
2069 case ICV_16:
2070 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2071 aeadctx->mayverify = VERIFY_HW;
2072 break;
2073 case ICV_13:
2074 case ICV_15:
2075 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2076 aeadctx->mayverify = VERIFY_SW;
2077 break;
2078 default:
2079
2080 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2081 CRYPTO_TFM_RES_BAD_KEY_LEN);
2082 return -EINVAL;
2083 }
0e93708d 2084 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 2085}
2086
2087static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2088 unsigned int authsize)
2089{
2090 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2091
2092 switch (authsize) {
2093 case ICV_8:
2094 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2095 aeadctx->mayverify = VERIFY_HW;
2096 break;
2097 case ICV_12:
2098 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2099 aeadctx->mayverify = VERIFY_HW;
2100 break;
2101 case ICV_16:
2102 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2103 aeadctx->mayverify = VERIFY_HW;
2104 break;
2105 default:
2106 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2107 CRYPTO_TFM_RES_BAD_KEY_LEN);
2108 return -EINVAL;
2109 }
0e93708d 2110 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 2111}
2112
2113static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
2114 unsigned int authsize)
2115{
2116 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2117
2118 switch (authsize) {
2119 case ICV_4:
2120 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2121 aeadctx->mayverify = VERIFY_HW;
2122 break;
2123 case ICV_6:
2124 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2125 aeadctx->mayverify = VERIFY_HW;
2126 break;
2127 case ICV_8:
2128 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2129 aeadctx->mayverify = VERIFY_HW;
2130 break;
2131 case ICV_10:
2132 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2133 aeadctx->mayverify = VERIFY_HW;
2134 break;
2135 case ICV_12:
2136 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2137 aeadctx->mayverify = VERIFY_HW;
2138 break;
2139 case ICV_14:
2140 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2141 aeadctx->mayverify = VERIFY_HW;
2142 break;
2143 case ICV_16:
2144 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2145 aeadctx->mayverify = VERIFY_HW;
2146 break;
2147 default:
2148 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2149 CRYPTO_TFM_RES_BAD_KEY_LEN);
2150 return -EINVAL;
2151 }
0e93708d 2152 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 2153}
2154
0e93708d 2155static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2debd332
HJ		 2156 const u8 *key,
2157 unsigned int keylen)
2158{
2159 struct chcr_context *ctx = crypto_aead_ctx(aead);
2160 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2161 unsigned char ck_size, mk_size;
2162 int key_ctx_size = 0;
2163
2debd332
HJ		 2164 key_ctx_size = sizeof(struct _key_ctx) +
2165 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
2166 if (keylen == AES_KEYSIZE_128) {
2167 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2168 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2169 } else if (keylen == AES_KEYSIZE_192) {
2170 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2171 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2172 } else if (keylen == AES_KEYSIZE_256) {
2173 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2174 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2175 } else {
2176 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2177 CRYPTO_TFM_RES_BAD_KEY_LEN);
2178 aeadctx->enckey_len = 0;
2179 return -EINVAL;
2180 }
2181 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
2182 key_ctx_size >> 4);
0e93708d
HJ		 2183 memcpy(aeadctx->key, key, keylen);
2184 aeadctx->enckey_len = keylen;
2185
2debd332
HJ		 2186 return 0;
2187}
2188
0e93708d
HJ		 2189static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
2190 const u8 *key,
2191 unsigned int keylen)
2192{
2193 struct chcr_context *ctx = crypto_aead_ctx(aead);
2194 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2195 int error;
2196
2197 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2198 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2199 CRYPTO_TFM_REQ_MASK);
2200 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2201 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2202 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2203 CRYPTO_TFM_RES_MASK);
2204 if (error)
2205 return error;
2206 return chcr_ccm_common_setkey(aead, key, keylen);
2207}
2208
2debd332
HJ		 2209static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
2210 unsigned int keylen)
2211{
2212 struct chcr_context *ctx = crypto_aead_ctx(aead);
4dbeae42
HJ		 2213 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2214 int error;
2debd332
HJ		 2215
2216 if (keylen < 3) {
2217 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2218 CRYPTO_TFM_RES_BAD_KEY_LEN);
2219 aeadctx->enckey_len = 0;
2220 return -EINVAL;
2221 }
4dbeae42
HJ		 2222 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2223 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2224 CRYPTO_TFM_REQ_MASK);
2225 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2226 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2227 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2228 CRYPTO_TFM_RES_MASK);
2229 if (error)
2230 return error;
2debd332
HJ		 2231 keylen -= 3;
2232 memcpy(aeadctx->salt, key + keylen, 3);
0e93708d 2233 return chcr_ccm_common_setkey(aead, key, keylen);
2debd332
HJ		 2234}
2235
2236static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2237 unsigned int keylen)
2238{
2239 struct chcr_context *ctx = crypto_aead_ctx(aead);
2240 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2241 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
8356ea51 2242 struct crypto_cipher *cipher;
2debd332
HJ		 2243 unsigned int ck_size;
2244 int ret = 0, key_ctx_size = 0;
2245
0e93708d
HJ		 2246 aeadctx->enckey_len = 0;
2247 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2248 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
2249 & CRYPTO_TFM_REQ_MASK);
2250 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2251 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2252 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2253 CRYPTO_TFM_RES_MASK);
2254 if (ret)
2255 goto out;
2256
7c2cf1c4
HJ		 2257 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
2258 keylen > 3) {
2debd332
HJ		 2259 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
2260 memcpy(aeadctx->salt, key + keylen, 4);
2261 }
2262 if (keylen == AES_KEYSIZE_128) {
2263 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2264 } else if (keylen == AES_KEYSIZE_192) {
2265 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2266 } else if (keylen == AES_KEYSIZE_256) {
2267 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2268 } else {
2269 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2270 CRYPTO_TFM_RES_BAD_KEY_LEN);
0e93708d 2271 pr_err("GCM: Invalid key length %d\n", keylen);
2debd332
HJ		 2272 ret = -EINVAL;
2273 goto out;
2274 }
2275
2276 memcpy(aeadctx->key, key, keylen);
2277 aeadctx->enckey_len = keylen;
2278 key_ctx_size = sizeof(struct _key_ctx) +
2279 ((DIV_ROUND_UP(keylen, 16)) << 4) +
2280 AEAD_H_SIZE;
2281 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2282 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2283 0, 0,
2284 key_ctx_size >> 4);
8356ea51
HJ		 2285 /* Calculate the H = CIPH(K, 0 repeated 16 times).
2286 * It will go in key context
2debd332 2287 */
8356ea51
HJ		 2288 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2289 if (IS_ERR(cipher)) {
2debd332
HJ		 2290 aeadctx->enckey_len = 0;
2291 ret = -ENOMEM;
2292 goto out;
2293 }
8356ea51
HJ		 2294
2295 ret = crypto_cipher_setkey(cipher, key, keylen);
2debd332
HJ		 2296 if (ret) {
2297 aeadctx->enckey_len = 0;
2298 goto out1;
2299 }
2300 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
8356ea51 2301 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2debd332
HJ		 2302
2303out1:
8356ea51 2304 crypto_free_cipher(cipher);
2debd332
HJ		 2305out:
2306 return ret;
2307}
2308
2309static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
2310 unsigned int keylen)
2311{
2312 struct chcr_context *ctx = crypto_aead_ctx(authenc);
2313 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2314 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2315 /* it contains auth and cipher key both*/
2316 struct crypto_authenc_keys keys;
2317 unsigned int bs;
2318 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
2319 int err = 0, i, key_ctx_len = 0;
2320 unsigned char ck_size = 0;
2321 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
ec1bca94 2322 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2debd332
HJ		 2323 struct algo_param param;
2324 int align;
2325 u8 *o_ptr = NULL;
2326
0e93708d
HJ		 2327 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2328 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
2329 & CRYPTO_TFM_REQ_MASK);
2330 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2331 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
2332 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
2333 & CRYPTO_TFM_RES_MASK);
2334 if (err)
2335 goto out;
2336
2debd332
HJ		 2337 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2338 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2339 goto out;
2340 }
2341
2342 if (get_alg_config(&param, max_authsize)) {
2343 pr_err("chcr : Unsupported digest size\n");
2344 goto out;
2345 }
2346 if (keys.enckeylen == AES_KEYSIZE_128) {
2347 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2348 } else if (keys.enckeylen == AES_KEYSIZE_192) {
2349 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2350 } else if (keys.enckeylen == AES_KEYSIZE_256) {
2351 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2352 } else {
2353 pr_err("chcr : Unsupported cipher key\n");
2354 goto out;
2355 }
2356
2357 /* Copy only encryption key. We use authkey to generate h(ipad) and
2358 * h(opad) so authkey is not needed again. authkeylen size have the
2359 * size of the hash digest size.
2360 */
2361 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2362 aeadctx->enckey_len = keys.enckeylen;
2363 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2364 aeadctx->enckey_len << 3);
2365
2366 base_hash = chcr_alloc_shash(max_authsize);
2367 if (IS_ERR(base_hash)) {
2368 pr_err("chcr : Base driver cannot be loaded\n");
0e93708d
HJ		 2369 aeadctx->enckey_len = 0;
2370 return -EINVAL;
324429d7 2371 }
2debd332
HJ		 2372 {
2373 SHASH_DESC_ON_STACK(shash, base_hash);
2374 shash->tfm = base_hash;
2375 shash->flags = crypto_shash_get_flags(base_hash);
2376 bs = crypto_shash_blocksize(base_hash);
2377 align = KEYCTX_ALIGN_PAD(max_authsize);
2378 o_ptr = actx->h_iopad + param.result_size + align;
2379
2380 if (keys.authkeylen > bs) {
2381 err = crypto_shash_digest(shash, keys.authkey,
2382 keys.authkeylen,
2383 o_ptr);
2384 if (err) {
2385 pr_err("chcr : Base driver cannot be loaded\n");
2386 goto out;
2387 }
2388 keys.authkeylen = max_authsize;
2389 } else
2390 memcpy(o_ptr, keys.authkey, keys.authkeylen);
2391
2392 /* Compute the ipad-digest*/
2393 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2394 memcpy(pad, o_ptr, keys.authkeylen);
2395 for (i = 0; i < bs >> 2; i++)
2396 *((unsigned int *)pad + i) ^= IPAD_DATA;
2397
2398 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
2399 max_authsize))
2400 goto out;
2401 /* Compute the opad-digest */
2402 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2403 memcpy(pad, o_ptr, keys.authkeylen);
2404 for (i = 0; i < bs >> 2; i++)
2405 *((unsigned int *)pad + i) ^= OPAD_DATA;
2406
2407 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
2408 goto out;
2409
2410 /* convert the ipad and opad digest to network order */
2411 chcr_change_order(actx->h_iopad, param.result_size);
2412 chcr_change_order(o_ptr, param.result_size);
2413 key_ctx_len = sizeof(struct _key_ctx) +
2414 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
2415 (param.result_size + align) * 2;
2416 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
2417 0, 1, key_ctx_len >> 4);
2418 actx->auth_mode = param.auth_mode;
2419 chcr_free_shash(base_hash);
2420
2421 return 0;
2422 }
2423out:
2424 aeadctx->enckey_len = 0;
ec1bca94 2425 if (!IS_ERR(base_hash))
2debd332
HJ		 2426 chcr_free_shash(base_hash);
2427 return -EINVAL;
324429d7
HS		 2428}
2429
2debd332
HJ		 2430static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
2431 const u8 *key, unsigned int keylen)
2432{
2433 struct chcr_context *ctx = crypto_aead_ctx(authenc);
2434 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2435 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2436 struct crypto_authenc_keys keys;
0e93708d 2437 int err;
2debd332
HJ		 2438 /* it contains auth and cipher key both*/
2439 int key_ctx_len = 0;
2440 unsigned char ck_size = 0;
2441
0e93708d
HJ		 2442 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2443 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
2444 & CRYPTO_TFM_REQ_MASK);
2445 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2446 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
2447 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
2448 & CRYPTO_TFM_RES_MASK);
2449 if (err)
2450 goto out;
2451
2debd332
HJ		 2452 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2453 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2454 goto out;
2455 }
2456 if (keys.enckeylen == AES_KEYSIZE_128) {
2457 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2458 } else if (keys.enckeylen == AES_KEYSIZE_192) {
2459 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2460 } else if (keys.enckeylen == AES_KEYSIZE_256) {
2461 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2462 } else {
2463 pr_err("chcr : Unsupported cipher key\n");
2464 goto out;
2465 }
2466 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2467 aeadctx->enckey_len = keys.enckeylen;
2468 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2469 aeadctx->enckey_len << 3);
2470 key_ctx_len = sizeof(struct _key_ctx)
2471 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
2472
2473 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
2474 0, key_ctx_len >> 4);
2475 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
2476 return 0;
2477out:
2478 aeadctx->enckey_len = 0;
2479 return -EINVAL;
2480}
2481static int chcr_aead_encrypt(struct aead_request *req)
2482{
2483 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2484 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2485
2486 reqctx->verify = VERIFY_HW;
2487
2488 switch (get_aead_subtype(tfm)) {
2489 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
2490 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
2491 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2492 create_authenc_wr);
2493 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
2494 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
2495 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2496 create_aead_ccm_wr);
2497 default:
2498 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2499 create_gcm_wr);
2500 }
2501}
2502
2503static int chcr_aead_decrypt(struct aead_request *req)
2504{
2505 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2506 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2507 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2508 int size;
2509
2510 if (aeadctx->mayverify == VERIFY_SW) {
2511 size = crypto_aead_maxauthsize(tfm);
2512 reqctx->verify = VERIFY_SW;
2513 } else {
2514 size = 0;
2515 reqctx->verify = VERIFY_HW;
2516 }
2517
2518 switch (get_aead_subtype(tfm)) {
2519 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
2520 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
2521 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2522 create_authenc_wr);
2523 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
2524 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
2525 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2526 create_aead_ccm_wr);
2527 default:
2528 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2529 create_gcm_wr);
2530 }
2531}
2532
2533static int chcr_aead_op(struct aead_request *req,
2534 unsigned short op_type,
2535 int size,
2536 create_wr_t create_wr_fn)
2537{
2538 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2539 struct chcr_context *ctx = crypto_aead_ctx(tfm);
5ba042c0 2540 struct uld_ctx *u_ctx;
2debd332
HJ		 2541 struct sk_buff *skb;
2542
5ba042c0 2543 if (!ctx->dev) {
2debd332
HJ		 2544 pr_err("chcr : %s : No crypto device.\n", __func__);
2545 return -ENXIO;
2546 }
5ba042c0 2547 u_ctx = ULD_CTX(ctx);
2debd332 2548 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
72a56ca9 2549 ctx->tx_qidx)) {
2debd332
HJ		 2550 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
2551 return -EBUSY;
2552 }
2553
2554 /* Form a WR from req */
72a56ca9 2555 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size,
2debd332
HJ		 2556 op_type);
2557
0e93708d 2558 if (IS_ERR(skb) || !skb)
2debd332 2559 return PTR_ERR(skb);
2debd332
HJ		 2560
2561 skb->dev = u_ctx->lldi.ports[0];
72a56ca9 2562 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
2debd332
HJ		 2563 chcr_send_wr(skb);
2564 return -EINPROGRESS;
2565}
324429d7
HS		 2566static struct chcr_alg_template driver_algs[] = {
2567 /* AES-CBC */
2568 {
2569 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2570 .is_registered = 0,
2571 .alg.crypto = {
2572 .cra_name = "cbc(aes)",
2debd332 2573 .cra_driver_name = "cbc-aes-chcr",
324429d7 2574 .cra_priority = CHCR_CRA_PRIORITY,
44e9f799 2575 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
324429d7
HS		 2576 CRYPTO_ALG_ASYNC,
2577 .cra_blocksize = AES_BLOCK_SIZE,
2578 .cra_ctxsize = sizeof(struct chcr_context)
2579 + sizeof(struct ablk_ctx),
2580 .cra_alignmask = 0,
2581 .cra_type = &crypto_ablkcipher_type,
2582 .cra_module = THIS_MODULE,
2583 .cra_init = chcr_cra_init,
2584 .cra_exit = NULL,
2585 .cra_u.ablkcipher = {
2586 .min_keysize = AES_MIN_KEY_SIZE,
2587 .max_keysize = AES_MAX_KEY_SIZE,
2588 .ivsize = AES_BLOCK_SIZE,
2589 .setkey = chcr_aes_cbc_setkey,
2590 .encrypt = chcr_aes_encrypt,
2591 .decrypt = chcr_aes_decrypt,
2592 }
2593 }
2594 },
2595 {
2596 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2597 .is_registered = 0,
2598 .alg.crypto = {
2599 .cra_name = "xts(aes)",
2debd332 2600 .cra_driver_name = "xts-aes-chcr",
324429d7 2601 .cra_priority = CHCR_CRA_PRIORITY,
44e9f799 2602 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
324429d7
HS		 2603 CRYPTO_ALG_ASYNC,
2604 .cra_blocksize = AES_BLOCK_SIZE,
2605 .cra_ctxsize = sizeof(struct chcr_context) +
2606 sizeof(struct ablk_ctx),
2607 .cra_alignmask = 0,
2608 .cra_type = &crypto_ablkcipher_type,
2609 .cra_module = THIS_MODULE,
2610 .cra_init = chcr_cra_init,
2611 .cra_exit = NULL,
2612 .cra_u = {
2613 .ablkcipher = {
2614 .min_keysize = 2 * AES_MIN_KEY_SIZE,
2615 .max_keysize = 2 * AES_MAX_KEY_SIZE,
2616 .ivsize = AES_BLOCK_SIZE,
2617 .setkey = chcr_aes_xts_setkey,
2618 .encrypt = chcr_aes_encrypt,
2619 .decrypt = chcr_aes_decrypt,
2620 }
2621 }
2622 }
2623 },
2624 /* SHA */
2625 {
2626 .type = CRYPTO_ALG_TYPE_AHASH,
2627 .is_registered = 0,
2628 .alg.hash = {
2629 .halg.digestsize = SHA1_DIGEST_SIZE,
2630 .halg.base = {
2631 .cra_name = "sha1",
2632 .cra_driver_name = "sha1-chcr",
2633 .cra_blocksize = SHA1_BLOCK_SIZE,
2634 }
2635 }
2636 },
2637 {
2638 .type = CRYPTO_ALG_TYPE_AHASH,
2639 .is_registered = 0,
2640 .alg.hash = {
2641 .halg.digestsize = SHA256_DIGEST_SIZE,
2642 .halg.base = {
2643 .cra_name = "sha256",
2644 .cra_driver_name = "sha256-chcr",
2645 .cra_blocksize = SHA256_BLOCK_SIZE,
2646 }
2647 }
2648 },
2649 {
2650 .type = CRYPTO_ALG_TYPE_AHASH,
2651 .is_registered = 0,
2652 .alg.hash = {
2653 .halg.digestsize = SHA224_DIGEST_SIZE,
2654 .halg.base = {
2655 .cra_name = "sha224",
2656 .cra_driver_name = "sha224-chcr",
2657 .cra_blocksize = SHA224_BLOCK_SIZE,
2658 }
2659 }
2660 },
2661 {
2662 .type = CRYPTO_ALG_TYPE_AHASH,
2663 .is_registered = 0,
2664 .alg.hash = {
2665 .halg.digestsize = SHA384_DIGEST_SIZE,
2666 .halg.base = {
2667 .cra_name = "sha384",
2668 .cra_driver_name = "sha384-chcr",
2669 .cra_blocksize = SHA384_BLOCK_SIZE,
2670 }
2671 }
2672 },
2673 {
2674 .type = CRYPTO_ALG_TYPE_AHASH,
2675 .is_registered = 0,
2676 .alg.hash = {
2677 .halg.digestsize = SHA512_DIGEST_SIZE,
2678 .halg.base = {
2679 .cra_name = "sha512",
2680 .cra_driver_name = "sha512-chcr",
2681 .cra_blocksize = SHA512_BLOCK_SIZE,
2682 }
2683 }
2684 },
2685 /* HMAC */
2686 {
2687 .type = CRYPTO_ALG_TYPE_HMAC,
2688 .is_registered = 0,
2689 .alg.hash = {
2690 .halg.digestsize = SHA1_DIGEST_SIZE,
2691 .halg.base = {
2692 .cra_name = "hmac(sha1)",
2debd332 2693 .cra_driver_name = "hmac-sha1-chcr",
324429d7
HS		 2694 .cra_blocksize = SHA1_BLOCK_SIZE,
2695 }
2696 }
2697 },
2698 {
2699 .type = CRYPTO_ALG_TYPE_HMAC,
2700 .is_registered = 0,
2701 .alg.hash = {
2702 .halg.digestsize = SHA224_DIGEST_SIZE,
2703 .halg.base = {
2704 .cra_name = "hmac(sha224)",
2debd332 2705 .cra_driver_name = "hmac-sha224-chcr",
324429d7
HS		 2706 .cra_blocksize = SHA224_BLOCK_SIZE,
2707 }
2708 }
2709 },
2710 {
2711 .type = CRYPTO_ALG_TYPE_HMAC,
2712 .is_registered = 0,
2713 .alg.hash = {
2714 .halg.digestsize = SHA256_DIGEST_SIZE,
2715 .halg.base = {
2716 .cra_name = "hmac(sha256)",
2debd332 2717 .cra_driver_name = "hmac-sha256-chcr",
324429d7
HS		 2718 .cra_blocksize = SHA256_BLOCK_SIZE,
2719 }
2720 }
2721 },
2722 {
2723 .type = CRYPTO_ALG_TYPE_HMAC,
2724 .is_registered = 0,
2725 .alg.hash = {
2726 .halg.digestsize = SHA384_DIGEST_SIZE,
2727 .halg.base = {
2728 .cra_name = "hmac(sha384)",
2debd332 2729 .cra_driver_name = "hmac-sha384-chcr",
324429d7
HS		 2730 .cra_blocksize = SHA384_BLOCK_SIZE,
2731 }
2732 }
2733 },
2734 {
2735 .type = CRYPTO_ALG_TYPE_HMAC,
2736 .is_registered = 0,
2737 .alg.hash = {
2738 .halg.digestsize = SHA512_DIGEST_SIZE,
2739 .halg.base = {
2740 .cra_name = "hmac(sha512)",
2debd332 2741 .cra_driver_name = "hmac-sha512-chcr",
324429d7
HS		 2742 .cra_blocksize = SHA512_BLOCK_SIZE,
2743 }
2744 }
2745 },
2debd332
HJ		 2746 /* Add AEAD Algorithms */
2747 {
2748 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
2749 .is_registered = 0,
2750 .alg.aead = {
2751 .base = {
2752 .cra_name = "gcm(aes)",
2753 .cra_driver_name = "gcm-aes-chcr",
2754 .cra_blocksize = 1,
e29abda5 2755 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2756 .cra_ctxsize = sizeof(struct chcr_context) +
2757 sizeof(struct chcr_aead_ctx) +
2758 sizeof(struct chcr_gcm_ctx),
2759 },
2760 .ivsize = 12,
2761 .maxauthsize = GHASH_DIGEST_SIZE,
2762 .setkey = chcr_gcm_setkey,
2763 .setauthsize = chcr_gcm_setauthsize,
2764 }
2765 },
2766 {
2767 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
2768 .is_registered = 0,
2769 .alg.aead = {
2770 .base = {
2771 .cra_name = "rfc4106(gcm(aes))",
2772 .cra_driver_name = "rfc4106-gcm-aes-chcr",
2773 .cra_blocksize = 1,
e29abda5 2774 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 2775 .cra_ctxsize = sizeof(struct chcr_context) +
2776 sizeof(struct chcr_aead_ctx) +
2777 sizeof(struct chcr_gcm_ctx),
2778
2779 },
2780 .ivsize = 8,
2781 .maxauthsize = GHASH_DIGEST_SIZE,
2782 .setkey = chcr_gcm_setkey,
2783 .setauthsize = chcr_4106_4309_setauthsize,
2784 }
2785 },
2786 {
2787 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
2788 .is_registered = 0,
2789 .alg.aead = {
2790 .base = {
2791 .cra_name = "ccm(aes)",
2792 .cra_driver_name = "ccm-aes-chcr",
2793 .cra_blocksize = 1,
e29abda5 2794 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2795 .cra_ctxsize = sizeof(struct chcr_context) +
2796 sizeof(struct chcr_aead_ctx),
2797
2798 },
2799 .ivsize = AES_BLOCK_SIZE,
2800 .maxauthsize = GHASH_DIGEST_SIZE,
2801 .setkey = chcr_aead_ccm_setkey,
2802 .setauthsize = chcr_ccm_setauthsize,
2803 }
2804 },
2805 {
2806 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
2807 .is_registered = 0,
2808 .alg.aead = {
2809 .base = {
2810 .cra_name = "rfc4309(ccm(aes))",
2811 .cra_driver_name = "rfc4309-ccm-aes-chcr",
2812 .cra_blocksize = 1,
e29abda5 2813 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 2814 .cra_ctxsize = sizeof(struct chcr_context) +
2815 sizeof(struct chcr_aead_ctx),
2816
2817 },
2818 .ivsize = 8,
2819 .maxauthsize = GHASH_DIGEST_SIZE,
2820 .setkey = chcr_aead_rfc4309_setkey,
2821 .setauthsize = chcr_4106_4309_setauthsize,
2822 }
2823 },
2824 {
2825 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2826 .is_registered = 0,
2827 .alg.aead = {
2828 .base = {
2829 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2830 .cra_driver_name =
2831 "authenc-hmac-sha1-cbc-aes-chcr",
2832 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2833 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2834 .cra_ctxsize = sizeof(struct chcr_context) +
2835 sizeof(struct chcr_aead_ctx) +
2836 sizeof(struct chcr_authenc_ctx),
2837
2838 },
2839 .ivsize = AES_BLOCK_SIZE,
2840 .maxauthsize = SHA1_DIGEST_SIZE,
2841 .setkey = chcr_authenc_setkey,
2842 .setauthsize = chcr_authenc_setauthsize,
2843 }
2844 },
2845 {
2846 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2847 .is_registered = 0,
2848 .alg.aead = {
2849 .base = {
2850
2851 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2852 .cra_driver_name =
2853 "authenc-hmac-sha256-cbc-aes-chcr",
2854 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2855 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2856 .cra_ctxsize = sizeof(struct chcr_context) +
2857 sizeof(struct chcr_aead_ctx) +
2858 sizeof(struct chcr_authenc_ctx),
2859
2860 },
2861 .ivsize = AES_BLOCK_SIZE,
2862 .maxauthsize = SHA256_DIGEST_SIZE,
2863 .setkey = chcr_authenc_setkey,
2864 .setauthsize = chcr_authenc_setauthsize,
2865 }
2866 },
2867 {
2868 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2869 .is_registered = 0,
2870 .alg.aead = {
2871 .base = {
2872 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2873 .cra_driver_name =
2874 "authenc-hmac-sha224-cbc-aes-chcr",
2875 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2876 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2877 .cra_ctxsize = sizeof(struct chcr_context) +
2878 sizeof(struct chcr_aead_ctx) +
2879 sizeof(struct chcr_authenc_ctx),
2880 },
2881 .ivsize = AES_BLOCK_SIZE,
2882 .maxauthsize = SHA224_DIGEST_SIZE,
2883 .setkey = chcr_authenc_setkey,
2884 .setauthsize = chcr_authenc_setauthsize,
2885 }
2886 },
2887 {
2888 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2889 .is_registered = 0,
2890 .alg.aead = {
2891 .base = {
2892 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2893 .cra_driver_name =
2894 "authenc-hmac-sha384-cbc-aes-chcr",
2895 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2896 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2897 .cra_ctxsize = sizeof(struct chcr_context) +
2898 sizeof(struct chcr_aead_ctx) +
2899 sizeof(struct chcr_authenc_ctx),
2900
2901 },
2902 .ivsize = AES_BLOCK_SIZE,
2903 .maxauthsize = SHA384_DIGEST_SIZE,
2904 .setkey = chcr_authenc_setkey,
2905 .setauthsize = chcr_authenc_setauthsize,
2906 }
2907 },
2908 {
2909 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2910 .is_registered = 0,
2911 .alg.aead = {
2912 .base = {
2913 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2914 .cra_driver_name =
2915 "authenc-hmac-sha512-cbc-aes-chcr",
2916 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2917 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2918 .cra_ctxsize = sizeof(struct chcr_context) +
2919 sizeof(struct chcr_aead_ctx) +
2920 sizeof(struct chcr_authenc_ctx),
2921
2922 },
2923 .ivsize = AES_BLOCK_SIZE,
2924 .maxauthsize = SHA512_DIGEST_SIZE,
2925 .setkey = chcr_authenc_setkey,
2926 .setauthsize = chcr_authenc_setauthsize,
2927 }
2928 },
2929 {
2930 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
2931 .is_registered = 0,
2932 .alg.aead = {
2933 .base = {
2934 .cra_name = "authenc(digest_null,cbc(aes))",
2935 .cra_driver_name =
2936 "authenc-digest_null-cbc-aes-chcr",
2937 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 2938 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 2939 .cra_ctxsize = sizeof(struct chcr_context) +
2940 sizeof(struct chcr_aead_ctx) +
2941 sizeof(struct chcr_authenc_ctx),
2942
2943 },
2944 .ivsize = AES_BLOCK_SIZE,
2945 .maxauthsize = 0,
2946 .setkey = chcr_aead_digest_null_setkey,
2947 .setauthsize = chcr_authenc_null_setauthsize,
2948 }
2949 },
324429d7
HS		 2950};
2951
2952/*
2953 * chcr_unregister_alg - Deregister crypto algorithms with
2954 * kernel framework.
2955 */
2956static int chcr_unregister_alg(void)
2957{
2958 int i;
2959
2960 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2961 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
2962 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2963 if (driver_algs[i].is_registered)
2964 crypto_unregister_alg(
2965 &driver_algs[i].alg.crypto);
2966 break;
2debd332
HJ		 2967 case CRYPTO_ALG_TYPE_AEAD:
2968 if (driver_algs[i].is_registered)
2969 crypto_unregister_aead(
2970 &driver_algs[i].alg.aead);
2971 break;
324429d7
HS		 2972 case CRYPTO_ALG_TYPE_AHASH:
2973 if (driver_algs[i].is_registered)
2974 crypto_unregister_ahash(
2975 &driver_algs[i].alg.hash);
2976 break;
2977 }
2978 driver_algs[i].is_registered = 0;
2979 }
2980 return 0;
2981}
2982
2983#define SZ_AHASH_CTX sizeof(struct chcr_context)
2984#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
2985#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
2986#define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
2987
2988/*
2989 * chcr_register_alg - Register crypto algorithms with kernel framework.
2990 */
2991static int chcr_register_alg(void)
2992{
2993 struct crypto_alg ai;
2994 struct ahash_alg *a_hash;
2995 int err = 0, i;
2996 char *name = NULL;
2997
2998 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2999 if (driver_algs[i].is_registered)
3000 continue;
3001 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3002 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3003 err = crypto_register_alg(&driver_algs[i].alg.crypto);
3004 name = driver_algs[i].alg.crypto.cra_driver_name;
3005 break;
2debd332 3006 case CRYPTO_ALG_TYPE_AEAD:
2debd332 3007 driver_algs[i].alg.aead.base.cra_flags =
0e93708d
HJ		 3008 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
3009 CRYPTO_ALG_NEED_FALLBACK;
2debd332
HJ		 3010 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
3011 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
3012 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
3013 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
3014 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
3015 err = crypto_register_aead(&driver_algs[i].alg.aead);
3016 name = driver_algs[i].alg.aead.base.cra_driver_name;
3017 break;
324429d7
HS		 3018 case CRYPTO_ALG_TYPE_AHASH:
3019 a_hash = &driver_algs[i].alg.hash;
3020 a_hash->update = chcr_ahash_update;
3021 a_hash->final = chcr_ahash_final;
3022 a_hash->finup = chcr_ahash_finup;
3023 a_hash->digest = chcr_ahash_digest;
3024 a_hash->export = chcr_ahash_export;
3025 a_hash->import = chcr_ahash_import;
3026 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
3027 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
3028 a_hash->halg.base.cra_module = THIS_MODULE;
3029 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
3030 a_hash->halg.base.cra_alignmask = 0;
3031 a_hash->halg.base.cra_exit = NULL;
3032 a_hash->halg.base.cra_type = &crypto_ahash_type;
3033
3034 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
3035 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
3036 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
3037 a_hash->init = chcr_hmac_init;
3038 a_hash->setkey = chcr_ahash_setkey;
3039 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
3040 } else {
3041 a_hash->init = chcr_sha_init;
3042 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
3043 a_hash->halg.base.cra_init = chcr_sha_cra_init;
3044 }
3045 err = crypto_register_ahash(&driver_algs[i].alg.hash);
3046 ai = driver_algs[i].alg.hash.halg.base;
3047 name = ai.cra_driver_name;
3048 break;
3049 }
3050 if (err) {
3051 pr_err("chcr : %s : Algorithm registration failed\n",
3052 name);
3053 goto register_err;
3054 } else {
3055 driver_algs[i].is_registered = 1;
3056 }
3057 }
3058 return 0;
3059
3060register_err:
3061 chcr_unregister_alg();
3062 return err;
3063}
3064
3065/*
3066 * start_crypto - Register the crypto algorithms.
3067 * This should called once when the first device comesup. After this
3068 * kernel will start calling driver APIs for crypto operations.
3069 */
3070int start_crypto(void)
3071{
3072 return chcr_register_alg();
3073}
3074
3075/*
3076 * stop_crypto - Deregister all the crypto algorithms with kernel.
3077 * This should be called once when the last device goes down. After this
3078 * kernel will not call the driver API for crypto operations.
3079 */
3080int stop_crypto(void)
3081{
3082 chcr_unregister_alg();
3083 return 0;
3084}
Linux 6.x block layer and io_uring tree(s)