drivers/crypto/inside-secure/safexcel_cipher.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <antoine.tenart@free-electrons.com>
   6  */
   7
   8 #include <asm/unaligned.h>
   9 #include <linux/device.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmapool.h>
  12 #include <crypto/aead.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/authenc.h>
  15 #include <crypto/chacha.h>
  16 #include <crypto/ctr.h>
  17 #include <crypto/internal/des.h>
  18 #include <crypto/gcm.h>
  19 #include <crypto/ghash.h>
  20 #include <crypto/poly1305.h>
  21 #include <crypto/sha1.h>
  22 #include <crypto/sha2.h>
  23 #include <crypto/sm3.h>
  24 #include <crypto/sm4.h>
  25 #include <crypto/xts.h>
  26 #include <crypto/skcipher.h>
  27 #include <crypto/internal/aead.h>
  28 #include <crypto/internal/skcipher.h>
  29
  30 #include "safexcel.h"
  31
  32 enum safexcel_cipher_direction {
  33         SAFEXCEL_ENCRYPT,
  34         SAFEXCEL_DECRYPT,
  35 };
  36
  37 enum safexcel_cipher_alg {
  38         SAFEXCEL_DES,
  39         SAFEXCEL_3DES,
  40         SAFEXCEL_AES,
  41         SAFEXCEL_CHACHA20,
  42         SAFEXCEL_SM4,
  43 };
  44
  45 struct safexcel_cipher_ctx {
  46         struct safexcel_context base;
  47         struct safexcel_crypto_priv *priv;
  48
  49         u32 mode;
  50         enum safexcel_cipher_alg alg;
  51         u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */
  52         u8 xcm;  /* 0=authenc, 1=GCM, 2 reserved for CCM */
  53         u8 aadskip;
  54         u8 blocksz;
  55         u32 ivmask;
  56         u32 ctrinit;
  57
  58         __le32 key[16];
  59         u32 nonce;
  60         unsigned int key_len, xts;
  61
  62         /* All the below is AEAD specific */
  63         u32 hash_alg;
  64         u32 state_sz;
  65
  66         struct crypto_aead *fback;
  67 };
  68
  69 struct safexcel_cipher_req {
  70         enum safexcel_cipher_direction direction;
  71         /* Number of result descriptors associated to the request */
  72         unsigned int rdescs;
  73         bool needs_inv;
  74         int  nr_src, nr_dst;
  75 };
  76
  77 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
  78                                 struct safexcel_command_desc *cdesc)
  79 {
  80         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
  81                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  82                 /* 32 bit nonce */
  83                 cdesc->control_data.token[0] = ctx->nonce;
  84                 /* 64 bit IV part */
  85                 memcpy(&cdesc->control_data.token[1], iv, 8);
  86                 /* 32 bit counter, start at 0 or 1 (big endian!) */
  87                 cdesc->control_data.token[3] =
  88                         (__force u32)cpu_to_be32(ctx->ctrinit);
  89                 return 4;
  90         }
  91         if (ctx->alg == SAFEXCEL_CHACHA20) {
  92                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  93                 /* 96 bit nonce part */
  94                 memcpy(&cdesc->control_data.token[0], &iv[4], 12);
  95                 /* 32 bit counter */
  96                 cdesc->control_data.token[3] = *(u32 *)iv;
  97                 return 4;
  98         }
  99
 100         cdesc->control_data.options |= ctx->ivmask;
 101         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 102         return ctx->blocksz / sizeof(u32);
 103 }
 104
 105 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 106                                     struct safexcel_command_desc *cdesc,
 107                                     struct safexcel_token *atoken,
 108                                     u32 length)
 109 {
 110         struct safexcel_token *token;
 111         int ivlen;
 112
 113         ivlen = safexcel_skcipher_iv(ctx, iv, cdesc);
 114         if (ivlen == 4) {
 115                 /* No space in cdesc, instruction moves to atoken */
 116                 cdesc->additional_cdata_size = 1;
 117                 token = atoken;
 118         } else {
 119                 /* Everything fits in cdesc */
 120                 token = (struct safexcel_token *)(cdesc->control_data.token + 2);
 121                 /* Need to pad with NOP */
 122                 eip197_noop_token(&token[1]);
 123         }
 124
 125         token->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 126         token->packet_length = length;
 127         token->stat = EIP197_TOKEN_STAT_LAST_PACKET |
 128                       EIP197_TOKEN_STAT_LAST_HASH;
 129         token->instructions = EIP197_TOKEN_INS_LAST |
 130                               EIP197_TOKEN_INS_TYPE_CRYPTO |
 131                               EIP197_TOKEN_INS_TYPE_OUTPUT;
 132 }
 133
 134 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv,
 135                              struct safexcel_command_desc *cdesc)
 136 {
 137         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD ||
 138             ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */
 139                 /* 32 bit nonce */
 140                 cdesc->control_data.token[0] = ctx->nonce;
 141                 /* 64 bit IV part */
 142                 memcpy(&cdesc->control_data.token[1], iv, 8);
 143                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 144                 cdesc->control_data.token[3] =
 145                         (__force u32)cpu_to_be32(ctx->ctrinit);
 146                 return;
 147         }
 148         if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) {
 149                 /* 96 bit IV part */
 150                 memcpy(&cdesc->control_data.token[0], iv, 12);
 151                 /* 32 bit counter, start at 0 or 1 (big endian!) */
 152                 cdesc->control_data.token[3] =
 153                         (__force u32)cpu_to_be32(ctx->ctrinit);
 154                 return;
 155         }
 156         /* CBC */
 157         memcpy(cdesc->control_data.token, iv, ctx->blocksz);
 158 }
 159
 160 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
 161                                 struct safexcel_command_desc *cdesc,
 162                                 struct safexcel_token *atoken,
 163                                 enum safexcel_cipher_direction direction,
 164                                 u32 cryptlen, u32 assoclen, u32 digestsize)
 165 {
 166         struct safexcel_token *aadref;
 167         int atoksize = 2; /* Start with minimum size */
 168         int assocadj = assoclen - ctx->aadskip, aadalign;
 169
 170         /* Always 4 dwords of embedded IV  for AEAD modes */
 171         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 172
 173         if (direction == SAFEXCEL_DECRYPT)
 174                 cryptlen -= digestsize;
 175
 176         if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) {
 177                 /* Construct IV block B0 for the CBC-MAC */
 178                 u8 *final_iv = (u8 *)cdesc->control_data.token;
 179                 u8 *cbcmaciv = (u8 *)&atoken[1];
 180                 __le32 *aadlen = (__le32 *)&atoken[5];
 181
 182                 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 183                         /* Length + nonce */
 184                         cdesc->control_data.token[0] = ctx->nonce;
 185                         /* Fixup flags byte */
 186                         *(__le32 *)cbcmaciv =
 187                                 cpu_to_le32(ctx->nonce |
 188                                             ((assocadj > 0) << 6) |
 189                                             ((digestsize - 2) << 2));
 190                         /* 64 bit IV part */
 191                         memcpy(&cdesc->control_data.token[1], iv, 8);
 192                         memcpy(cbcmaciv + 4, iv, 8);
 193                         /* Start counter at 0 */
 194                         cdesc->control_data.token[3] = 0;
 195                         /* Message length */
 196                         *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen);
 197                 } else {
 198                         /* Variable length IV part */
 199                         memcpy(final_iv, iv, 15 - iv[0]);
 200                         memcpy(cbcmaciv, iv, 15 - iv[0]);
 201                         /* Start variable length counter at 0 */
 202                         memset(final_iv + 15 - iv[0], 0, iv[0] + 1);
 203                         memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1);
 204                         /* fixup flags byte */
 205                         cbcmaciv[0] |= ((assocadj > 0) << 6) |
 206                                        ((digestsize - 2) << 2);
 207                         /* insert lower 2 bytes of message length */
 208                         cbcmaciv[14] = cryptlen >> 8;
 209                         cbcmaciv[15] = cryptlen & 255;
 210                 }
 211
 212                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 213                 atoken->packet_length = AES_BLOCK_SIZE +
 214                                         ((assocadj > 0) << 1);
 215                 atoken->stat = 0;
 216                 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN |
 217                                        EIP197_TOKEN_INS_TYPE_HASH;
 218
 219                 if (likely(assocadj)) {
 220                         *aadlen = cpu_to_le32((assocadj >> 8) |
 221                                               (assocadj & 255) << 8);
 222                         atoken += 6;
 223                         atoksize += 7;
 224                 } else {
 225                         atoken += 5;
 226                         atoksize += 6;
 227                 }
 228
 229                 /* Process AAD data */
 230                 aadref = atoken;
 231                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 232                 atoken->packet_length = assocadj;
 233                 atoken->stat = 0;
 234                 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 235                 atoken++;
 236
 237                 /* For CCM only, align AAD data towards hash engine */
 238                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 239                 aadalign = (assocadj + 2) & 15;
 240                 atoken->packet_length = assocadj && aadalign ?
 241                                                 16 - aadalign :
 242                                                 0;
 243                 if (likely(cryptlen)) {
 244                         atoken->stat = 0;
 245                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 246                 } else {
 247                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 248                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 249                                                EIP197_TOKEN_INS_TYPE_HASH;
 250                 }
 251         } else {
 252                 safexcel_aead_iv(ctx, iv, cdesc);
 253
 254                 /* Process AAD data */
 255                 aadref = atoken;
 256                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 257                 atoken->packet_length = assocadj;
 258                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 259                 atoken->instructions = EIP197_TOKEN_INS_LAST |
 260                                        EIP197_TOKEN_INS_TYPE_HASH;
 261         }
 262         atoken++;
 263
 264         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
 265                 /* For ESP mode (and not GMAC), skip over the IV */
 266                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 267                 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE;
 268                 atoken->stat = 0;
 269                 atoken->instructions = 0;
 270                 atoken++;
 271                 atoksize++;
 272         } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 &&
 273                             direction == SAFEXCEL_DECRYPT)) {
 274                 /* Poly-chacha decryption needs a dummy NOP here ... */
 275                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 276                 atoken->packet_length = 16; /* According to Op Manual */
 277                 atoken->stat = 0;
 278                 atoken->instructions = 0;
 279                 atoken++;
 280                 atoksize++;
 281         }
 282
 283         if  (ctx->xcm) {
 284                 /* For GCM and CCM, obtain enc(Y0) */
 285                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES;
 286                 atoken->packet_length = 0;
 287                 atoken->stat = 0;
 288                 atoken->instructions = AES_BLOCK_SIZE;
 289                 atoken++;
 290
 291                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 292                 atoken->packet_length = AES_BLOCK_SIZE;
 293                 atoken->stat = 0;
 294                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 295                                        EIP197_TOKEN_INS_TYPE_CRYPTO;
 296                 atoken++;
 297                 atoksize += 2;
 298         }
 299
 300         if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) {
 301                 /* Fixup stat field for AAD direction instruction */
 302                 aadref->stat = 0;
 303
 304                 /* Process crypto data */
 305                 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 306                 atoken->packet_length = cryptlen;
 307
 308                 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) {
 309                         /* Fixup instruction field for AAD dir instruction */
 310                         aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 311
 312                         /* Do not send to crypt engine in case of GMAC */
 313                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 314                                                EIP197_TOKEN_INS_TYPE_HASH |
 315                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 316                 } else {
 317                         atoken->instructions = EIP197_TOKEN_INS_LAST |
 318                                                EIP197_TOKEN_INS_TYPE_CRYPTO |
 319                                                EIP197_TOKEN_INS_TYPE_HASH |
 320                                                EIP197_TOKEN_INS_TYPE_OUTPUT;
 321                 }
 322
 323                 cryptlen &= 15;
 324                 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) {
 325                         atoken->stat = 0;
 326                         /* For CCM only, pad crypto data to the hash engine */
 327                         atoken++;
 328                         atoksize++;
 329                         atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 330                         atoken->packet_length = 16 - cryptlen;
 331                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 332                         atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH;
 333                 } else {
 334                         atoken->stat = EIP197_TOKEN_STAT_LAST_HASH;
 335                 }
 336                 atoken++;
 337                 atoksize++;
 338         }
 339
 340         if (direction == SAFEXCEL_ENCRYPT) {
 341                 /* Append ICV */
 342                 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT;
 343                 atoken->packet_length = digestsize;
 344                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 345                                EIP197_TOKEN_STAT_LAST_PACKET;
 346                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 347                                        EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 348         } else {
 349                 /* Extract ICV */
 350                 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
 351                 atoken->packet_length = digestsize;
 352                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 353                                EIP197_TOKEN_STAT_LAST_PACKET;
 354                 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 355                 atoken++;
 356                 atoksize++;
 357
 358                 /* Verify ICV */
 359                 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY;
 360                 atoken->packet_length = digestsize |
 361                                         EIP197_TOKEN_HASH_RESULT_VERIFY;
 362                 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH |
 363                                EIP197_TOKEN_STAT_LAST_PACKET;
 364                 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
 365         }
 366
 367         /* Fixup length of the token in the command descriptor */
 368         cdesc->additional_cdata_size = atoksize;
 369 }
 370
 371 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 372                                         const u8 *key, unsigned int len)
 373 {
 374         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 375         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 376         struct safexcel_crypto_priv *priv = ctx->base.priv;
 377         struct crypto_aes_ctx aes;
 378         int ret, i;
 379
 380         ret = aes_expandkey(&aes, key, len);
 381         if (ret)
 382                 return ret;
 383
 384         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 385                 for (i = 0; i < len / sizeof(u32); i++) {
 386                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
 387                                 ctx->base.needs_inv = true;
 388                                 break;
 389                         }
 390                 }
 391         }
 392
 393         for (i = 0; i < len / sizeof(u32); i++)
 394                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
 395
 396         ctx->key_len = len;
 397
 398         memzero_explicit(&aes, sizeof(aes));
 399         return 0;
 400 }
 401
 402 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 403                                 unsigned int len)
 404 {
 405         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 406         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 407         struct safexcel_crypto_priv *priv = ctx->base.priv;
 408         struct crypto_authenc_keys keys;
 409         struct crypto_aes_ctx aes;
 410         int err = -EINVAL, i;
 411         const char *alg;
 412
 413         if (unlikely(crypto_authenc_extractkeys(&keys, key, len)))
 414                 goto badkey;
 415
 416         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) {
 417                 /* Must have at least space for the nonce here */
 418                 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE))
 419                         goto badkey;
 420                 /* last 4 bytes of key are the nonce! */
 421                 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen -
 422                                       CTR_RFC3686_NONCE_SIZE);
 423                 /* exclude the nonce here */
 424                 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
 425         }
 426
 427         /* Encryption key */
 428         switch (ctx->alg) {
 429         case SAFEXCEL_DES:
 430                 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen);
 431                 if (unlikely(err))
 432                         goto badkey;
 433                 break;
 434         case SAFEXCEL_3DES:
 435                 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen);
 436                 if (unlikely(err))
 437                         goto badkey;
 438                 break;
 439         case SAFEXCEL_AES:
 440                 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
 441                 if (unlikely(err))
 442                         goto badkey;
 443                 break;
 444         case SAFEXCEL_SM4:
 445                 if (unlikely(keys.enckeylen != SM4_KEY_SIZE))
 446                         goto badkey;
 447                 break;
 448         default:
 449                 dev_err(priv->dev, "aead: unsupported cipher algorithm\n");
 450                 goto badkey;
 451         }
 452
 453         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 454                 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) {
 455                         if (le32_to_cpu(ctx->key[i]) !=
 456                             ((u32 *)keys.enckey)[i]) {
 457                                 ctx->base.needs_inv = true;
 458                                 break;
 459                         }
 460                 }
 461         }
 462
 463         /* Auth key */
 464         switch (ctx->hash_alg) {
 465         case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 466                 alg = "safexcel-sha1";
 467                 break;
 468         case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
 469                 alg = "safexcel-sha224";
 470                 break;
 471         case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
 472                 alg = "safexcel-sha256";
 473                 break;
 474         case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
 475                 alg = "safexcel-sha384";
 476                 break;
 477         case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
 478                 alg = "safexcel-sha512";
 479                 break;
 480         case CONTEXT_CONTROL_CRYPTO_ALG_SM3:
 481                 alg = "safexcel-sm3";
 482                 break;
 483         default:
 484                 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
 485                 goto badkey;
 486         }
 487
 488         if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen,
 489                                  alg, ctx->state_sz))
 490                 goto badkey;
 491
 492         /* Now copy the keys into the context */
 493         for (i = 0; i < keys.enckeylen / sizeof(u32); i++)
 494                 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]);
 495         ctx->key_len = keys.enckeylen;
 496
 497         memzero_explicit(&keys, sizeof(keys));
 498         return 0;
 499
 500 badkey:
 501         memzero_explicit(&keys, sizeof(keys));
 502         return err;
 503 }
 504
 505 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 506                                     struct crypto_async_request *async,
 507                                     struct safexcel_cipher_req *sreq,
 508                                     struct safexcel_command_desc *cdesc)
 509 {
 510         struct safexcel_crypto_priv *priv = ctx->base.priv;
 511         int ctrl_size = ctx->key_len / sizeof(u32);
 512
 513         cdesc->control_data.control1 = ctx->mode;
 514
 515         if (ctx->aead) {
 516                 /* Take in account the ipad+opad digests */
 517                 if (ctx->xcm) {
 518                         ctrl_size += ctx->state_sz / sizeof(u32);
 519                         cdesc->control_data.control0 =
 520                                 CONTEXT_CONTROL_KEY_EN |
 521                                 CONTEXT_CONTROL_DIGEST_XCM |
 522                                 ctx->hash_alg |
 523                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 524                 } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 525                         /* Chacha20-Poly1305 */
 526                         cdesc->control_data.control0 =
 527                                 CONTEXT_CONTROL_KEY_EN |
 528                                 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 |
 529                                 (sreq->direction == SAFEXCEL_ENCRYPT ?
 530                                         CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT :
 531                                         CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) |
 532                                 ctx->hash_alg |
 533                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 534                         return 0;
 535                 } else {
 536                         ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 537                         cdesc->control_data.control0 =
 538                                 CONTEXT_CONTROL_KEY_EN |
 539                                 CONTEXT_CONTROL_DIGEST_HMAC |
 540                                 ctx->hash_alg |
 541                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 542                 }
 543
 544                 if (sreq->direction == SAFEXCEL_ENCRYPT &&
 545                     (ctx->xcm == EIP197_XCM_MODE_CCM ||
 546                      ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC))
 547                         cdesc->control_data.control0 |=
 548                                 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT;
 549                 else if (sreq->direction == SAFEXCEL_ENCRYPT)
 550                         cdesc->control_data.control0 |=
 551                                 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
 552                 else if (ctx->xcm == EIP197_XCM_MODE_CCM)
 553                         cdesc->control_data.control0 |=
 554                                 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN;
 555                 else
 556                         cdesc->control_data.control0 |=
 557                                 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
 558         } else {
 559                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 560                         cdesc->control_data.control0 =
 561                                 CONTEXT_CONTROL_TYPE_CRYPTO_OUT |
 562                                 CONTEXT_CONTROL_KEY_EN |
 563                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 564                 else
 565                         cdesc->control_data.control0 =
 566                                 CONTEXT_CONTROL_TYPE_CRYPTO_IN |
 567                                 CONTEXT_CONTROL_KEY_EN |
 568                                 CONTEXT_CONTROL_SIZE(ctrl_size);
 569         }
 570
 571         if (ctx->alg == SAFEXCEL_DES) {
 572                 cdesc->control_data.control0 |=
 573                         CONTEXT_CONTROL_CRYPTO_ALG_DES;
 574         } else if (ctx->alg == SAFEXCEL_3DES) {
 575                 cdesc->control_data.control0 |=
 576                         CONTEXT_CONTROL_CRYPTO_ALG_3DES;
 577         } else if (ctx->alg == SAFEXCEL_AES) {
 578                 switch (ctx->key_len >> ctx->xts) {
 579                 case AES_KEYSIZE_128:
 580                         cdesc->control_data.control0 |=
 581                                 CONTEXT_CONTROL_CRYPTO_ALG_AES128;
 582                         break;
 583                 case AES_KEYSIZE_192:
 584                         cdesc->control_data.control0 |=
 585                                 CONTEXT_CONTROL_CRYPTO_ALG_AES192;
 586                         break;
 587                 case AES_KEYSIZE_256:
 588                         cdesc->control_data.control0 |=
 589                                 CONTEXT_CONTROL_CRYPTO_ALG_AES256;
 590                         break;
 591                 default:
 592                         dev_err(priv->dev, "aes keysize not supported: %u\n",
 593                                 ctx->key_len >> ctx->xts);
 594                         return -EINVAL;
 595                 }
 596         } else if (ctx->alg == SAFEXCEL_CHACHA20) {
 597                 cdesc->control_data.control0 |=
 598                         CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20;
 599         } else if (ctx->alg == SAFEXCEL_SM4) {
 600                 cdesc->control_data.control0 |=
 601                         CONTEXT_CONTROL_CRYPTO_ALG_SM4;
 602         }
 603
 604         return 0;
 605 }
 606
 607 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 608                                       struct crypto_async_request *async,
 609                                       struct scatterlist *src,
 610                                       struct scatterlist *dst,
 611                                       unsigned int cryptlen,
 612                                       struct safexcel_cipher_req *sreq,
 613                                       bool *should_complete, int *ret)
 614 {
 615         struct skcipher_request *areq = skcipher_request_cast(async);
 616         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 617         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher);
 618         struct safexcel_result_desc *rdesc;
 619         int ndesc = 0;
 620
 621         *ret = 0;
 622
 623         if (unlikely(!sreq->rdescs))
 624                 return 0;
 625
 626         while (sreq->rdescs--) {
 627                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 628                 if (IS_ERR(rdesc)) {
 629                         dev_err(priv->dev,
 630                                 "cipher: result: could not retrieve the result descriptor\n");
 631                         *ret = PTR_ERR(rdesc);
 632                         break;
 633                 }
 634
 635                 if (likely(!*ret))
 636                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 637
 638                 ndesc++;
 639         }
 640
 641         safexcel_complete(priv, ring);
 642
 643         if (src == dst) {
 644                 if (sreq->nr_src > 0)
 645                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 646                                      DMA_BIDIRECTIONAL);
 647         } else {
 648                 if (sreq->nr_src > 0)
 649                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 650                                      DMA_TO_DEVICE);
 651                 if (sreq->nr_dst > 0)
 652                         dma_unmap_sg(priv->dev, dst, sreq->nr_dst,
 653                                      DMA_FROM_DEVICE);
 654         }
 655
 656         /*
 657          * Update IV in req from last crypto output word for CBC modes
 658          */
 659         if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 660             (sreq->direction == SAFEXCEL_ENCRYPT)) {
 661                 /* For encrypt take the last output word */
 662                 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv,
 663                                    crypto_skcipher_ivsize(skcipher),
 664                                    (cryptlen -
 665                                     crypto_skcipher_ivsize(skcipher)));
 666         }
 667
 668         *should_complete = true;
 669
 670         return ndesc;
 671 }
 672
 673 static int safexcel_send_req(struct crypto_async_request *base, int ring,
 674                              struct safexcel_cipher_req *sreq,
 675                              struct scatterlist *src, struct scatterlist *dst,
 676                              unsigned int cryptlen, unsigned int assoclen,
 677                              unsigned int digestsize, u8 *iv, int *commands,
 678                              int *results)
 679 {
 680         struct skcipher_request *areq = skcipher_request_cast(base);
 681         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
 682         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 683         struct safexcel_crypto_priv *priv = ctx->base.priv;
 684         struct safexcel_command_desc *cdesc;
 685         struct safexcel_command_desc *first_cdesc = NULL;
 686         struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
 687         struct scatterlist *sg;
 688         unsigned int totlen;
 689         unsigned int totlen_src = cryptlen + assoclen;
 690         unsigned int totlen_dst = totlen_src;
 691         struct safexcel_token *atoken;
 692         int n_cdesc = 0, n_rdesc = 0;
 693         int queued, i, ret = 0;
 694         bool first = true;
 695
 696         sreq->nr_src = sg_nents_for_len(src, totlen_src);
 697
 698         if (ctx->aead) {
 699                 /*
 700                  * AEAD has auth tag appended to output for encrypt and
 701                  * removed from the output for decrypt!
 702                  */
 703                 if (sreq->direction == SAFEXCEL_DECRYPT)
 704                         totlen_dst -= digestsize;
 705                 else
 706                         totlen_dst += digestsize;
 707
 708                 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
 709                        &ctx->base.ipad, ctx->state_sz);
 710                 if (!ctx->xcm)
 711                         memcpy(ctx->base.ctxr->data + (ctx->key_len +
 712                                ctx->state_sz) / sizeof(u32), &ctx->base.opad,
 713                                ctx->state_sz);
 714         } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
 715                    (sreq->direction == SAFEXCEL_DECRYPT)) {
 716                 /*
 717                  * Save IV from last crypto input word for CBC modes in decrypt
 718                  * direction. Need to do this first in case of inplace operation
 719                  * as it will be overwritten.
 720                  */
 721                 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv,
 722                                    crypto_skcipher_ivsize(skcipher),
 723                                    (totlen_src -
 724                                     crypto_skcipher_ivsize(skcipher)));
 725         }
 726
 727         sreq->nr_dst = sg_nents_for_len(dst, totlen_dst);
 728
 729         /*
 730          * Remember actual input length, source buffer length may be
 731          * updated in case of inline operation below.
 732          */
 733         totlen = totlen_src;
 734         queued = totlen_src;
 735
 736         if (src == dst) {
 737                 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst);
 738                 sreq->nr_dst = sreq->nr_src;
 739                 if (unlikely((totlen_src || totlen_dst) &&
 740                     (sreq->nr_src <= 0))) {
 741                         dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!",
 742                                 max(totlen_src, totlen_dst));
 743                         return -EINVAL;
 744                 }
 745                 if (sreq->nr_src > 0)
 746                         dma_map_sg(priv->dev, src, sreq->nr_src,
 747                                    DMA_BIDIRECTIONAL);
 748         } else {
 749                 if (unlikely(totlen_src && (sreq->nr_src <= 0))) {
 750                         dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!",
 751                                 totlen_src);
 752                         return -EINVAL;
 753                 }
 754
 755                 if (sreq->nr_src > 0)
 756                         dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE);
 757
 758                 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) {
 759                         dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!",
 760                                 totlen_dst);
 761                         ret = -EINVAL;
 762                         goto unmap;
 763                 }
 764
 765                 if (sreq->nr_dst > 0)
 766                         dma_map_sg(priv->dev, dst, sreq->nr_dst,
 767                                    DMA_FROM_DEVICE);
 768         }
 769
 770         memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 771
 772         if (!totlen) {
 773                 /*
 774                  * The EIP97 cannot deal with zero length input packets!
 775                  * So stuff a dummy command descriptor indicating a 1 byte
 776                  * (dummy) input packet, using the context record as source.
 777                  */
 778                 first_cdesc = safexcel_add_cdesc(priv, ring,
 779                                                  1, 1, ctx->base.ctxr_dma,
 780                                                  1, 1, ctx->base.ctxr_dma,
 781                                                  &atoken);
 782                 if (IS_ERR(first_cdesc)) {
 783                         /* No space left in the command descriptor ring */
 784                         ret = PTR_ERR(first_cdesc);
 785                         goto cdesc_rollback;
 786                 }
 787                 n_cdesc = 1;
 788                 goto skip_cdesc;
 789         }
 790
 791         /* command descriptors */
 792         for_each_sg(src, sg, sreq->nr_src, i) {
 793                 int len = sg_dma_len(sg);
 794
 795                 /* Do not overflow the request */
 796                 if (queued < len)
 797                         len = queued;
 798
 799                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
 800                                            !(queued - len),
 801                                            sg_dma_address(sg), len, totlen,
 802                                            ctx->base.ctxr_dma, &atoken);
 803                 if (IS_ERR(cdesc)) {
 804                         /* No space left in the command descriptor ring */
 805                         ret = PTR_ERR(cdesc);
 806                         goto cdesc_rollback;
 807                 }
 808
 809                 if (!n_cdesc)
 810                         first_cdesc = cdesc;
 811
 812                 n_cdesc++;
 813                 queued -= len;
 814                 if (!queued)
 815                         break;
 816         }
 817 skip_cdesc:
 818         /* Add context control words and token to first command descriptor */
 819         safexcel_context_control(ctx, base, sreq, first_cdesc);
 820         if (ctx->aead)
 821                 safexcel_aead_token(ctx, iv, first_cdesc, atoken,
 822                                     sreq->direction, cryptlen,
 823                                     assoclen, digestsize);
 824         else
 825                 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken,
 826                                         cryptlen);
 827
 828         /* result descriptors */
 829         for_each_sg(dst, sg, sreq->nr_dst, i) {
 830                 bool last = (i == sreq->nr_dst - 1);
 831                 u32 len = sg_dma_len(sg);
 832
 833                 /* only allow the part of the buffer we know we need */
 834                 if (len > totlen_dst)
 835                         len = totlen_dst;
 836                 if (unlikely(!len))
 837                         break;
 838                 totlen_dst -= len;
 839
 840                 /* skip over AAD space in buffer - not written */
 841                 if (assoclen) {
 842                         if (assoclen >= len) {
 843                                 assoclen -= len;
 844                                 continue;
 845                         }
 846                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 847                                                    sg_dma_address(sg) +
 848                                                    assoclen,
 849                                                    len - assoclen);
 850                         assoclen = 0;
 851                 } else {
 852                         rdesc = safexcel_add_rdesc(priv, ring, first, last,
 853                                                    sg_dma_address(sg),
 854                                                    len);
 855                 }
 856                 if (IS_ERR(rdesc)) {
 857                         /* No space left in the result descriptor ring */
 858                         ret = PTR_ERR(rdesc);
 859                         goto rdesc_rollback;
 860                 }
 861                 if (first) {
 862                         first_rdesc = rdesc;
 863                         first = false;
 864                 }
 865                 n_rdesc++;
 866         }
 867
 868         if (unlikely(first)) {
 869                 /*
 870                  * Special case: AEAD decrypt with only AAD data.
 871                  * In this case there is NO output data from the engine,
 872                  * but the engine still needs a result descriptor!
 873                  * Create a dummy one just for catching the result token.
 874                  */
 875                 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
 876                 if (IS_ERR(rdesc)) {
 877                         /* No space left in the result descriptor ring */
 878                         ret = PTR_ERR(rdesc);
 879                         goto rdesc_rollback;
 880                 }
 881                 first_rdesc = rdesc;
 882                 n_rdesc = 1;
 883         }
 884
 885         safexcel_rdr_req_set(priv, ring, first_rdesc, base);
 886
 887         *commands = n_cdesc;
 888         *results = n_rdesc;
 889         return 0;
 890
 891 rdesc_rollback:
 892         for (i = 0; i < n_rdesc; i++)
 893                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
 894 cdesc_rollback:
 895         for (i = 0; i < n_cdesc; i++)
 896                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 897 unmap:
 898         if (src == dst) {
 899                 if (sreq->nr_src > 0)
 900                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 901                                      DMA_BIDIRECTIONAL);
 902         } else {
 903                 if (sreq->nr_src > 0)
 904                         dma_unmap_sg(priv->dev, src, sreq->nr_src,
 905                                      DMA_TO_DEVICE);
 906                 if (sreq->nr_dst > 0)
 907                         dma_unmap_sg(priv->dev, dst, sreq->nr_dst,
 908                                      DMA_FROM_DEVICE);
 909         }
 910
 911         return ret;
 912 }
 913
 914 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 915                                       int ring,
 916                                       struct crypto_async_request *base,
 917                                       struct safexcel_cipher_req *sreq,
 918                                       bool *should_complete, int *ret)
 919 {
 920         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 921         struct safexcel_result_desc *rdesc;
 922         int ndesc = 0, enq_ret;
 923
 924         *ret = 0;
 925
 926         if (unlikely(!sreq->rdescs))
 927                 return 0;
 928
 929         while (sreq->rdescs--) {
 930                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 931                 if (IS_ERR(rdesc)) {
 932                         dev_err(priv->dev,
 933                                 "cipher: invalidate: could not retrieve the result descriptor\n");
 934                         *ret = PTR_ERR(rdesc);
 935                         break;
 936                 }
 937
 938                 if (likely(!*ret))
 939                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 940
 941                 ndesc++;
 942         }
 943
 944         safexcel_complete(priv, ring);
 945
 946         if (ctx->base.exit_inv) {
 947                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 948                               ctx->base.ctxr_dma);
 949
 950                 *should_complete = true;
 951
 952                 return ndesc;
 953         }
 954
 955         ring = safexcel_select_ring(priv);
 956         ctx->base.ring = ring;
 957
 958         spin_lock_bh(&priv->ring[ring].queue_lock);
 959         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 960         spin_unlock_bh(&priv->ring[ring].queue_lock);
 961
 962         if (enq_ret != -EINPROGRESS)
 963                 *ret = enq_ret;
 964
 965         queue_work(priv->ring[ring].workqueue,
 966                    &priv->ring[ring].work_data.work);
 967
 968         *should_complete = false;
 969
 970         return ndesc;
 971 }
 972
 973 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 974                                            int ring,
 975                                            struct crypto_async_request *async,
 976                                            bool *should_complete, int *ret)
 977 {
 978         struct skcipher_request *req = skcipher_request_cast(async);
 979         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 980         int err;
 981
 982         if (sreq->needs_inv) {
 983                 sreq->needs_inv = false;
 984                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 985                                                  should_complete, ret);
 986         } else {
 987                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 988                                                  req->dst, req->cryptlen, sreq,
 989                                                  should_complete, ret);
 990         }
 991
 992         return err;
 993 }
 994
 995 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 996                                        int ring,
 997                                        struct crypto_async_request *async,
 998                                        bool *should_complete, int *ret)
 999 {
1000         struct aead_request *req = aead_request_cast(async);
1001         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1002         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1003         int err;
1004
1005         if (sreq->needs_inv) {
1006                 sreq->needs_inv = false;
1007                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
1008                                                  should_complete, ret);
1009         } else {
1010                 err = safexcel_handle_req_result(priv, ring, async, req->src,
1011                                                  req->dst,
1012                                                  req->cryptlen + crypto_aead_authsize(tfm),
1013                                                  sreq, should_complete, ret);
1014         }
1015
1016         return err;
1017 }
1018
1019 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
1020                                     int ring, int *commands, int *results)
1021 {
1022         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1023         struct safexcel_crypto_priv *priv = ctx->base.priv;
1024         int ret;
1025
1026         ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
1027         if (unlikely(ret))
1028                 return ret;
1029
1030         *commands = 1;
1031         *results = 1;
1032
1033         return 0;
1034 }
1035
1036 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
1037                                   int *commands, int *results)
1038 {
1039         struct skcipher_request *req = skcipher_request_cast(async);
1040         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1041         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1042         struct safexcel_crypto_priv *priv = ctx->base.priv;
1043         int ret;
1044
1045         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1046
1047         if (sreq->needs_inv) {
1048                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1049         } else {
1050                 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1051                 u8 input_iv[AES_BLOCK_SIZE];
1052
1053                 /*
1054                  * Save input IV in case of CBC decrypt mode
1055                  * Will be overwritten with output IV prior to use!
1056                  */
1057                 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher));
1058
1059                 ret = safexcel_send_req(async, ring, sreq, req->src,
1060                                         req->dst, req->cryptlen, 0, 0, input_iv,
1061                                         commands, results);
1062         }
1063
1064         sreq->rdescs = *results;
1065         return ret;
1066 }
1067
1068 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
1069                               int *commands, int *results)
1070 {
1071         struct aead_request *req = aead_request_cast(async);
1072         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1073         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1074         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1075         struct safexcel_crypto_priv *priv = ctx->base.priv;
1076         int ret;
1077
1078         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
1079
1080         if (sreq->needs_inv)
1081                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
1082         else
1083                 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
1084                                         req->cryptlen, req->assoclen,
1085                                         crypto_aead_authsize(tfm), req->iv,
1086                                         commands, results);
1087         sreq->rdescs = *results;
1088         return ret;
1089 }
1090
1091 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
1092                                     struct crypto_async_request *base,
1093                                     struct safexcel_cipher_req *sreq,
1094                                     struct safexcel_inv_result *result)
1095 {
1096         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1097         struct safexcel_crypto_priv *priv = ctx->base.priv;
1098         int ring = ctx->base.ring;
1099
1100         init_completion(&result->completion);
1101
1102         ctx = crypto_tfm_ctx(base->tfm);
1103         ctx->base.exit_inv = true;
1104         sreq->needs_inv = true;
1105
1106         spin_lock_bh(&priv->ring[ring].queue_lock);
1107         crypto_enqueue_request(&priv->ring[ring].queue, base);
1108         spin_unlock_bh(&priv->ring[ring].queue_lock);
1109
1110         queue_work(priv->ring[ring].workqueue,
1111                    &priv->ring[ring].work_data.work);
1112
1113         wait_for_completion(&result->completion);
1114
1115         if (result->error) {
1116                 dev_warn(priv->dev,
1117                         "cipher: sync: invalidate: completion error %d\n",
1118                          result->error);
1119                 return result->error;
1120         }
1121
1122         return 0;
1123 }
1124
1125 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
1126 {
1127         EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
1128         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
1129         struct safexcel_inv_result result = {};
1130
1131         memset(req, 0, sizeof(struct skcipher_request));
1132
1133         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1134                                       safexcel_inv_complete, &result);
1135         skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
1136
1137         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1138 }
1139
1140 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
1141 {
1142         EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
1143         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
1144         struct safexcel_inv_result result = {};
1145
1146         memset(req, 0, sizeof(struct aead_request));
1147
1148         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1149                                   safexcel_inv_complete, &result);
1150         aead_request_set_tfm(req, __crypto_aead_cast(tfm));
1151
1152         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
1153 }
1154
1155 static int safexcel_queue_req(struct crypto_async_request *base,
1156                         struct safexcel_cipher_req *sreq,
1157                         enum safexcel_cipher_direction dir)
1158 {
1159         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
1160         struct safexcel_crypto_priv *priv = ctx->base.priv;
1161         int ret, ring;
1162
1163         sreq->needs_inv = false;
1164         sreq->direction = dir;
1165
1166         if (ctx->base.ctxr) {
1167                 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
1168                         sreq->needs_inv = true;
1169                         ctx->base.needs_inv = false;
1170                 }
1171         } else {
1172                 ctx->base.ring = safexcel_select_ring(priv);
1173                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
1174                                                  EIP197_GFP_FLAGS(*base),
1175                                                  &ctx->base.ctxr_dma);
1176                 if (!ctx->base.ctxr)
1177                         return -ENOMEM;
1178         }
1179
1180         ring = ctx->base.ring;
1181
1182         spin_lock_bh(&priv->ring[ring].queue_lock);
1183         ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
1184         spin_unlock_bh(&priv->ring[ring].queue_lock);
1185
1186         queue_work(priv->ring[ring].workqueue,
1187                    &priv->ring[ring].work_data.work);
1188
1189         return ret;
1190 }
1191
1192 static int safexcel_encrypt(struct skcipher_request *req)
1193 {
1194         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1195                         SAFEXCEL_ENCRYPT);
1196 }
1197
1198 static int safexcel_decrypt(struct skcipher_request *req)
1199 {
1200         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1201                         SAFEXCEL_DECRYPT);
1202 }
1203
1204 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
1205 {
1206         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1207         struct safexcel_alg_template *tmpl =
1208                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1209                              alg.skcipher.base);
1210
1211         crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
1212                                     sizeof(struct safexcel_cipher_req));
1213
1214         ctx->base.priv = tmpl->priv;
1215
1216         ctx->base.send = safexcel_skcipher_send;
1217         ctx->base.handle_result = safexcel_skcipher_handle_result;
1218         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1219         ctx->ctrinit = 1;
1220         return 0;
1221 }
1222
1223 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
1224 {
1225         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1226
1227         memzero_explicit(ctx->key, sizeof(ctx->key));
1228
1229         /* context not allocated, skip invalidation */
1230         if (!ctx->base.ctxr)
1231                 return -ENOMEM;
1232
1233         memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
1234         return 0;
1235 }
1236
1237 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
1238 {
1239         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1240         struct safexcel_crypto_priv *priv = ctx->base.priv;
1241         int ret;
1242
1243         if (safexcel_cipher_cra_exit(tfm))
1244                 return;
1245
1246         if (priv->flags & EIP197_TRC_CACHE) {
1247                 ret = safexcel_skcipher_exit_inv(tfm);
1248                 if (ret)
1249                         dev_warn(priv->dev, "skcipher: invalidation error %d\n",
1250                                  ret);
1251         } else {
1252                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1253                               ctx->base.ctxr_dma);
1254         }
1255 }
1256
1257 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
1258 {
1259         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1260         struct safexcel_crypto_priv *priv = ctx->base.priv;
1261         int ret;
1262
1263         if (safexcel_cipher_cra_exit(tfm))
1264                 return;
1265
1266         if (priv->flags & EIP197_TRC_CACHE) {
1267                 ret = safexcel_aead_exit_inv(tfm);
1268                 if (ret)
1269                         dev_warn(priv->dev, "aead: invalidation error %d\n",
1270                                  ret);
1271         } else {
1272                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
1273                               ctx->base.ctxr_dma);
1274         }
1275 }
1276
1277 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm)
1278 {
1279         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1280
1281         safexcel_skcipher_cra_init(tfm);
1282         ctx->alg  = SAFEXCEL_AES;
1283         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1284         ctx->blocksz = 0;
1285         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1286         return 0;
1287 }
1288
1289 struct safexcel_alg_template safexcel_alg_ecb_aes = {
1290         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1291         .algo_mask = SAFEXCEL_ALG_AES,
1292         .alg.skcipher = {
1293                 .setkey = safexcel_skcipher_aes_setkey,
1294                 .encrypt = safexcel_encrypt,
1295                 .decrypt = safexcel_decrypt,
1296                 .min_keysize = AES_MIN_KEY_SIZE,
1297                 .max_keysize = AES_MAX_KEY_SIZE,
1298                 .base = {
1299                         .cra_name = "ecb(aes)",
1300                         .cra_driver_name = "safexcel-ecb-aes",
1301                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1302                         .cra_flags = CRYPTO_ALG_ASYNC |
1303                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1304                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1305                         .cra_blocksize = AES_BLOCK_SIZE,
1306                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1307                         .cra_alignmask = 0,
1308                         .cra_init = safexcel_skcipher_aes_ecb_cra_init,
1309                         .cra_exit = safexcel_skcipher_cra_exit,
1310                         .cra_module = THIS_MODULE,
1311                 },
1312         },
1313 };
1314
1315 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm)
1316 {
1317         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1318
1319         safexcel_skcipher_cra_init(tfm);
1320         ctx->alg  = SAFEXCEL_AES;
1321         ctx->blocksz = AES_BLOCK_SIZE;
1322         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1323         return 0;
1324 }
1325
1326 struct safexcel_alg_template safexcel_alg_cbc_aes = {
1327         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1328         .algo_mask = SAFEXCEL_ALG_AES,
1329         .alg.skcipher = {
1330                 .setkey = safexcel_skcipher_aes_setkey,
1331                 .encrypt = safexcel_encrypt,
1332                 .decrypt = safexcel_decrypt,
1333                 .min_keysize = AES_MIN_KEY_SIZE,
1334                 .max_keysize = AES_MAX_KEY_SIZE,
1335                 .ivsize = AES_BLOCK_SIZE,
1336                 .base = {
1337                         .cra_name = "cbc(aes)",
1338                         .cra_driver_name = "safexcel-cbc-aes",
1339                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1340                         .cra_flags = CRYPTO_ALG_ASYNC |
1341                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1342                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1343                         .cra_blocksize = AES_BLOCK_SIZE,
1344                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1345                         .cra_alignmask = 0,
1346                         .cra_init = safexcel_skcipher_aes_cbc_cra_init,
1347                         .cra_exit = safexcel_skcipher_cra_exit,
1348                         .cra_module = THIS_MODULE,
1349                 },
1350         },
1351 };
1352
1353 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm)
1354 {
1355         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1356
1357         safexcel_skcipher_cra_init(tfm);
1358         ctx->alg  = SAFEXCEL_AES;
1359         ctx->blocksz = AES_BLOCK_SIZE;
1360         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
1361         return 0;
1362 }
1363
1364 struct safexcel_alg_template safexcel_alg_cfb_aes = {
1365         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1366         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1367         .alg.skcipher = {
1368                 .setkey = safexcel_skcipher_aes_setkey,
1369                 .encrypt = safexcel_encrypt,
1370                 .decrypt = safexcel_decrypt,
1371                 .min_keysize = AES_MIN_KEY_SIZE,
1372                 .max_keysize = AES_MAX_KEY_SIZE,
1373                 .ivsize = AES_BLOCK_SIZE,
1374                 .base = {
1375                         .cra_name = "cfb(aes)",
1376                         .cra_driver_name = "safexcel-cfb-aes",
1377                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1378                         .cra_flags = CRYPTO_ALG_ASYNC |
1379                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1380                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1381                         .cra_blocksize = 1,
1382                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1383                         .cra_alignmask = 0,
1384                         .cra_init = safexcel_skcipher_aes_cfb_cra_init,
1385                         .cra_exit = safexcel_skcipher_cra_exit,
1386                         .cra_module = THIS_MODULE,
1387                 },
1388         },
1389 };
1390
1391 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm)
1392 {
1393         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1394
1395         safexcel_skcipher_cra_init(tfm);
1396         ctx->alg  = SAFEXCEL_AES;
1397         ctx->blocksz = AES_BLOCK_SIZE;
1398         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
1399         return 0;
1400 }
1401
1402 struct safexcel_alg_template safexcel_alg_ofb_aes = {
1403         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1404         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB,
1405         .alg.skcipher = {
1406                 .setkey = safexcel_skcipher_aes_setkey,
1407                 .encrypt = safexcel_encrypt,
1408                 .decrypt = safexcel_decrypt,
1409                 .min_keysize = AES_MIN_KEY_SIZE,
1410                 .max_keysize = AES_MAX_KEY_SIZE,
1411                 .ivsize = AES_BLOCK_SIZE,
1412                 .base = {
1413                         .cra_name = "ofb(aes)",
1414                         .cra_driver_name = "safexcel-ofb-aes",
1415                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1416                         .cra_flags = CRYPTO_ALG_ASYNC |
1417                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1418                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1419                         .cra_blocksize = 1,
1420                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1421                         .cra_alignmask = 0,
1422                         .cra_init = safexcel_skcipher_aes_ofb_cra_init,
1423                         .cra_exit = safexcel_skcipher_cra_exit,
1424                         .cra_module = THIS_MODULE,
1425                 },
1426         },
1427 };
1428
1429 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
1430                                            const u8 *key, unsigned int len)
1431 {
1432         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1433         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1434         struct safexcel_crypto_priv *priv = ctx->base.priv;
1435         struct crypto_aes_ctx aes;
1436         int ret, i;
1437         unsigned int keylen;
1438
1439         /* last 4 bytes of key are the nonce! */
1440         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
1441         /* exclude the nonce here */
1442         keylen = len - CTR_RFC3686_NONCE_SIZE;
1443         ret = aes_expandkey(&aes, key, keylen);
1444         if (ret)
1445                 return ret;
1446
1447         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
1448                 for (i = 0; i < keylen / sizeof(u32); i++) {
1449                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
1450                                 ctx->base.needs_inv = true;
1451                                 break;
1452                         }
1453                 }
1454         }
1455
1456         for (i = 0; i < keylen / sizeof(u32); i++)
1457                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
1458
1459         ctx->key_len = keylen;
1460
1461         memzero_explicit(&aes, sizeof(aes));
1462         return 0;
1463 }
1464
1465 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm)
1466 {
1467         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1468
1469         safexcel_skcipher_cra_init(tfm);
1470         ctx->alg  = SAFEXCEL_AES;
1471         ctx->blocksz = AES_BLOCK_SIZE;
1472         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
1473         return 0;
1474 }
1475
1476 struct safexcel_alg_template safexcel_alg_ctr_aes = {
1477         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1478         .algo_mask = SAFEXCEL_ALG_AES,
1479         .alg.skcipher = {
1480                 .setkey = safexcel_skcipher_aesctr_setkey,
1481                 .encrypt = safexcel_encrypt,
1482                 .decrypt = safexcel_decrypt,
1483                 /* Add nonce size */
1484                 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1485                 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
1486                 .ivsize = CTR_RFC3686_IV_SIZE,
1487                 .base = {
1488                         .cra_name = "rfc3686(ctr(aes))",
1489                         .cra_driver_name = "safexcel-ctr-aes",
1490                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1491                         .cra_flags = CRYPTO_ALG_ASYNC |
1492                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1493                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1494                         .cra_blocksize = 1,
1495                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1496                         .cra_alignmask = 0,
1497                         .cra_init = safexcel_skcipher_aes_ctr_cra_init,
1498                         .cra_exit = safexcel_skcipher_cra_exit,
1499                         .cra_module = THIS_MODULE,
1500                 },
1501         },
1502 };
1503
1504 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
1505                                unsigned int len)
1506 {
1507         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1508         struct safexcel_crypto_priv *priv = ctx->base.priv;
1509         int ret;
1510
1511         ret = verify_skcipher_des_key(ctfm, key);
1512         if (ret)
1513                 return ret;
1514
1515         /* if context exits and key changed, need to invalidate it */
1516         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1517                 if (memcmp(ctx->key, key, len))
1518                         ctx->base.needs_inv = true;
1519
1520         memcpy(ctx->key, key, len);
1521         ctx->key_len = len;
1522
1523         return 0;
1524 }
1525
1526 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm)
1527 {
1528         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1529
1530         safexcel_skcipher_cra_init(tfm);
1531         ctx->alg  = SAFEXCEL_DES;
1532         ctx->blocksz = DES_BLOCK_SIZE;
1533         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1534         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1535         return 0;
1536 }
1537
1538 struct safexcel_alg_template safexcel_alg_cbc_des = {
1539         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1540         .algo_mask = SAFEXCEL_ALG_DES,
1541         .alg.skcipher = {
1542                 .setkey = safexcel_des_setkey,
1543                 .encrypt = safexcel_encrypt,
1544                 .decrypt = safexcel_decrypt,
1545                 .min_keysize = DES_KEY_SIZE,
1546                 .max_keysize = DES_KEY_SIZE,
1547                 .ivsize = DES_BLOCK_SIZE,
1548                 .base = {
1549                         .cra_name = "cbc(des)",
1550                         .cra_driver_name = "safexcel-cbc-des",
1551                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1552                         .cra_flags = CRYPTO_ALG_ASYNC |
1553                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1554                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1555                         .cra_blocksize = DES_BLOCK_SIZE,
1556                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1557                         .cra_alignmask = 0,
1558                         .cra_init = safexcel_skcipher_des_cbc_cra_init,
1559                         .cra_exit = safexcel_skcipher_cra_exit,
1560                         .cra_module = THIS_MODULE,
1561                 },
1562         },
1563 };
1564
1565 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm)
1566 {
1567         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1568
1569         safexcel_skcipher_cra_init(tfm);
1570         ctx->alg  = SAFEXCEL_DES;
1571         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1572         ctx->blocksz = 0;
1573         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1574         return 0;
1575 }
1576
1577 struct safexcel_alg_template safexcel_alg_ecb_des = {
1578         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1579         .algo_mask = SAFEXCEL_ALG_DES,
1580         .alg.skcipher = {
1581                 .setkey = safexcel_des_setkey,
1582                 .encrypt = safexcel_encrypt,
1583                 .decrypt = safexcel_decrypt,
1584                 .min_keysize = DES_KEY_SIZE,
1585                 .max_keysize = DES_KEY_SIZE,
1586                 .base = {
1587                         .cra_name = "ecb(des)",
1588                         .cra_driver_name = "safexcel-ecb-des",
1589                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1590                         .cra_flags = CRYPTO_ALG_ASYNC |
1591                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1592                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1593                         .cra_blocksize = DES_BLOCK_SIZE,
1594                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1595                         .cra_alignmask = 0,
1596                         .cra_init = safexcel_skcipher_des_ecb_cra_init,
1597                         .cra_exit = safexcel_skcipher_cra_exit,
1598                         .cra_module = THIS_MODULE,
1599                 },
1600         },
1601 };
1602
1603 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1604                                    const u8 *key, unsigned int len)
1605 {
1606         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
1607         struct safexcel_crypto_priv *priv = ctx->base.priv;
1608         int err;
1609
1610         err = verify_skcipher_des3_key(ctfm, key);
1611         if (err)
1612                 return err;
1613
1614         /* if context exits and key changed, need to invalidate it */
1615         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
1616                 if (memcmp(ctx->key, key, len))
1617                         ctx->base.needs_inv = true;
1618
1619         memcpy(ctx->key, key, len);
1620         ctx->key_len = len;
1621
1622         return 0;
1623 }
1624
1625 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm)
1626 {
1627         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1628
1629         safexcel_skcipher_cra_init(tfm);
1630         ctx->alg  = SAFEXCEL_3DES;
1631         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1632         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1633         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
1634         return 0;
1635 }
1636
1637 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1638         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1639         .algo_mask = SAFEXCEL_ALG_DES,
1640         .alg.skcipher = {
1641                 .setkey = safexcel_des3_ede_setkey,
1642                 .encrypt = safexcel_encrypt,
1643                 .decrypt = safexcel_decrypt,
1644                 .min_keysize = DES3_EDE_KEY_SIZE,
1645                 .max_keysize = DES3_EDE_KEY_SIZE,
1646                 .ivsize = DES3_EDE_BLOCK_SIZE,
1647                 .base = {
1648                         .cra_name = "cbc(des3_ede)",
1649                         .cra_driver_name = "safexcel-cbc-des3_ede",
1650                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1651                         .cra_flags = CRYPTO_ALG_ASYNC |
1652                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1653                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1654                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1655                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1656                         .cra_alignmask = 0,
1657                         .cra_init = safexcel_skcipher_des3_cbc_cra_init,
1658                         .cra_exit = safexcel_skcipher_cra_exit,
1659                         .cra_module = THIS_MODULE,
1660                 },
1661         },
1662 };
1663
1664 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm)
1665 {
1666         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1667
1668         safexcel_skcipher_cra_init(tfm);
1669         ctx->alg  = SAFEXCEL_3DES;
1670         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
1671         ctx->blocksz = 0;
1672         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1673         return 0;
1674 }
1675
1676 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1677         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1678         .algo_mask = SAFEXCEL_ALG_DES,
1679         .alg.skcipher = {
1680                 .setkey = safexcel_des3_ede_setkey,
1681                 .encrypt = safexcel_encrypt,
1682                 .decrypt = safexcel_decrypt,
1683                 .min_keysize = DES3_EDE_KEY_SIZE,
1684                 .max_keysize = DES3_EDE_KEY_SIZE,
1685                 .base = {
1686                         .cra_name = "ecb(des3_ede)",
1687                         .cra_driver_name = "safexcel-ecb-des3_ede",
1688                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1689                         .cra_flags = CRYPTO_ALG_ASYNC |
1690                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1691                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1692                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1693                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1694                         .cra_alignmask = 0,
1695                         .cra_init = safexcel_skcipher_des3_ecb_cra_init,
1696                         .cra_exit = safexcel_skcipher_cra_exit,
1697                         .cra_module = THIS_MODULE,
1698                 },
1699         },
1700 };
1701
1702 static int safexcel_aead_encrypt(struct aead_request *req)
1703 {
1704         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1705
1706         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
1707 }
1708
1709 static int safexcel_aead_decrypt(struct aead_request *req)
1710 {
1711         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1712
1713         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
1714 }
1715
1716 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1717 {
1718         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1719         struct safexcel_alg_template *tmpl =
1720                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1721                              alg.aead.base);
1722
1723         crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1724                                 sizeof(struct safexcel_cipher_req));
1725
1726         ctx->base.priv = tmpl->priv;
1727
1728         ctx->alg  = SAFEXCEL_AES; /* default */
1729         ctx->blocksz = AES_BLOCK_SIZE;
1730         ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD;
1731         ctx->ctrinit = 1;
1732         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */
1733         ctx->aead = true;
1734         ctx->base.send = safexcel_aead_send;
1735         ctx->base.handle_result = safexcel_aead_handle_result;
1736         return 0;
1737 }
1738
1739 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1740 {
1741         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1742
1743         safexcel_aead_cra_init(tfm);
1744         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1745         ctx->state_sz = SHA1_DIGEST_SIZE;
1746         return 0;
1747 }
1748
1749 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1750         .type = SAFEXCEL_ALG_TYPE_AEAD,
1751         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
1752         .alg.aead = {
1753                 .setkey = safexcel_aead_setkey,
1754                 .encrypt = safexcel_aead_encrypt,
1755                 .decrypt = safexcel_aead_decrypt,
1756                 .ivsize = AES_BLOCK_SIZE,
1757                 .maxauthsize = SHA1_DIGEST_SIZE,
1758                 .base = {
1759                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1760                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1761                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1762                         .cra_flags = CRYPTO_ALG_ASYNC |
1763                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1764                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1765                         .cra_blocksize = AES_BLOCK_SIZE,
1766                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1767                         .cra_alignmask = 0,
1768                         .cra_init = safexcel_aead_sha1_cra_init,
1769                         .cra_exit = safexcel_aead_cra_exit,
1770                         .cra_module = THIS_MODULE,
1771                 },
1772         },
1773 };
1774
1775 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1776 {
1777         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1778
1779         safexcel_aead_cra_init(tfm);
1780         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1781         ctx->state_sz = SHA256_DIGEST_SIZE;
1782         return 0;
1783 }
1784
1785 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1786         .type = SAFEXCEL_ALG_TYPE_AEAD,
1787         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1788         .alg.aead = {
1789                 .setkey = safexcel_aead_setkey,
1790                 .encrypt = safexcel_aead_encrypt,
1791                 .decrypt = safexcel_aead_decrypt,
1792                 .ivsize = AES_BLOCK_SIZE,
1793                 .maxauthsize = SHA256_DIGEST_SIZE,
1794                 .base = {
1795                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1796                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1797                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1798                         .cra_flags = CRYPTO_ALG_ASYNC |
1799                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1800                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1801                         .cra_blocksize = AES_BLOCK_SIZE,
1802                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1803                         .cra_alignmask = 0,
1804                         .cra_init = safexcel_aead_sha256_cra_init,
1805                         .cra_exit = safexcel_aead_cra_exit,
1806                         .cra_module = THIS_MODULE,
1807                 },
1808         },
1809 };
1810
1811 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1812 {
1813         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1814
1815         safexcel_aead_cra_init(tfm);
1816         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1817         ctx->state_sz = SHA256_DIGEST_SIZE;
1818         return 0;
1819 }
1820
1821 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1822         .type = SAFEXCEL_ALG_TYPE_AEAD,
1823         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
1824         .alg.aead = {
1825                 .setkey = safexcel_aead_setkey,
1826                 .encrypt = safexcel_aead_encrypt,
1827                 .decrypt = safexcel_aead_decrypt,
1828                 .ivsize = AES_BLOCK_SIZE,
1829                 .maxauthsize = SHA224_DIGEST_SIZE,
1830                 .base = {
1831                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
1832                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1833                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1834                         .cra_flags = CRYPTO_ALG_ASYNC |
1835                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1836                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1837                         .cra_blocksize = AES_BLOCK_SIZE,
1838                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1839                         .cra_alignmask = 0,
1840                         .cra_init = safexcel_aead_sha224_cra_init,
1841                         .cra_exit = safexcel_aead_cra_exit,
1842                         .cra_module = THIS_MODULE,
1843                 },
1844         },
1845 };
1846
1847 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1848 {
1849         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1850
1851         safexcel_aead_cra_init(tfm);
1852         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1853         ctx->state_sz = SHA512_DIGEST_SIZE;
1854         return 0;
1855 }
1856
1857 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1858         .type = SAFEXCEL_ALG_TYPE_AEAD,
1859         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1860         .alg.aead = {
1861                 .setkey = safexcel_aead_setkey,
1862                 .encrypt = safexcel_aead_encrypt,
1863                 .decrypt = safexcel_aead_decrypt,
1864                 .ivsize = AES_BLOCK_SIZE,
1865                 .maxauthsize = SHA512_DIGEST_SIZE,
1866                 .base = {
1867                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1868                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1869                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1870                         .cra_flags = CRYPTO_ALG_ASYNC |
1871                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1872                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1873                         .cra_blocksize = AES_BLOCK_SIZE,
1874                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1875                         .cra_alignmask = 0,
1876                         .cra_init = safexcel_aead_sha512_cra_init,
1877                         .cra_exit = safexcel_aead_cra_exit,
1878                         .cra_module = THIS_MODULE,
1879                 },
1880         },
1881 };
1882
1883 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1884 {
1885         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1886
1887         safexcel_aead_cra_init(tfm);
1888         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1889         ctx->state_sz = SHA512_DIGEST_SIZE;
1890         return 0;
1891 }
1892
1893 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1894         .type = SAFEXCEL_ALG_TYPE_AEAD,
1895         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
1896         .alg.aead = {
1897                 .setkey = safexcel_aead_setkey,
1898                 .encrypt = safexcel_aead_encrypt,
1899                 .decrypt = safexcel_aead_decrypt,
1900                 .ivsize = AES_BLOCK_SIZE,
1901                 .maxauthsize = SHA384_DIGEST_SIZE,
1902                 .base = {
1903                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
1904                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1905                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1906                         .cra_flags = CRYPTO_ALG_ASYNC |
1907                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1908                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1909                         .cra_blocksize = AES_BLOCK_SIZE,
1910                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1911                         .cra_alignmask = 0,
1912                         .cra_init = safexcel_aead_sha384_cra_init,
1913                         .cra_exit = safexcel_aead_cra_exit,
1914                         .cra_module = THIS_MODULE,
1915                 },
1916         },
1917 };
1918
1919 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm)
1920 {
1921         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1922
1923         safexcel_aead_sha1_cra_init(tfm);
1924         ctx->alg = SAFEXCEL_3DES; /* override default */
1925         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1926         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1927         return 0;
1928 }
1929
1930 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = {
1931         .type = SAFEXCEL_ALG_TYPE_AEAD,
1932         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
1933         .alg.aead = {
1934                 .setkey = safexcel_aead_setkey,
1935                 .encrypt = safexcel_aead_encrypt,
1936                 .decrypt = safexcel_aead_decrypt,
1937                 .ivsize = DES3_EDE_BLOCK_SIZE,
1938                 .maxauthsize = SHA1_DIGEST_SIZE,
1939                 .base = {
1940                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1941                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede",
1942                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1943                         .cra_flags = CRYPTO_ALG_ASYNC |
1944                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1945                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1946                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1947                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1948                         .cra_alignmask = 0,
1949                         .cra_init = safexcel_aead_sha1_des3_cra_init,
1950                         .cra_exit = safexcel_aead_cra_exit,
1951                         .cra_module = THIS_MODULE,
1952                 },
1953         },
1954 };
1955
1956 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm)
1957 {
1958         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1959
1960         safexcel_aead_sha256_cra_init(tfm);
1961         ctx->alg = SAFEXCEL_3DES; /* override default */
1962         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
1963         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
1964         return 0;
1965 }
1966
1967 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = {
1968         .type = SAFEXCEL_ALG_TYPE_AEAD,
1969         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
1970         .alg.aead = {
1971                 .setkey = safexcel_aead_setkey,
1972                 .encrypt = safexcel_aead_encrypt,
1973                 .decrypt = safexcel_aead_decrypt,
1974                 .ivsize = DES3_EDE_BLOCK_SIZE,
1975                 .maxauthsize = SHA256_DIGEST_SIZE,
1976                 .base = {
1977                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1978                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede",
1979                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
1980                         .cra_flags = CRYPTO_ALG_ASYNC |
1981                                      CRYPTO_ALG_ALLOCATES_MEMORY |
1982                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1983                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1984                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1985                         .cra_alignmask = 0,
1986                         .cra_init = safexcel_aead_sha256_des3_cra_init,
1987                         .cra_exit = safexcel_aead_cra_exit,
1988                         .cra_module = THIS_MODULE,
1989                 },
1990         },
1991 };
1992
1993 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm)
1994 {
1995         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1996
1997         safexcel_aead_sha224_cra_init(tfm);
1998         ctx->alg = SAFEXCEL_3DES; /* override default */
1999         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2000         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2001         return 0;
2002 }
2003
2004 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = {
2005         .type = SAFEXCEL_ALG_TYPE_AEAD,
2006         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2007         .alg.aead = {
2008                 .setkey = safexcel_aead_setkey,
2009                 .encrypt = safexcel_aead_encrypt,
2010                 .decrypt = safexcel_aead_decrypt,
2011                 .ivsize = DES3_EDE_BLOCK_SIZE,
2012                 .maxauthsize = SHA224_DIGEST_SIZE,
2013                 .base = {
2014                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
2015                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede",
2016                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2017                         .cra_flags = CRYPTO_ALG_ASYNC |
2018                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2019                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2020                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2021                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2022                         .cra_alignmask = 0,
2023                         .cra_init = safexcel_aead_sha224_des3_cra_init,
2024                         .cra_exit = safexcel_aead_cra_exit,
2025                         .cra_module = THIS_MODULE,
2026                 },
2027         },
2028 };
2029
2030 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm)
2031 {
2032         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2033
2034         safexcel_aead_sha512_cra_init(tfm);
2035         ctx->alg = SAFEXCEL_3DES; /* override default */
2036         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2037         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2038         return 0;
2039 }
2040
2041 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = {
2042         .type = SAFEXCEL_ALG_TYPE_AEAD,
2043         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2044         .alg.aead = {
2045                 .setkey = safexcel_aead_setkey,
2046                 .encrypt = safexcel_aead_encrypt,
2047                 .decrypt = safexcel_aead_decrypt,
2048                 .ivsize = DES3_EDE_BLOCK_SIZE,
2049                 .maxauthsize = SHA512_DIGEST_SIZE,
2050                 .base = {
2051                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
2052                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede",
2053                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2054                         .cra_flags = CRYPTO_ALG_ASYNC |
2055                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2056                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2057                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2058                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2059                         .cra_alignmask = 0,
2060                         .cra_init = safexcel_aead_sha512_des3_cra_init,
2061                         .cra_exit = safexcel_aead_cra_exit,
2062                         .cra_module = THIS_MODULE,
2063                 },
2064         },
2065 };
2066
2067 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm)
2068 {
2069         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2070
2071         safexcel_aead_sha384_cra_init(tfm);
2072         ctx->alg = SAFEXCEL_3DES; /* override default */
2073         ctx->blocksz = DES3_EDE_BLOCK_SIZE;
2074         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2075         return 0;
2076 }
2077
2078 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = {
2079         .type = SAFEXCEL_ALG_TYPE_AEAD,
2080         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2081         .alg.aead = {
2082                 .setkey = safexcel_aead_setkey,
2083                 .encrypt = safexcel_aead_encrypt,
2084                 .decrypt = safexcel_aead_decrypt,
2085                 .ivsize = DES3_EDE_BLOCK_SIZE,
2086                 .maxauthsize = SHA384_DIGEST_SIZE,
2087                 .base = {
2088                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
2089                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede",
2090                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2091                         .cra_flags = CRYPTO_ALG_ASYNC |
2092                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2093                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2094                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2095                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2096                         .cra_alignmask = 0,
2097                         .cra_init = safexcel_aead_sha384_des3_cra_init,
2098                         .cra_exit = safexcel_aead_cra_exit,
2099                         .cra_module = THIS_MODULE,
2100                 },
2101         },
2102 };
2103
2104 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm)
2105 {
2106         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2107
2108         safexcel_aead_sha1_cra_init(tfm);
2109         ctx->alg = SAFEXCEL_DES; /* override default */
2110         ctx->blocksz = DES_BLOCK_SIZE;
2111         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2112         return 0;
2113 }
2114
2115 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = {
2116         .type = SAFEXCEL_ALG_TYPE_AEAD,
2117         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1,
2118         .alg.aead = {
2119                 .setkey = safexcel_aead_setkey,
2120                 .encrypt = safexcel_aead_encrypt,
2121                 .decrypt = safexcel_aead_decrypt,
2122                 .ivsize = DES_BLOCK_SIZE,
2123                 .maxauthsize = SHA1_DIGEST_SIZE,
2124                 .base = {
2125                         .cra_name = "authenc(hmac(sha1),cbc(des))",
2126                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des",
2127                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2128                         .cra_flags = CRYPTO_ALG_ASYNC |
2129                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2130                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2131                         .cra_blocksize = DES_BLOCK_SIZE,
2132                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2133                         .cra_alignmask = 0,
2134                         .cra_init = safexcel_aead_sha1_des_cra_init,
2135                         .cra_exit = safexcel_aead_cra_exit,
2136                         .cra_module = THIS_MODULE,
2137                 },
2138         },
2139 };
2140
2141 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm)
2142 {
2143         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2144
2145         safexcel_aead_sha256_cra_init(tfm);
2146         ctx->alg = SAFEXCEL_DES; /* override default */
2147         ctx->blocksz = DES_BLOCK_SIZE;
2148         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2149         return 0;
2150 }
2151
2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = {
2153         .type = SAFEXCEL_ALG_TYPE_AEAD,
2154         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2155         .alg.aead = {
2156                 .setkey = safexcel_aead_setkey,
2157                 .encrypt = safexcel_aead_encrypt,
2158                 .decrypt = safexcel_aead_decrypt,
2159                 .ivsize = DES_BLOCK_SIZE,
2160                 .maxauthsize = SHA256_DIGEST_SIZE,
2161                 .base = {
2162                         .cra_name = "authenc(hmac(sha256),cbc(des))",
2163                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des",
2164                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2165                         .cra_flags = CRYPTO_ALG_ASYNC |
2166                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2167                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2168                         .cra_blocksize = DES_BLOCK_SIZE,
2169                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2170                         .cra_alignmask = 0,
2171                         .cra_init = safexcel_aead_sha256_des_cra_init,
2172                         .cra_exit = safexcel_aead_cra_exit,
2173                         .cra_module = THIS_MODULE,
2174                 },
2175         },
2176 };
2177
2178 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm)
2179 {
2180         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2181
2182         safexcel_aead_sha224_cra_init(tfm);
2183         ctx->alg = SAFEXCEL_DES; /* override default */
2184         ctx->blocksz = DES_BLOCK_SIZE;
2185         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2186         return 0;
2187 }
2188
2189 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = {
2190         .type = SAFEXCEL_ALG_TYPE_AEAD,
2191         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256,
2192         .alg.aead = {
2193                 .setkey = safexcel_aead_setkey,
2194                 .encrypt = safexcel_aead_encrypt,
2195                 .decrypt = safexcel_aead_decrypt,
2196                 .ivsize = DES_BLOCK_SIZE,
2197                 .maxauthsize = SHA224_DIGEST_SIZE,
2198                 .base = {
2199                         .cra_name = "authenc(hmac(sha224),cbc(des))",
2200                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des",
2201                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2202                         .cra_flags = CRYPTO_ALG_ASYNC |
2203                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2204                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2205                         .cra_blocksize = DES_BLOCK_SIZE,
2206                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2207                         .cra_alignmask = 0,
2208                         .cra_init = safexcel_aead_sha224_des_cra_init,
2209                         .cra_exit = safexcel_aead_cra_exit,
2210                         .cra_module = THIS_MODULE,
2211                 },
2212         },
2213 };
2214
2215 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm)
2216 {
2217         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2218
2219         safexcel_aead_sha512_cra_init(tfm);
2220         ctx->alg = SAFEXCEL_DES; /* override default */
2221         ctx->blocksz = DES_BLOCK_SIZE;
2222         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2223         return 0;
2224 }
2225
2226 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = {
2227         .type = SAFEXCEL_ALG_TYPE_AEAD,
2228         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2229         .alg.aead = {
2230                 .setkey = safexcel_aead_setkey,
2231                 .encrypt = safexcel_aead_encrypt,
2232                 .decrypt = safexcel_aead_decrypt,
2233                 .ivsize = DES_BLOCK_SIZE,
2234                 .maxauthsize = SHA512_DIGEST_SIZE,
2235                 .base = {
2236                         .cra_name = "authenc(hmac(sha512),cbc(des))",
2237                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des",
2238                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2239                         .cra_flags = CRYPTO_ALG_ASYNC |
2240                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2241                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2242                         .cra_blocksize = DES_BLOCK_SIZE,
2243                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2244                         .cra_alignmask = 0,
2245                         .cra_init = safexcel_aead_sha512_des_cra_init,
2246                         .cra_exit = safexcel_aead_cra_exit,
2247                         .cra_module = THIS_MODULE,
2248                 },
2249         },
2250 };
2251
2252 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm)
2253 {
2254         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2255
2256         safexcel_aead_sha384_cra_init(tfm);
2257         ctx->alg = SAFEXCEL_DES; /* override default */
2258         ctx->blocksz = DES_BLOCK_SIZE;
2259         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
2260         return 0;
2261 }
2262
2263 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = {
2264         .type = SAFEXCEL_ALG_TYPE_AEAD,
2265         .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512,
2266         .alg.aead = {
2267                 .setkey = safexcel_aead_setkey,
2268                 .encrypt = safexcel_aead_encrypt,
2269                 .decrypt = safexcel_aead_decrypt,
2270                 .ivsize = DES_BLOCK_SIZE,
2271                 .maxauthsize = SHA384_DIGEST_SIZE,
2272                 .base = {
2273                         .cra_name = "authenc(hmac(sha384),cbc(des))",
2274                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des",
2275                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2276                         .cra_flags = CRYPTO_ALG_ASYNC |
2277                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2278                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2279                         .cra_blocksize = DES_BLOCK_SIZE,
2280                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2281                         .cra_alignmask = 0,
2282                         .cra_init = safexcel_aead_sha384_des_cra_init,
2283                         .cra_exit = safexcel_aead_cra_exit,
2284                         .cra_module = THIS_MODULE,
2285                 },
2286         },
2287 };
2288
2289 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm)
2290 {
2291         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2292
2293         safexcel_aead_sha1_cra_init(tfm);
2294         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2295         return 0;
2296 }
2297
2298 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = {
2299         .type = SAFEXCEL_ALG_TYPE_AEAD,
2300         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1,
2301         .alg.aead = {
2302                 .setkey = safexcel_aead_setkey,
2303                 .encrypt = safexcel_aead_encrypt,
2304                 .decrypt = safexcel_aead_decrypt,
2305                 .ivsize = CTR_RFC3686_IV_SIZE,
2306                 .maxauthsize = SHA1_DIGEST_SIZE,
2307                 .base = {
2308                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2309                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes",
2310                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2311                         .cra_flags = CRYPTO_ALG_ASYNC |
2312                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2313                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2314                         .cra_blocksize = 1,
2315                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2316                         .cra_alignmask = 0,
2317                         .cra_init = safexcel_aead_sha1_ctr_cra_init,
2318                         .cra_exit = safexcel_aead_cra_exit,
2319                         .cra_module = THIS_MODULE,
2320                 },
2321         },
2322 };
2323
2324 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm)
2325 {
2326         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2327
2328         safexcel_aead_sha256_cra_init(tfm);
2329         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2330         return 0;
2331 }
2332
2333 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = {
2334         .type = SAFEXCEL_ALG_TYPE_AEAD,
2335         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2336         .alg.aead = {
2337                 .setkey = safexcel_aead_setkey,
2338                 .encrypt = safexcel_aead_encrypt,
2339                 .decrypt = safexcel_aead_decrypt,
2340                 .ivsize = CTR_RFC3686_IV_SIZE,
2341                 .maxauthsize = SHA256_DIGEST_SIZE,
2342                 .base = {
2343                         .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2344                         .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes",
2345                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2346                         .cra_flags = CRYPTO_ALG_ASYNC |
2347                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2348                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2349                         .cra_blocksize = 1,
2350                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2351                         .cra_alignmask = 0,
2352                         .cra_init = safexcel_aead_sha256_ctr_cra_init,
2353                         .cra_exit = safexcel_aead_cra_exit,
2354                         .cra_module = THIS_MODULE,
2355                 },
2356         },
2357 };
2358
2359 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm)
2360 {
2361         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2362
2363         safexcel_aead_sha224_cra_init(tfm);
2364         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2365         return 0;
2366 }
2367
2368 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = {
2369         .type = SAFEXCEL_ALG_TYPE_AEAD,
2370         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256,
2371         .alg.aead = {
2372                 .setkey = safexcel_aead_setkey,
2373                 .encrypt = safexcel_aead_encrypt,
2374                 .decrypt = safexcel_aead_decrypt,
2375                 .ivsize = CTR_RFC3686_IV_SIZE,
2376                 .maxauthsize = SHA224_DIGEST_SIZE,
2377                 .base = {
2378                         .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
2379                         .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes",
2380                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2381                         .cra_flags = CRYPTO_ALG_ASYNC |
2382                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2383                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2384                         .cra_blocksize = 1,
2385                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2386                         .cra_alignmask = 0,
2387                         .cra_init = safexcel_aead_sha224_ctr_cra_init,
2388                         .cra_exit = safexcel_aead_cra_exit,
2389                         .cra_module = THIS_MODULE,
2390                 },
2391         },
2392 };
2393
2394 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm)
2395 {
2396         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2397
2398         safexcel_aead_sha512_cra_init(tfm);
2399         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2400         return 0;
2401 }
2402
2403 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = {
2404         .type = SAFEXCEL_ALG_TYPE_AEAD,
2405         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2406         .alg.aead = {
2407                 .setkey = safexcel_aead_setkey,
2408                 .encrypt = safexcel_aead_encrypt,
2409                 .decrypt = safexcel_aead_decrypt,
2410                 .ivsize = CTR_RFC3686_IV_SIZE,
2411                 .maxauthsize = SHA512_DIGEST_SIZE,
2412                 .base = {
2413                         .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
2414                         .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes",
2415                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2416                         .cra_flags = CRYPTO_ALG_ASYNC |
2417                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2418                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2419                         .cra_blocksize = 1,
2420                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2421                         .cra_alignmask = 0,
2422                         .cra_init = safexcel_aead_sha512_ctr_cra_init,
2423                         .cra_exit = safexcel_aead_cra_exit,
2424                         .cra_module = THIS_MODULE,
2425                 },
2426         },
2427 };
2428
2429 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm)
2430 {
2431         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2432
2433         safexcel_aead_sha384_cra_init(tfm);
2434         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */
2435         return 0;
2436 }
2437
2438 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = {
2439         .type = SAFEXCEL_ALG_TYPE_AEAD,
2440         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512,
2441         .alg.aead = {
2442                 .setkey = safexcel_aead_setkey,
2443                 .encrypt = safexcel_aead_encrypt,
2444                 .decrypt = safexcel_aead_decrypt,
2445                 .ivsize = CTR_RFC3686_IV_SIZE,
2446                 .maxauthsize = SHA384_DIGEST_SIZE,
2447                 .base = {
2448                         .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
2449                         .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes",
2450                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2451                         .cra_flags = CRYPTO_ALG_ASYNC |
2452                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2453                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2454                         .cra_blocksize = 1,
2455                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2456                         .cra_alignmask = 0,
2457                         .cra_init = safexcel_aead_sha384_ctr_cra_init,
2458                         .cra_exit = safexcel_aead_cra_exit,
2459                         .cra_module = THIS_MODULE,
2460                 },
2461         },
2462 };
2463
2464 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
2465                                            const u8 *key, unsigned int len)
2466 {
2467         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
2468         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2469         struct safexcel_crypto_priv *priv = ctx->base.priv;
2470         struct crypto_aes_ctx aes;
2471         int ret, i;
2472         unsigned int keylen;
2473
2474         /* Check for illegal XTS keys */
2475         ret = xts_verify_key(ctfm, key, len);
2476         if (ret)
2477                 return ret;
2478
2479         /* Only half of the key data is cipher key */
2480         keylen = (len >> 1);
2481         ret = aes_expandkey(&aes, key, keylen);
2482         if (ret)
2483                 return ret;
2484
2485         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2486                 for (i = 0; i < keylen / sizeof(u32); i++) {
2487                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2488                                 ctx->base.needs_inv = true;
2489                                 break;
2490                         }
2491                 }
2492         }
2493
2494         for (i = 0; i < keylen / sizeof(u32); i++)
2495                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2496
2497         /* The other half is the tweak key */
2498         ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
2499         if (ret)
2500                 return ret;
2501
2502         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2503                 for (i = 0; i < keylen / sizeof(u32); i++) {
2504                         if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) !=
2505                             aes.key_enc[i]) {
2506                                 ctx->base.needs_inv = true;
2507                                 break;
2508                         }
2509                 }
2510         }
2511
2512         for (i = 0; i < keylen / sizeof(u32); i++)
2513                 ctx->key[i + keylen / sizeof(u32)] =
2514                         cpu_to_le32(aes.key_enc[i]);
2515
2516         ctx->key_len = keylen << 1;
2517
2518         memzero_explicit(&aes, sizeof(aes));
2519         return 0;
2520 }
2521
2522 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm)
2523 {
2524         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2525
2526         safexcel_skcipher_cra_init(tfm);
2527         ctx->alg  = SAFEXCEL_AES;
2528         ctx->blocksz = AES_BLOCK_SIZE;
2529         ctx->xts  = 1;
2530         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS;
2531         return 0;
2532 }
2533
2534 static int safexcel_encrypt_xts(struct skcipher_request *req)
2535 {
2536         if (req->cryptlen < XTS_BLOCK_SIZE)
2537                 return -EINVAL;
2538         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2539                                   SAFEXCEL_ENCRYPT);
2540 }
2541
2542 static int safexcel_decrypt_xts(struct skcipher_request *req)
2543 {
2544         if (req->cryptlen < XTS_BLOCK_SIZE)
2545                 return -EINVAL;
2546         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
2547                                   SAFEXCEL_DECRYPT);
2548 }
2549
2550 struct safexcel_alg_template safexcel_alg_xts_aes = {
2551         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2552         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS,
2553         .alg.skcipher = {
2554                 .setkey = safexcel_skcipher_aesxts_setkey,
2555                 .encrypt = safexcel_encrypt_xts,
2556                 .decrypt = safexcel_decrypt_xts,
2557                 /* XTS actually uses 2 AES keys glued together */
2558                 .min_keysize = AES_MIN_KEY_SIZE * 2,
2559                 .max_keysize = AES_MAX_KEY_SIZE * 2,
2560                 .ivsize = XTS_BLOCK_SIZE,
2561                 .base = {
2562                         .cra_name = "xts(aes)",
2563                         .cra_driver_name = "safexcel-xts-aes",
2564                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2565                         .cra_flags = CRYPTO_ALG_ASYNC |
2566                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2567                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2568                         .cra_blocksize = XTS_BLOCK_SIZE,
2569                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2570                         .cra_alignmask = 0,
2571                         .cra_init = safexcel_skcipher_aes_xts_cra_init,
2572                         .cra_exit = safexcel_skcipher_cra_exit,
2573                         .cra_module = THIS_MODULE,
2574                 },
2575         },
2576 };
2577
2578 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
2579                                     unsigned int len)
2580 {
2581         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2582         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2583         struct safexcel_crypto_priv *priv = ctx->base.priv;
2584         struct crypto_aes_ctx aes;
2585         u32 hashkey[AES_BLOCK_SIZE >> 2];
2586         int ret, i;
2587
2588         ret = aes_expandkey(&aes, key, len);
2589         if (ret) {
2590                 memzero_explicit(&aes, sizeof(aes));
2591                 return ret;
2592         }
2593
2594         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2595                 for (i = 0; i < len / sizeof(u32); i++) {
2596                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2597                                 ctx->base.needs_inv = true;
2598                                 break;
2599                         }
2600                 }
2601         }
2602
2603         for (i = 0; i < len / sizeof(u32); i++)
2604                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2605
2606         ctx->key_len = len;
2607
2608         /* Compute hash key by encrypting zeroes with cipher key */
2609         memset(hashkey, 0, AES_BLOCK_SIZE);
2610         aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey);
2611
2612         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2613                 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
2614                         if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) {
2615                                 ctx->base.needs_inv = true;
2616                                 break;
2617                         }
2618                 }
2619         }
2620
2621         for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
2622                 ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]);
2623
2624         memzero_explicit(hashkey, AES_BLOCK_SIZE);
2625         memzero_explicit(&aes, sizeof(aes));
2626         return 0;
2627 }
2628
2629 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm)
2630 {
2631         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2632
2633         safexcel_aead_cra_init(tfm);
2634         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH;
2635         ctx->state_sz = GHASH_BLOCK_SIZE;
2636         ctx->xcm = EIP197_XCM_MODE_GCM;
2637         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2638
2639         return 0;
2640 }
2641
2642 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm)
2643 {
2644         safexcel_aead_cra_exit(tfm);
2645 }
2646
2647 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm,
2648                                          unsigned int authsize)
2649 {
2650         return crypto_gcm_check_authsize(authsize);
2651 }
2652
2653 struct safexcel_alg_template safexcel_alg_gcm = {
2654         .type = SAFEXCEL_ALG_TYPE_AEAD,
2655         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
2656         .alg.aead = {
2657                 .setkey = safexcel_aead_gcm_setkey,
2658                 .setauthsize = safexcel_aead_gcm_setauthsize,
2659                 .encrypt = safexcel_aead_encrypt,
2660                 .decrypt = safexcel_aead_decrypt,
2661                 .ivsize = GCM_AES_IV_SIZE,
2662                 .maxauthsize = GHASH_DIGEST_SIZE,
2663                 .base = {
2664                         .cra_name = "gcm(aes)",
2665                         .cra_driver_name = "safexcel-gcm-aes",
2666                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2667                         .cra_flags = CRYPTO_ALG_ASYNC |
2668                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2669                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2670                         .cra_blocksize = 1,
2671                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2672                         .cra_alignmask = 0,
2673                         .cra_init = safexcel_aead_gcm_cra_init,
2674                         .cra_exit = safexcel_aead_gcm_cra_exit,
2675                         .cra_module = THIS_MODULE,
2676                 },
2677         },
2678 };
2679
2680 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
2681                                     unsigned int len)
2682 {
2683         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
2684         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2685         struct safexcel_crypto_priv *priv = ctx->base.priv;
2686         struct crypto_aes_ctx aes;
2687         int ret, i;
2688
2689         ret = aes_expandkey(&aes, key, len);
2690         if (ret) {
2691                 memzero_explicit(&aes, sizeof(aes));
2692                 return ret;
2693         }
2694
2695         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
2696                 for (i = 0; i < len / sizeof(u32); i++) {
2697                         if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) {
2698                                 ctx->base.needs_inv = true;
2699                                 break;
2700                         }
2701                 }
2702         }
2703
2704         for (i = 0; i < len / sizeof(u32); i++) {
2705                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
2706                 ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
2707                         cpu_to_be32(aes.key_enc[i]);
2708         }
2709
2710         ctx->key_len = len;
2711         ctx->state_sz = 2 * AES_BLOCK_SIZE + len;
2712
2713         if (len == AES_KEYSIZE_192)
2714                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2715         else if (len == AES_KEYSIZE_256)
2716                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2717         else
2718                 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2719
2720         memzero_explicit(&aes, sizeof(aes));
2721         return 0;
2722 }
2723
2724 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm)
2725 {
2726         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2727
2728         safexcel_aead_cra_init(tfm);
2729         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2730         ctx->state_sz = 3 * AES_BLOCK_SIZE;
2731         ctx->xcm = EIP197_XCM_MODE_CCM;
2732         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */
2733         ctx->ctrinit = 0;
2734         return 0;
2735 }
2736
2737 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm,
2738                                          unsigned int authsize)
2739 {
2740         /* Borrowed from crypto/ccm.c */
2741         switch (authsize) {
2742         case 4:
2743         case 6:
2744         case 8:
2745         case 10:
2746         case 12:
2747         case 14:
2748         case 16:
2749                 break;
2750         default:
2751                 return -EINVAL;
2752         }
2753
2754         return 0;
2755 }
2756
2757 static int safexcel_ccm_encrypt(struct aead_request *req)
2758 {
2759         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2760
2761         if (req->iv[0] < 1 || req->iv[0] > 7)
2762                 return -EINVAL;
2763
2764         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
2765 }
2766
2767 static int safexcel_ccm_decrypt(struct aead_request *req)
2768 {
2769         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2770
2771         if (req->iv[0] < 1 || req->iv[0] > 7)
2772                 return -EINVAL;
2773
2774         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
2775 }
2776
2777 struct safexcel_alg_template safexcel_alg_ccm = {
2778         .type = SAFEXCEL_ALG_TYPE_AEAD,
2779         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
2780         .alg.aead = {
2781                 .setkey = safexcel_aead_ccm_setkey,
2782                 .setauthsize = safexcel_aead_ccm_setauthsize,
2783                 .encrypt = safexcel_ccm_encrypt,
2784                 .decrypt = safexcel_ccm_decrypt,
2785                 .ivsize = AES_BLOCK_SIZE,
2786                 .maxauthsize = AES_BLOCK_SIZE,
2787                 .base = {
2788                         .cra_name = "ccm(aes)",
2789                         .cra_driver_name = "safexcel-ccm-aes",
2790                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2791                         .cra_flags = CRYPTO_ALG_ASYNC |
2792                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2793                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2794                         .cra_blocksize = 1,
2795                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2796                         .cra_alignmask = 0,
2797                         .cra_init = safexcel_aead_ccm_cra_init,
2798                         .cra_exit = safexcel_aead_cra_exit,
2799                         .cra_module = THIS_MODULE,
2800                 },
2801         },
2802 };
2803
2804 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx,
2805                                      const u8 *key)
2806 {
2807         struct safexcel_crypto_priv *priv = ctx->base.priv;
2808
2809         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
2810                 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE))
2811                         ctx->base.needs_inv = true;
2812
2813         memcpy(ctx->key, key, CHACHA_KEY_SIZE);
2814         ctx->key_len = CHACHA_KEY_SIZE;
2815 }
2816
2817 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm,
2818                                              const u8 *key, unsigned int len)
2819 {
2820         struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
2821
2822         if (len != CHACHA_KEY_SIZE)
2823                 return -EINVAL;
2824
2825         safexcel_chacha20_setkey(ctx, key);
2826
2827         return 0;
2828 }
2829
2830 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm)
2831 {
2832         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2833
2834         safexcel_skcipher_cra_init(tfm);
2835         ctx->alg  = SAFEXCEL_CHACHA20;
2836         ctx->ctrinit = 0;
2837         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32;
2838         return 0;
2839 }
2840
2841 struct safexcel_alg_template safexcel_alg_chacha20 = {
2842         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
2843         .algo_mask = SAFEXCEL_ALG_CHACHA20,
2844         .alg.skcipher = {
2845                 .setkey = safexcel_skcipher_chacha20_setkey,
2846                 .encrypt = safexcel_encrypt,
2847                 .decrypt = safexcel_decrypt,
2848                 .min_keysize = CHACHA_KEY_SIZE,
2849                 .max_keysize = CHACHA_KEY_SIZE,
2850                 .ivsize = CHACHA_IV_SIZE,
2851                 .base = {
2852                         .cra_name = "chacha20",
2853                         .cra_driver_name = "safexcel-chacha20",
2854                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
2855                         .cra_flags = CRYPTO_ALG_ASYNC |
2856                                      CRYPTO_ALG_ALLOCATES_MEMORY |
2857                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
2858                         .cra_blocksize = 1,
2859                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
2860                         .cra_alignmask = 0,
2861                         .cra_init = safexcel_skcipher_chacha20_cra_init,
2862                         .cra_exit = safexcel_skcipher_cra_exit,
2863                         .cra_module = THIS_MODULE,
2864                 },
2865         },
2866 };
2867
2868 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm,
2869                                     const u8 *key, unsigned int len)
2870 {
2871         struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm);
2872
2873         if (ctx->aead  == EIP197_AEAD_TYPE_IPSEC_ESP &&
2874             len > EIP197_AEAD_IPSEC_NONCE_SIZE) {
2875                 /* ESP variant has nonce appended to key */
2876                 len -= EIP197_AEAD_IPSEC_NONCE_SIZE;
2877                 ctx->nonce = *(u32 *)(key + len);
2878         }
2879         if (len != CHACHA_KEY_SIZE)
2880                 return -EINVAL;
2881
2882         safexcel_chacha20_setkey(ctx, key);
2883
2884         return 0;
2885 }
2886
2887 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm,
2888                                          unsigned int authsize)
2889 {
2890         if (authsize != POLY1305_DIGEST_SIZE)
2891                 return -EINVAL;
2892         return 0;
2893 }
2894
2895 static int safexcel_aead_chachapoly_crypt(struct aead_request *req,
2896                                           enum safexcel_cipher_direction dir)
2897 {
2898         struct safexcel_cipher_req *creq = aead_request_ctx(req);
2899         struct crypto_aead *aead = crypto_aead_reqtfm(req);
2900         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2901         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2902         struct aead_request *subreq = aead_request_ctx(req);
2903         u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1];
2904         int ret = 0;
2905
2906         /*
2907          * Instead of wasting time detecting umpteen silly corner cases,
2908          * just dump all "small" requests to the fallback implementation.
2909          * HW would not be faster on such small requests anyway.
2910          */
2911         if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP ||
2912                     req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) &&
2913                    req->cryptlen > POLY1305_DIGEST_SIZE)) {
2914                 return safexcel_queue_req(&req->base, creq, dir);
2915         }
2916
2917         /* HW cannot do full (AAD+payload) zero length, use fallback */
2918         memcpy(key, ctx->key, CHACHA_KEY_SIZE);
2919         if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
2920                 /* ESP variant has nonce appended to the key */
2921                 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce;
2922                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2923                                          CHACHA_KEY_SIZE +
2924                                          EIP197_AEAD_IPSEC_NONCE_SIZE);
2925         } else {
2926                 ret = crypto_aead_setkey(ctx->fback, (u8 *)key,
2927                                          CHACHA_KEY_SIZE);
2928         }
2929         if (ret) {
2930                 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK);
2931                 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) &
2932                                             CRYPTO_TFM_REQ_MASK);
2933                 return ret;
2934         }
2935
2936         aead_request_set_tfm(subreq, ctx->fback);
2937         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
2938                                   req->base.data);
2939         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2940                                req->iv);
2941         aead_request_set_ad(subreq, req->assoclen);
2942
2943         return (dir ==  SAFEXCEL_ENCRYPT) ?
2944                 crypto_aead_encrypt(subreq) :
2945                 crypto_aead_decrypt(subreq);
2946 }
2947
2948 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req)
2949 {
2950         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT);
2951 }
2952
2953 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req)
2954 {
2955         return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT);
2956 }
2957
2958 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm)
2959 {
2960         struct crypto_aead *aead = __crypto_aead_cast(tfm);
2961         struct aead_alg *alg = crypto_aead_alg(aead);
2962         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2963
2964         safexcel_aead_cra_init(tfm);
2965
2966         /* Allocate fallback implementation */
2967         ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0,
2968                                        CRYPTO_ALG_ASYNC |
2969                                        CRYPTO_ALG_NEED_FALLBACK);
2970         if (IS_ERR(ctx->fback))
2971                 return PTR_ERR(ctx->fback);
2972
2973         crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req),
2974                                           sizeof(struct aead_request) +
2975                                           crypto_aead_reqsize(ctx->fback)));
2976
2977         return 0;
2978 }
2979
2980 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm)
2981 {
2982         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2983
2984         safexcel_aead_fallback_cra_init(tfm);
2985         ctx->alg  = SAFEXCEL_CHACHA20;
2986         ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 |
2987                     CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK;
2988         ctx->ctrinit = 0;
2989         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305;
2990         ctx->state_sz = 0; /* Precomputed by HW */
2991         return 0;
2992 }
2993
2994 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm)
2995 {
2996         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
2997
2998         crypto_free_aead(ctx->fback);
2999         safexcel_aead_cra_exit(tfm);
3000 }
3001
3002 struct safexcel_alg_template safexcel_alg_chachapoly = {
3003         .type = SAFEXCEL_ALG_TYPE_AEAD,
3004         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
3005         .alg.aead = {
3006                 .setkey = safexcel_aead_chachapoly_setkey,
3007                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
3008                 .encrypt = safexcel_aead_chachapoly_encrypt,
3009                 .decrypt = safexcel_aead_chachapoly_decrypt,
3010                 .ivsize = CHACHAPOLY_IV_SIZE,
3011                 .maxauthsize = POLY1305_DIGEST_SIZE,
3012                 .base = {
3013                         .cra_name = "rfc7539(chacha20,poly1305)",
3014                         .cra_driver_name = "safexcel-chacha20-poly1305",
3015                         /* +1 to put it above HW chacha + SW poly */
3016                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
3017                         .cra_flags = CRYPTO_ALG_ASYNC |
3018                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3019                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3020                                      CRYPTO_ALG_NEED_FALLBACK,
3021                         .cra_blocksize = 1,
3022                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3023                         .cra_alignmask = 0,
3024                         .cra_init = safexcel_aead_chachapoly_cra_init,
3025                         .cra_exit = safexcel_aead_fallback_cra_exit,
3026                         .cra_module = THIS_MODULE,
3027                 },
3028         },
3029 };
3030
3031 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm)
3032 {
3033         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3034         int ret;
3035
3036         ret = safexcel_aead_chachapoly_cra_init(tfm);
3037         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3038         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3039         return ret;
3040 }
3041
3042 struct safexcel_alg_template safexcel_alg_chachapoly_esp = {
3043         .type = SAFEXCEL_ALG_TYPE_AEAD,
3044         .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305,
3045         .alg.aead = {
3046                 .setkey = safexcel_aead_chachapoly_setkey,
3047                 .setauthsize = safexcel_aead_chachapoly_setauthsize,
3048                 .encrypt = safexcel_aead_chachapoly_encrypt,
3049                 .decrypt = safexcel_aead_chachapoly_decrypt,
3050                 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE,
3051                 .maxauthsize = POLY1305_DIGEST_SIZE,
3052                 .base = {
3053                         .cra_name = "rfc7539esp(chacha20,poly1305)",
3054                         .cra_driver_name = "safexcel-chacha20-poly1305-esp",
3055                         /* +1 to put it above HW chacha + SW poly */
3056                         .cra_priority = SAFEXCEL_CRA_PRIORITY + 1,
3057                         .cra_flags = CRYPTO_ALG_ASYNC |
3058                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3059                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3060                                      CRYPTO_ALG_NEED_FALLBACK,
3061                         .cra_blocksize = 1,
3062                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3063                         .cra_alignmask = 0,
3064                         .cra_init = safexcel_aead_chachapolyesp_cra_init,
3065                         .cra_exit = safexcel_aead_fallback_cra_exit,
3066                         .cra_module = THIS_MODULE,
3067                 },
3068         },
3069 };
3070
3071 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm,
3072                                         const u8 *key, unsigned int len)
3073 {
3074         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3075         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3076         struct safexcel_crypto_priv *priv = ctx->base.priv;
3077
3078         if (len != SM4_KEY_SIZE)
3079                 return -EINVAL;
3080
3081         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
3082                 if (memcmp(ctx->key, key, SM4_KEY_SIZE))
3083                         ctx->base.needs_inv = true;
3084
3085         memcpy(ctx->key, key, SM4_KEY_SIZE);
3086         ctx->key_len = SM4_KEY_SIZE;
3087
3088         return 0;
3089 }
3090
3091 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req)
3092 {
3093         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3094         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3095                 return -EINVAL;
3096         else
3097                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3098                                           SAFEXCEL_ENCRYPT);
3099 }
3100
3101 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req)
3102 {
3103         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3104         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3105                 return -EINVAL;
3106         else
3107                 return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
3108                                           SAFEXCEL_DECRYPT);
3109 }
3110
3111 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm)
3112 {
3113         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3114
3115         safexcel_skcipher_cra_init(tfm);
3116         ctx->alg  = SAFEXCEL_SM4;
3117         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB;
3118         ctx->blocksz = 0;
3119         ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD;
3120         return 0;
3121 }
3122
3123 struct safexcel_alg_template safexcel_alg_ecb_sm4 = {
3124         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3125         .algo_mask = SAFEXCEL_ALG_SM4,
3126         .alg.skcipher = {
3127                 .setkey = safexcel_skcipher_sm4_setkey,
3128                 .encrypt = safexcel_sm4_blk_encrypt,
3129                 .decrypt = safexcel_sm4_blk_decrypt,
3130                 .min_keysize = SM4_KEY_SIZE,
3131                 .max_keysize = SM4_KEY_SIZE,
3132                 .base = {
3133                         .cra_name = "ecb(sm4)",
3134                         .cra_driver_name = "safexcel-ecb-sm4",
3135                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3136                         .cra_flags = CRYPTO_ALG_ASYNC |
3137                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3138                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3139                         .cra_blocksize = SM4_BLOCK_SIZE,
3140                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3141                         .cra_alignmask = 0,
3142                         .cra_init = safexcel_skcipher_sm4_ecb_cra_init,
3143                         .cra_exit = safexcel_skcipher_cra_exit,
3144                         .cra_module = THIS_MODULE,
3145                 },
3146         },
3147 };
3148
3149 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm)
3150 {
3151         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3152
3153         safexcel_skcipher_cra_init(tfm);
3154         ctx->alg  = SAFEXCEL_SM4;
3155         ctx->blocksz = SM4_BLOCK_SIZE;
3156         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC;
3157         return 0;
3158 }
3159
3160 struct safexcel_alg_template safexcel_alg_cbc_sm4 = {
3161         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3162         .algo_mask = SAFEXCEL_ALG_SM4,
3163         .alg.skcipher = {
3164                 .setkey = safexcel_skcipher_sm4_setkey,
3165                 .encrypt = safexcel_sm4_blk_encrypt,
3166                 .decrypt = safexcel_sm4_blk_decrypt,
3167                 .min_keysize = SM4_KEY_SIZE,
3168                 .max_keysize = SM4_KEY_SIZE,
3169                 .ivsize = SM4_BLOCK_SIZE,
3170                 .base = {
3171                         .cra_name = "cbc(sm4)",
3172                         .cra_driver_name = "safexcel-cbc-sm4",
3173                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3174                         .cra_flags = CRYPTO_ALG_ASYNC |
3175                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3176                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3177                         .cra_blocksize = SM4_BLOCK_SIZE,
3178                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3179                         .cra_alignmask = 0,
3180                         .cra_init = safexcel_skcipher_sm4_cbc_cra_init,
3181                         .cra_exit = safexcel_skcipher_cra_exit,
3182                         .cra_module = THIS_MODULE,
3183                 },
3184         },
3185 };
3186
3187 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm)
3188 {
3189         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3190
3191         safexcel_skcipher_cra_init(tfm);
3192         ctx->alg  = SAFEXCEL_SM4;
3193         ctx->blocksz = SM4_BLOCK_SIZE;
3194         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB;
3195         return 0;
3196 }
3197
3198 struct safexcel_alg_template safexcel_alg_ofb_sm4 = {
3199         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3200         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3201         .alg.skcipher = {
3202                 .setkey = safexcel_skcipher_sm4_setkey,
3203                 .encrypt = safexcel_encrypt,
3204                 .decrypt = safexcel_decrypt,
3205                 .min_keysize = SM4_KEY_SIZE,
3206                 .max_keysize = SM4_KEY_SIZE,
3207                 .ivsize = SM4_BLOCK_SIZE,
3208                 .base = {
3209                         .cra_name = "ofb(sm4)",
3210                         .cra_driver_name = "safexcel-ofb-sm4",
3211                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3212                         .cra_flags = CRYPTO_ALG_ASYNC |
3213                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3214                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3215                         .cra_blocksize = 1,
3216                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3217                         .cra_alignmask = 0,
3218                         .cra_init = safexcel_skcipher_sm4_ofb_cra_init,
3219                         .cra_exit = safexcel_skcipher_cra_exit,
3220                         .cra_module = THIS_MODULE,
3221                 },
3222         },
3223 };
3224
3225 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm)
3226 {
3227         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3228
3229         safexcel_skcipher_cra_init(tfm);
3230         ctx->alg  = SAFEXCEL_SM4;
3231         ctx->blocksz = SM4_BLOCK_SIZE;
3232         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB;
3233         return 0;
3234 }
3235
3236 struct safexcel_alg_template safexcel_alg_cfb_sm4 = {
3237         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3238         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB,
3239         .alg.skcipher = {
3240                 .setkey = safexcel_skcipher_sm4_setkey,
3241                 .encrypt = safexcel_encrypt,
3242                 .decrypt = safexcel_decrypt,
3243                 .min_keysize = SM4_KEY_SIZE,
3244                 .max_keysize = SM4_KEY_SIZE,
3245                 .ivsize = SM4_BLOCK_SIZE,
3246                 .base = {
3247                         .cra_name = "cfb(sm4)",
3248                         .cra_driver_name = "safexcel-cfb-sm4",
3249                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3250                         .cra_flags = CRYPTO_ALG_ASYNC |
3251                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3252                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3253                         .cra_blocksize = 1,
3254                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3255                         .cra_alignmask = 0,
3256                         .cra_init = safexcel_skcipher_sm4_cfb_cra_init,
3257                         .cra_exit = safexcel_skcipher_cra_exit,
3258                         .cra_module = THIS_MODULE,
3259                 },
3260         },
3261 };
3262
3263 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm,
3264                                            const u8 *key, unsigned int len)
3265 {
3266         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
3267         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3268
3269         /* last 4 bytes of key are the nonce! */
3270         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3271         /* exclude the nonce here */
3272         len -= CTR_RFC3686_NONCE_SIZE;
3273
3274         return safexcel_skcipher_sm4_setkey(ctfm, key, len);
3275 }
3276
3277 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm)
3278 {
3279         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3280
3281         safexcel_skcipher_cra_init(tfm);
3282         ctx->alg  = SAFEXCEL_SM4;
3283         ctx->blocksz = SM4_BLOCK_SIZE;
3284         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3285         return 0;
3286 }
3287
3288 struct safexcel_alg_template safexcel_alg_ctr_sm4 = {
3289         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
3290         .algo_mask = SAFEXCEL_ALG_SM4,
3291         .alg.skcipher = {
3292                 .setkey = safexcel_skcipher_sm4ctr_setkey,
3293                 .encrypt = safexcel_encrypt,
3294                 .decrypt = safexcel_decrypt,
3295                 /* Add nonce size */
3296                 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3297                 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3298                 .ivsize = CTR_RFC3686_IV_SIZE,
3299                 .base = {
3300                         .cra_name = "rfc3686(ctr(sm4))",
3301                         .cra_driver_name = "safexcel-ctr-sm4",
3302                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3303                         .cra_flags = CRYPTO_ALG_ASYNC |
3304                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3305                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3306                         .cra_blocksize = 1,
3307                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3308                         .cra_alignmask = 0,
3309                         .cra_init = safexcel_skcipher_sm4_ctr_cra_init,
3310                         .cra_exit = safexcel_skcipher_cra_exit,
3311                         .cra_module = THIS_MODULE,
3312                 },
3313         },
3314 };
3315
3316 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req)
3317 {
3318         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3319         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3320                 return -EINVAL;
3321
3322         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3323                                   SAFEXCEL_ENCRYPT);
3324 }
3325
3326 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req)
3327 {
3328         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3329
3330         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3331         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3332                 return -EINVAL;
3333
3334         return safexcel_queue_req(&req->base, aead_request_ctx(req),
3335                                   SAFEXCEL_DECRYPT);
3336 }
3337
3338 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm)
3339 {
3340         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3341
3342         safexcel_aead_cra_init(tfm);
3343         ctx->alg = SAFEXCEL_SM4;
3344         ctx->blocksz = SM4_BLOCK_SIZE;
3345         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
3346         ctx->state_sz = SHA1_DIGEST_SIZE;
3347         return 0;
3348 }
3349
3350 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = {
3351         .type = SAFEXCEL_ALG_TYPE_AEAD,
3352         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3353         .alg.aead = {
3354                 .setkey = safexcel_aead_setkey,
3355                 .encrypt = safexcel_aead_sm4_blk_encrypt,
3356                 .decrypt = safexcel_aead_sm4_blk_decrypt,
3357                 .ivsize = SM4_BLOCK_SIZE,
3358                 .maxauthsize = SHA1_DIGEST_SIZE,
3359                 .base = {
3360                         .cra_name = "authenc(hmac(sha1),cbc(sm4))",
3361                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4",
3362                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3363                         .cra_flags = CRYPTO_ALG_ASYNC |
3364                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3365                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3366                         .cra_blocksize = SM4_BLOCK_SIZE,
3367                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3368                         .cra_alignmask = 0,
3369                         .cra_init = safexcel_aead_sm4cbc_sha1_cra_init,
3370                         .cra_exit = safexcel_aead_cra_exit,
3371                         .cra_module = THIS_MODULE,
3372                 },
3373         },
3374 };
3375
3376 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm,
3377                                          const u8 *key, unsigned int len)
3378 {
3379         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3380         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3381
3382         /* Keep fallback cipher synchronized */
3383         return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?:
3384                safexcel_aead_setkey(ctfm, key, len);
3385 }
3386
3387 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm,
3388                                               unsigned int authsize)
3389 {
3390         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3391         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3392
3393         /* Keep fallback cipher synchronized */
3394         return crypto_aead_setauthsize(ctx->fback, authsize);
3395 }
3396
3397 static int safexcel_aead_fallback_crypt(struct aead_request *req,
3398                                         enum safexcel_cipher_direction dir)
3399 {
3400         struct crypto_aead *aead = crypto_aead_reqtfm(req);
3401         struct crypto_tfm *tfm = crypto_aead_tfm(aead);
3402         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3403         struct aead_request *subreq = aead_request_ctx(req);
3404
3405         aead_request_set_tfm(subreq, ctx->fback);
3406         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
3407                                   req->base.data);
3408         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
3409                                req->iv);
3410         aead_request_set_ad(subreq, req->assoclen);
3411
3412         return (dir ==  SAFEXCEL_ENCRYPT) ?
3413                 crypto_aead_encrypt(subreq) :
3414                 crypto_aead_decrypt(subreq);
3415 }
3416
3417 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req)
3418 {
3419         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3420
3421         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3422         if (req->cryptlen & (SM4_BLOCK_SIZE - 1))
3423                 return -EINVAL;
3424         else if (req->cryptlen || req->assoclen) /* If input length > 0 only */
3425                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3426
3427         /* HW cannot do full (AAD+payload) zero length, use fallback */
3428         return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT);
3429 }
3430
3431 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req)
3432 {
3433         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3434         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3435
3436         /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */
3437         if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1))
3438                 return -EINVAL;
3439         else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen)
3440                 /* If input length > 0 only */
3441                 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3442
3443         /* HW cannot do full (AAD+payload) zero length, use fallback */
3444         return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT);
3445 }
3446
3447 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm)
3448 {
3449         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3450
3451         safexcel_aead_fallback_cra_init(tfm);
3452         ctx->alg = SAFEXCEL_SM4;
3453         ctx->blocksz = SM4_BLOCK_SIZE;
3454         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
3455         ctx->state_sz = SM3_DIGEST_SIZE;
3456         return 0;
3457 }
3458
3459 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = {
3460         .type = SAFEXCEL_ALG_TYPE_AEAD,
3461         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3462         .alg.aead = {
3463                 .setkey = safexcel_aead_fallback_setkey,
3464                 .setauthsize = safexcel_aead_fallback_setauthsize,
3465                 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt,
3466                 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt,
3467                 .ivsize = SM4_BLOCK_SIZE,
3468                 .maxauthsize = SM3_DIGEST_SIZE,
3469                 .base = {
3470                         .cra_name = "authenc(hmac(sm3),cbc(sm4))",
3471                         .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4",
3472                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3473                         .cra_flags = CRYPTO_ALG_ASYNC |
3474                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3475                                      CRYPTO_ALG_KERN_DRIVER_ONLY |
3476                                      CRYPTO_ALG_NEED_FALLBACK,
3477                         .cra_blocksize = SM4_BLOCK_SIZE,
3478                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3479                         .cra_alignmask = 0,
3480                         .cra_init = safexcel_aead_sm4cbc_sm3_cra_init,
3481                         .cra_exit = safexcel_aead_fallback_cra_exit,
3482                         .cra_module = THIS_MODULE,
3483                 },
3484         },
3485 };
3486
3487 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm)
3488 {
3489         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3490
3491         safexcel_aead_sm4cbc_sha1_cra_init(tfm);
3492         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3493         return 0;
3494 }
3495
3496 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = {
3497         .type = SAFEXCEL_ALG_TYPE_AEAD,
3498         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1,
3499         .alg.aead = {
3500                 .setkey = safexcel_aead_setkey,
3501                 .encrypt = safexcel_aead_encrypt,
3502                 .decrypt = safexcel_aead_decrypt,
3503                 .ivsize = CTR_RFC3686_IV_SIZE,
3504                 .maxauthsize = SHA1_DIGEST_SIZE,
3505                 .base = {
3506                         .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))",
3507                         .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4",
3508                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3509                         .cra_flags = CRYPTO_ALG_ASYNC |
3510                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3511                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3512                         .cra_blocksize = 1,
3513                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3514                         .cra_alignmask = 0,
3515                         .cra_init = safexcel_aead_sm4ctr_sha1_cra_init,
3516                         .cra_exit = safexcel_aead_cra_exit,
3517                         .cra_module = THIS_MODULE,
3518                 },
3519         },
3520 };
3521
3522 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm)
3523 {
3524         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3525
3526         safexcel_aead_sm4cbc_sm3_cra_init(tfm);
3527         ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD;
3528         return 0;
3529 }
3530
3531 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = {
3532         .type = SAFEXCEL_ALG_TYPE_AEAD,
3533         .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3,
3534         .alg.aead = {
3535                 .setkey = safexcel_aead_setkey,
3536                 .encrypt = safexcel_aead_encrypt,
3537                 .decrypt = safexcel_aead_decrypt,
3538                 .ivsize = CTR_RFC3686_IV_SIZE,
3539                 .maxauthsize = SM3_DIGEST_SIZE,
3540                 .base = {
3541                         .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))",
3542                         .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4",
3543                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3544                         .cra_flags = CRYPTO_ALG_ASYNC |
3545                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3546                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3547                         .cra_blocksize = 1,
3548                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3549                         .cra_alignmask = 0,
3550                         .cra_init = safexcel_aead_sm4ctr_sm3_cra_init,
3551                         .cra_exit = safexcel_aead_cra_exit,
3552                         .cra_module = THIS_MODULE,
3553                 },
3554         },
3555 };
3556
3557 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key,
3558                                        unsigned int len)
3559 {
3560         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3561         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3562
3563         /* last 4 bytes of key are the nonce! */
3564         ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE);
3565
3566         len -= CTR_RFC3686_NONCE_SIZE;
3567         return safexcel_aead_gcm_setkey(ctfm, key, len);
3568 }
3569
3570 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm,
3571                                             unsigned int authsize)
3572 {
3573         return crypto_rfc4106_check_authsize(authsize);
3574 }
3575
3576 static int safexcel_rfc4106_encrypt(struct aead_request *req)
3577 {
3578         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3579                safexcel_aead_encrypt(req);
3580 }
3581
3582 static int safexcel_rfc4106_decrypt(struct aead_request *req)
3583 {
3584         return crypto_ipsec_check_assoclen(req->assoclen) ?:
3585                safexcel_aead_decrypt(req);
3586 }
3587
3588 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm)
3589 {
3590         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3591         int ret;
3592
3593         ret = safexcel_aead_gcm_cra_init(tfm);
3594         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3595         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3596         return ret;
3597 }
3598
3599 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = {
3600         .type = SAFEXCEL_ALG_TYPE_AEAD,
3601         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3602         .alg.aead = {
3603                 .setkey = safexcel_rfc4106_gcm_setkey,
3604                 .setauthsize = safexcel_rfc4106_gcm_setauthsize,
3605                 .encrypt = safexcel_rfc4106_encrypt,
3606                 .decrypt = safexcel_rfc4106_decrypt,
3607                 .ivsize = GCM_RFC4106_IV_SIZE,
3608                 .maxauthsize = GHASH_DIGEST_SIZE,
3609                 .base = {
3610                         .cra_name = "rfc4106(gcm(aes))",
3611                         .cra_driver_name = "safexcel-rfc4106-gcm-aes",
3612                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3613                         .cra_flags = CRYPTO_ALG_ASYNC |
3614                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3615                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3616                         .cra_blocksize = 1,
3617                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3618                         .cra_alignmask = 0,
3619                         .cra_init = safexcel_rfc4106_gcm_cra_init,
3620                         .cra_exit = safexcel_aead_gcm_cra_exit,
3621                 },
3622         },
3623 };
3624
3625 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm,
3626                                             unsigned int authsize)
3627 {
3628         if (authsize != GHASH_DIGEST_SIZE)
3629                 return -EINVAL;
3630
3631         return 0;
3632 }
3633
3634 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm)
3635 {
3636         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3637         int ret;
3638
3639         ret = safexcel_aead_gcm_cra_init(tfm);
3640         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC;
3641         return ret;
3642 }
3643
3644 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = {
3645         .type = SAFEXCEL_ALG_TYPE_AEAD,
3646         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH,
3647         .alg.aead = {
3648                 .setkey = safexcel_rfc4106_gcm_setkey,
3649                 .setauthsize = safexcel_rfc4543_gcm_setauthsize,
3650                 .encrypt = safexcel_rfc4106_encrypt,
3651                 .decrypt = safexcel_rfc4106_decrypt,
3652                 .ivsize = GCM_RFC4543_IV_SIZE,
3653                 .maxauthsize = GHASH_DIGEST_SIZE,
3654                 .base = {
3655                         .cra_name = "rfc4543(gcm(aes))",
3656                         .cra_driver_name = "safexcel-rfc4543-gcm-aes",
3657                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3658                         .cra_flags = CRYPTO_ALG_ASYNC |
3659                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3660                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3661                         .cra_blocksize = 1,
3662                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3663                         .cra_alignmask = 0,
3664                         .cra_init = safexcel_rfc4543_gcm_cra_init,
3665                         .cra_exit = safexcel_aead_gcm_cra_exit,
3666                 },
3667         },
3668 };
3669
3670 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
3671                                        unsigned int len)
3672 {
3673         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
3674         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3675
3676         /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */
3677         *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1;
3678         /* last 3 bytes of key are the nonce! */
3679         memcpy((u8 *)&ctx->nonce + 1, key + len -
3680                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE,
3681                EIP197_AEAD_IPSEC_CCM_NONCE_SIZE);
3682
3683         len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE;
3684         return safexcel_aead_ccm_setkey(ctfm, key, len);
3685 }
3686
3687 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm,
3688                                             unsigned int authsize)
3689 {
3690         /* Borrowed from crypto/ccm.c */
3691         switch (authsize) {
3692         case 8:
3693         case 12:
3694         case 16:
3695                 break;
3696         default:
3697                 return -EINVAL;
3698         }
3699
3700         return 0;
3701 }
3702
3703 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req)
3704 {
3705         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3706
3707         /* Borrowed from crypto/ccm.c */
3708         if (req->assoclen != 16 && req->assoclen != 20)
3709                 return -EINVAL;
3710
3711         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT);
3712 }
3713
3714 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req)
3715 {
3716         struct safexcel_cipher_req *creq = aead_request_ctx(req);
3717
3718         /* Borrowed from crypto/ccm.c */
3719         if (req->assoclen != 16 && req->assoclen != 20)
3720                 return -EINVAL;
3721
3722         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT);
3723 }
3724
3725 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm)
3726 {
3727         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
3728         int ret;
3729
3730         ret = safexcel_aead_ccm_cra_init(tfm);
3731         ctx->aead  = EIP197_AEAD_TYPE_IPSEC_ESP;
3732         ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE;
3733         return ret;
3734 }
3735
3736 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = {
3737         .type = SAFEXCEL_ALG_TYPE_AEAD,
3738         .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL,
3739         .alg.aead = {
3740                 .setkey = safexcel_rfc4309_ccm_setkey,
3741                 .setauthsize = safexcel_rfc4309_ccm_setauthsize,
3742                 .encrypt = safexcel_rfc4309_ccm_encrypt,
3743                 .decrypt = safexcel_rfc4309_ccm_decrypt,
3744                 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE,
3745                 .maxauthsize = AES_BLOCK_SIZE,
3746                 .base = {
3747                         .cra_name = "rfc4309(ccm(aes))",
3748                         .cra_driver_name = "safexcel-rfc4309-ccm-aes",
3749                         .cra_priority = SAFEXCEL_CRA_PRIORITY,
3750                         .cra_flags = CRYPTO_ALG_ASYNC |
3751                                      CRYPTO_ALG_ALLOCATES_MEMORY |
3752                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
3753                         .cra_blocksize = 1,
3754                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
3755                         .cra_alignmask = 0,
3756                         .cra_init = safexcel_rfc4309_ccm_cra_init,
3757                         .cra_exit = safexcel_aead_cra_exit,
3758                         .cra_module = THIS_MODULE,
3759                 },
3760         },
3761 };