drivers/crypto/omap-aes-gcm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Cryptographic API.
   4  *
   5  * Support for OMAP AES GCM HW acceleration.
   6  *
   7  * Copyright (c) 2016 Texas Instruments Incorporated
   8  */
   9
  10 #include <linux/errno.h>
  11 #include <linux/scatterlist.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/omap-dma.h>
  15 #include <linux/interrupt.h>
  16 #include <crypto/aes.h>
  17 #include <crypto/gcm.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/skcipher.h>
  20 #include <crypto/internal/aead.h>
  21
  22 #include "omap-crypto.h"
  23 #include "omap-aes.h"
  24
  25 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
  26                                      struct aead_request *req);
  27
  28 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
  29 {
  30         struct aead_request *req = dd->aead_req;
  31
  32         dd->flags &= ~FLAGS_BUSY;
  33         dd->in_sg = NULL;
  34         dd->out_sg = NULL;
  35
  36         req->base.complete(&req->base, ret);
  37 }
  38
  39 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
  40 {
  41         u8 *tag;
  42         int alen, clen, i, ret = 0, nsg;
  43         struct omap_aes_reqctx *rctx;
  44
  45         alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
  46         clen = ALIGN(dd->total, AES_BLOCK_SIZE);
  47         rctx = aead_request_ctx(dd->aead_req);
  48
  49         nsg = !!(dd->assoc_len && dd->total);
  50
  51         dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  52                                DMA_FROM_DEVICE);
  53         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  54         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
  55         omap_aes_crypt_dma_stop(dd);
  56
  57         omap_crypto_cleanup(dd->out_sg, dd->orig_out,
  58                             dd->aead_req->assoclen, dd->total,
  59                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  60
  61         if (dd->flags & FLAGS_ENCRYPT)
  62                 scatterwalk_map_and_copy(rctx->auth_tag,
  63                                          dd->aead_req->dst,
  64                                          dd->total + dd->aead_req->assoclen,
  65                                          dd->authsize, 1);
  66
  67         omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
  68                             FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
  69
  70         omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
  71                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  72
  73         if (!(dd->flags & FLAGS_ENCRYPT)) {
  74                 tag = (u8 *)rctx->auth_tag;
  75                 for (i = 0; i < dd->authsize; i++) {
  76                         if (tag[i]) {
  77                                 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
  78                                 ret = -EBADMSG;
  79                         }
  80                 }
  81         }
  82
  83         omap_aes_gcm_finish_req(dd, ret);
  84         omap_aes_gcm_handle_queue(dd, NULL);
  85 }
  86
  87 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
  88                                      struct aead_request *req)
  89 {
  90         int alen, clen, cryptlen, assoclen, ret;
  91         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  92         unsigned int authlen = crypto_aead_authsize(aead);
  93         struct scatterlist *tmp, sg_arr[2];
  94         int nsg;
  95         u16 flags;
  96
  97         assoclen = req->assoclen;
  98         cryptlen = req->cryptlen;
  99
 100         if (dd->flags & FLAGS_RFC4106_GCM)
 101                 assoclen -= 8;
 102
 103         if (!(dd->flags & FLAGS_ENCRYPT))
 104                 cryptlen -= authlen;
 105
 106         alen = ALIGN(assoclen, AES_BLOCK_SIZE);
 107         clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
 108
 109         nsg = !!(assoclen && cryptlen);
 110
 111         omap_aes_clear_copy_flags(dd);
 112
 113         sg_init_table(dd->in_sgl, nsg + 1);
 114         if (assoclen) {
 115                 tmp = req->src;
 116                 ret = omap_crypto_align_sg(&tmp, assoclen,
 117                                            AES_BLOCK_SIZE, dd->in_sgl,
 118                                            OMAP_CRYPTO_COPY_DATA |
 119                                            OMAP_CRYPTO_ZERO_BUF |
 120                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 121                                            FLAGS_ASSOC_DATA_ST_SHIFT,
 122                                            &dd->flags);
 123         }
 124
 125         if (cryptlen) {
 126                 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
 127
 128                 ret = omap_crypto_align_sg(&tmp, cryptlen,
 129                                            AES_BLOCK_SIZE, &dd->in_sgl[nsg],
 130                                            OMAP_CRYPTO_COPY_DATA |
 131                                            OMAP_CRYPTO_ZERO_BUF |
 132                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 133                                            FLAGS_IN_DATA_ST_SHIFT,
 134                                            &dd->flags);
 135         }
 136
 137         dd->in_sg = dd->in_sgl;
 138         dd->total = cryptlen;
 139         dd->assoc_len = assoclen;
 140         dd->authsize = authlen;
 141
 142         dd->out_sg = req->dst;
 143         dd->orig_out = req->dst;
 144
 145         dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
 146
 147         flags = 0;
 148         if (req->src == req->dst || dd->out_sg == sg_arr)
 149                 flags |= OMAP_CRYPTO_FORCE_COPY;
 150
 151         ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
 152                                    AES_BLOCK_SIZE, &dd->out_sgl,
 153                                    flags,
 154                                    FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 155         if (ret)
 156                 return ret;
 157
 158         dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
 159         dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
 160
 161         return 0;
 162 }
 163
 164 static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
 165 {
 166         struct omap_aes_gcm_result *res = req->data;
 167
 168         if (err == -EINPROGRESS)
 169                 return;
 170
 171         res->err = err;
 172         complete(&res->completion);
 173 }
 174
 175 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
 176 {
 177         struct scatterlist iv_sg, tag_sg;
 178         struct skcipher_request *sk_req;
 179         struct omap_aes_gcm_result result;
 180         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 181         int ret = 0;
 182
 183         sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
 184         if (!sk_req) {
 185                 pr_err("skcipher: Failed to allocate request\n");
 186                 return -ENOMEM;
 187         }
 188
 189         init_completion(&result.completion);
 190
 191         sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
 192         sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
 193         skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 194                                       omap_aes_gcm_complete, &result);
 195         ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
 196         skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
 197                                    NULL);
 198         ret = crypto_skcipher_encrypt(sk_req);
 199         switch (ret) {
 200         case 0:
 201                 break;
 202         case -EINPROGRESS:
 203         case -EBUSY:
 204                 ret = wait_for_completion_interruptible(&result.completion);
 205                 if (!ret) {
 206                         ret = result.err;
 207                         if (!ret) {
 208                                 reinit_completion(&result.completion);
 209                                 break;
 210                         }
 211                 }
 212                 /* fall through */
 213         default:
 214                 pr_err("Encryption of IV failed for GCM mode\n");
 215                 break;
 216         }
 217
 218         skcipher_request_free(sk_req);
 219         return ret;
 220 }
 221
 222 void omap_aes_gcm_dma_out_callback(void *data)
 223 {
 224         struct omap_aes_dev *dd = data;
 225         struct omap_aes_reqctx *rctx;
 226         int i, val;
 227         u32 *auth_tag, tag[4];
 228
 229         if (!(dd->flags & FLAGS_ENCRYPT))
 230                 scatterwalk_map_and_copy(tag, dd->aead_req->src,
 231                                          dd->total + dd->aead_req->assoclen,
 232                                          dd->authsize, 0);
 233
 234         rctx = aead_request_ctx(dd->aead_req);
 235         auth_tag = (u32 *)rctx->auth_tag;
 236         for (i = 0; i < 4; i++) {
 237                 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
 238                 auth_tag[i] = val ^ auth_tag[i];
 239                 if (!(dd->flags & FLAGS_ENCRYPT))
 240                         auth_tag[i] = auth_tag[i] ^ tag[i];
 241         }
 242
 243         omap_aes_gcm_done_task(dd);
 244 }
 245
 246 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
 247                                      struct aead_request *req)
 248 {
 249         struct omap_aes_ctx *ctx;
 250         struct aead_request *backlog;
 251         struct omap_aes_reqctx *rctx;
 252         unsigned long flags;
 253         int err, ret = 0;
 254
 255         spin_lock_irqsave(&dd->lock, flags);
 256         if (req)
 257                 ret = aead_enqueue_request(&dd->aead_queue, req);
 258         if (dd->flags & FLAGS_BUSY) {
 259                 spin_unlock_irqrestore(&dd->lock, flags);
 260                 return ret;
 261         }
 262
 263         backlog = aead_get_backlog(&dd->aead_queue);
 264         req = aead_dequeue_request(&dd->aead_queue);
 265         if (req)
 266                 dd->flags |= FLAGS_BUSY;
 267         spin_unlock_irqrestore(&dd->lock, flags);
 268
 269         if (!req)
 270                 return ret;
 271
 272         if (backlog)
 273                 backlog->base.complete(&backlog->base, -EINPROGRESS);
 274
 275         ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 276         rctx = aead_request_ctx(req);
 277
 278         dd->ctx = ctx;
 279         rctx->dd = dd;
 280         dd->aead_req = req;
 281
 282         rctx->mode &= FLAGS_MODE_MASK;
 283         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 284
 285         err = omap_aes_gcm_copy_buffers(dd, req);
 286         if (err)
 287                 return err;
 288
 289         err = omap_aes_write_ctrl(dd);
 290         if (!err)
 291                 err = omap_aes_crypt_dma_start(dd);
 292
 293         if (err) {
 294                 omap_aes_gcm_finish_req(dd, err);
 295                 omap_aes_gcm_handle_queue(dd, NULL);
 296         }
 297
 298         return ret;
 299 }
 300
 301 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
 302 {
 303         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 304         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 305         unsigned int authlen = crypto_aead_authsize(aead);
 306         struct omap_aes_dev *dd;
 307         __be32 counter = cpu_to_be32(1);
 308         int err, assoclen;
 309
 310         memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
 311         memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
 312
 313         err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
 314         if (err)
 315                 return err;
 316
 317         if (mode & FLAGS_RFC4106_GCM)
 318                 assoclen = req->assoclen - 8;
 319         else
 320                 assoclen = req->assoclen;
 321         if (assoclen + req->cryptlen == 0) {
 322                 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
 323                                          1);
 324                 return 0;
 325         }
 326
 327         dd = omap_aes_find_dev(rctx);
 328         if (!dd)
 329                 return -ENODEV;
 330         rctx->mode = mode;
 331
 332         return omap_aes_gcm_handle_queue(dd, req);
 333 }
 334
 335 int omap_aes_gcm_encrypt(struct aead_request *req)
 336 {
 337         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 338
 339         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 340         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
 341 }
 342
 343 int omap_aes_gcm_decrypt(struct aead_request *req)
 344 {
 345         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 346
 347         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 348         return omap_aes_gcm_crypt(req, FLAGS_GCM);
 349 }
 350
 351 int omap_aes_4106gcm_encrypt(struct aead_request *req)
 352 {
 353         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 354         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 355
 356         memcpy(rctx->iv, ctx->nonce, 4);
 357         memcpy(rctx->iv + 4, req->iv, 8);
 358         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
 359                                   FLAGS_RFC4106_GCM);
 360 }
 361
 362 int omap_aes_4106gcm_decrypt(struct aead_request *req)
 363 {
 364         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 365         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 366
 367         memcpy(rctx->iv, ctx->nonce, 4);
 368         memcpy(rctx->iv + 4, req->iv, 8);
 369         return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
 370 }
 371
 372 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 373                         unsigned int keylen)
 374 {
 375         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 376
 377         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 378             keylen != AES_KEYSIZE_256)
 379                 return -EINVAL;
 380
 381         memcpy(ctx->key, key, keylen);
 382         ctx->keylen = keylen;
 383
 384         return 0;
 385 }
 386
 387 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 388                             unsigned int keylen)
 389 {
 390         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 391
 392         if (keylen < 4)
 393                 return -EINVAL;
 394
 395         keylen -= 4;
 396         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 397             keylen != AES_KEYSIZE_256)
 398                 return -EINVAL;
 399
 400         memcpy(ctx->key, key, keylen);
 401         memcpy(ctx->nonce, key + keylen, 4);
 402         ctx->keylen = keylen;
 403
 404         return 0;
 405 }