Commit | Line | Data |
---|---|---|
ad18cc9d TK |
1 | /* |
2 | * Cryptographic API. | |
3 | * | |
4 | * Support for OMAP AES GCM HW acceleration. | |
5 | * | |
6 | * Copyright (c) 2016 Texas Instruments Incorporated | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as published | |
10 | * by the Free Software Foundation. | |
11 | * | |
12 | */ | |
13 | ||
14 | #include <linux/errno.h> | |
15 | #include <linux/scatterlist.h> | |
16 | #include <linux/dma-mapping.h> | |
17 | #include <linux/dmaengine.h> | |
18 | #include <linux/omap-dma.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <crypto/aes.h> | |
cb3f3817 | 21 | #include <crypto/gcm.h> |
ad18cc9d TK |
22 | #include <crypto/scatterwalk.h> |
23 | #include <crypto/skcipher.h> | |
24 | #include <crypto/internal/aead.h> | |
25 | ||
26 | #include "omap-crypto.h" | |
27 | #include "omap-aes.h" | |
28 | ||
29 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, | |
30 | struct aead_request *req); | |
31 | ||
32 | static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) | |
33 | { | |
34 | struct aead_request *req = dd->aead_req; | |
35 | ||
36 | dd->flags &= ~FLAGS_BUSY; | |
37 | dd->in_sg = NULL; | |
38 | dd->out_sg = NULL; | |
39 | ||
40 | req->base.complete(&req->base, ret); | |
41 | } | |
42 | ||
43 | static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) | |
44 | { | |
45 | u8 *tag; | |
46 | int alen, clen, i, ret = 0, nsg; | |
47 | struct omap_aes_reqctx *rctx; | |
48 | ||
49 | alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); | |
50 | clen = ALIGN(dd->total, AES_BLOCK_SIZE); | |
51 | rctx = aead_request_ctx(dd->aead_req); | |
52 | ||
53 | nsg = !!(dd->assoc_len && dd->total); | |
54 | ||
55 | dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, | |
56 | DMA_FROM_DEVICE); | |
57 | dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); | |
58 | dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); | |
59 | omap_aes_crypt_dma_stop(dd); | |
60 | ||
61 | omap_crypto_cleanup(dd->out_sg, dd->orig_out, | |
62 | dd->aead_req->assoclen, dd->total, | |
63 | FLAGS_OUT_DATA_ST_SHIFT, dd->flags); | |
64 | ||
65 | if (dd->flags & FLAGS_ENCRYPT) | |
66 | scatterwalk_map_and_copy(rctx->auth_tag, | |
67 | dd->aead_req->dst, | |
68 | dd->total + dd->aead_req->assoclen, | |
69 | dd->authsize, 1); | |
70 | ||
71 | omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, | |
72 | FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); | |
73 | ||
74 | omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, | |
75 | FLAGS_IN_DATA_ST_SHIFT, dd->flags); | |
76 | ||
77 | if (!(dd->flags & FLAGS_ENCRYPT)) { | |
78 | tag = (u8 *)rctx->auth_tag; | |
79 | for (i = 0; i < dd->authsize; i++) { | |
80 | if (tag[i]) { | |
81 | dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); | |
82 | ret = -EBADMSG; | |
83 | } | |
84 | } | |
85 | } | |
86 | ||
87 | omap_aes_gcm_finish_req(dd, ret); | |
88 | omap_aes_gcm_handle_queue(dd, NULL); | |
89 | } | |
90 | ||
91 | static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, | |
92 | struct aead_request *req) | |
93 | { | |
94 | int alen, clen, cryptlen, assoclen, ret; | |
95 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | |
96 | unsigned int authlen = crypto_aead_authsize(aead); | |
97 | struct scatterlist *tmp, sg_arr[2]; | |
98 | int nsg; | |
99 | u16 flags; | |
100 | ||
101 | assoclen = req->assoclen; | |
102 | cryptlen = req->cryptlen; | |
103 | ||
104 | if (dd->flags & FLAGS_RFC4106_GCM) | |
105 | assoclen -= 8; | |
106 | ||
107 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
108 | cryptlen -= authlen; | |
109 | ||
110 | alen = ALIGN(assoclen, AES_BLOCK_SIZE); | |
111 | clen = ALIGN(cryptlen, AES_BLOCK_SIZE); | |
112 | ||
113 | nsg = !!(assoclen && cryptlen); | |
114 | ||
115 | omap_aes_clear_copy_flags(dd); | |
116 | ||
117 | sg_init_table(dd->in_sgl, nsg + 1); | |
118 | if (assoclen) { | |
119 | tmp = req->src; | |
120 | ret = omap_crypto_align_sg(&tmp, assoclen, | |
121 | AES_BLOCK_SIZE, dd->in_sgl, | |
122 | OMAP_CRYPTO_COPY_DATA | | |
123 | OMAP_CRYPTO_ZERO_BUF | | |
124 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, | |
125 | FLAGS_ASSOC_DATA_ST_SHIFT, | |
126 | &dd->flags); | |
127 | } | |
128 | ||
129 | if (cryptlen) { | |
130 | tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); | |
131 | ||
132 | ret = omap_crypto_align_sg(&tmp, cryptlen, | |
133 | AES_BLOCK_SIZE, &dd->in_sgl[nsg], | |
134 | OMAP_CRYPTO_COPY_DATA | | |
135 | OMAP_CRYPTO_ZERO_BUF | | |
136 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, | |
137 | FLAGS_IN_DATA_ST_SHIFT, | |
138 | &dd->flags); | |
139 | } | |
140 | ||
141 | dd->in_sg = dd->in_sgl; | |
142 | dd->total = cryptlen; | |
143 | dd->assoc_len = assoclen; | |
144 | dd->authsize = authlen; | |
145 | ||
146 | dd->out_sg = req->dst; | |
147 | dd->orig_out = req->dst; | |
148 | ||
149 | dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen); | |
150 | ||
151 | flags = 0; | |
152 | if (req->src == req->dst || dd->out_sg == sg_arr) | |
153 | flags |= OMAP_CRYPTO_FORCE_COPY; | |
154 | ||
155 | ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, | |
156 | AES_BLOCK_SIZE, &dd->out_sgl, | |
157 | flags, | |
158 | FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); | |
159 | if (ret) | |
160 | return ret; | |
161 | ||
162 | dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); | |
163 | dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); | |
164 | ||
165 | return 0; | |
166 | } | |
167 | ||
168 | static void omap_aes_gcm_complete(struct crypto_async_request *req, int err) | |
169 | { | |
170 | struct omap_aes_gcm_result *res = req->data; | |
171 | ||
172 | if (err == -EINPROGRESS) | |
173 | return; | |
174 | ||
175 | res->err = err; | |
176 | complete(&res->completion); | |
177 | } | |
178 | ||
179 | static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) | |
180 | { | |
181 | struct scatterlist iv_sg, tag_sg; | |
182 | struct skcipher_request *sk_req; | |
183 | struct omap_aes_gcm_result result; | |
184 | struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
185 | int ret = 0; | |
186 | ||
187 | sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL); | |
188 | if (!sk_req) { | |
189 | pr_err("skcipher: Failed to allocate request\n"); | |
2fd23f2b | 190 | return -ENOMEM; |
ad18cc9d TK |
191 | } |
192 | ||
193 | init_completion(&result.completion); | |
194 | ||
195 | sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE); | |
196 | sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE); | |
197 | skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG, | |
198 | omap_aes_gcm_complete, &result); | |
199 | ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen); | |
200 | skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE, | |
201 | NULL); | |
202 | ret = crypto_skcipher_encrypt(sk_req); | |
203 | switch (ret) { | |
204 | case 0: | |
205 | break; | |
206 | case -EINPROGRESS: | |
207 | case -EBUSY: | |
208 | ret = wait_for_completion_interruptible(&result.completion); | |
209 | if (!ret) { | |
210 | ret = result.err; | |
211 | if (!ret) { | |
212 | reinit_completion(&result.completion); | |
213 | break; | |
214 | } | |
215 | } | |
216 | /* fall through */ | |
217 | default: | |
bfbd6254 | 218 | pr_err("Encryption of IV failed for GCM mode\n"); |
ad18cc9d TK |
219 | break; |
220 | } | |
221 | ||
222 | skcipher_request_free(sk_req); | |
223 | return ret; | |
224 | } | |
225 | ||
226 | void omap_aes_gcm_dma_out_callback(void *data) | |
227 | { | |
228 | struct omap_aes_dev *dd = data; | |
229 | struct omap_aes_reqctx *rctx; | |
230 | int i, val; | |
231 | u32 *auth_tag, tag[4]; | |
232 | ||
233 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
234 | scatterwalk_map_and_copy(tag, dd->aead_req->src, | |
235 | dd->total + dd->aead_req->assoclen, | |
236 | dd->authsize, 0); | |
237 | ||
238 | rctx = aead_request_ctx(dd->aead_req); | |
239 | auth_tag = (u32 *)rctx->auth_tag; | |
240 | for (i = 0; i < 4; i++) { | |
241 | val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); | |
242 | auth_tag[i] = val ^ auth_tag[i]; | |
243 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
244 | auth_tag[i] = auth_tag[i] ^ tag[i]; | |
245 | } | |
246 | ||
247 | omap_aes_gcm_done_task(dd); | |
248 | } | |
249 | ||
250 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, | |
251 | struct aead_request *req) | |
252 | { | |
253 | struct omap_aes_ctx *ctx; | |
254 | struct aead_request *backlog; | |
255 | struct omap_aes_reqctx *rctx; | |
256 | unsigned long flags; | |
257 | int err, ret = 0; | |
258 | ||
259 | spin_lock_irqsave(&dd->lock, flags); | |
260 | if (req) | |
261 | ret = aead_enqueue_request(&dd->aead_queue, req); | |
262 | if (dd->flags & FLAGS_BUSY) { | |
263 | spin_unlock_irqrestore(&dd->lock, flags); | |
264 | return ret; | |
265 | } | |
266 | ||
267 | backlog = aead_get_backlog(&dd->aead_queue); | |
268 | req = aead_dequeue_request(&dd->aead_queue); | |
269 | if (req) | |
270 | dd->flags |= FLAGS_BUSY; | |
271 | spin_unlock_irqrestore(&dd->lock, flags); | |
272 | ||
273 | if (!req) | |
274 | return ret; | |
275 | ||
276 | if (backlog) | |
277 | backlog->base.complete(&backlog->base, -EINPROGRESS); | |
278 | ||
279 | ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
280 | rctx = aead_request_ctx(req); | |
281 | ||
282 | dd->ctx = ctx; | |
283 | rctx->dd = dd; | |
284 | dd->aead_req = req; | |
285 | ||
286 | rctx->mode &= FLAGS_MODE_MASK; | |
287 | dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; | |
288 | ||
289 | err = omap_aes_gcm_copy_buffers(dd, req); | |
290 | if (err) | |
291 | return err; | |
292 | ||
293 | err = omap_aes_write_ctrl(dd); | |
294 | if (!err) | |
295 | err = omap_aes_crypt_dma_start(dd); | |
296 | ||
297 | if (err) { | |
298 | omap_aes_gcm_finish_req(dd, err); | |
299 | omap_aes_gcm_handle_queue(dd, NULL); | |
300 | } | |
301 | ||
302 | return ret; | |
303 | } | |
304 | ||
305 | static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) | |
306 | { | |
307 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
308 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | |
309 | unsigned int authlen = crypto_aead_authsize(aead); | |
310 | struct omap_aes_dev *dd; | |
311 | __be32 counter = cpu_to_be32(1); | |
312 | int err, assoclen; | |
313 | ||
314 | memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); | |
cb3f3817 | 315 | memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); |
ad18cc9d TK |
316 | |
317 | err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); | |
318 | if (err) | |
319 | return err; | |
320 | ||
321 | if (mode & FLAGS_RFC4106_GCM) | |
322 | assoclen = req->assoclen - 8; | |
323 | else | |
324 | assoclen = req->assoclen; | |
325 | if (assoclen + req->cryptlen == 0) { | |
326 | scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, | |
327 | 1); | |
328 | return 0; | |
329 | } | |
330 | ||
331 | dd = omap_aes_find_dev(rctx); | |
332 | if (!dd) | |
333 | return -ENODEV; | |
334 | rctx->mode = mode; | |
335 | ||
336 | return omap_aes_gcm_handle_queue(dd, req); | |
337 | } | |
338 | ||
339 | int omap_aes_gcm_encrypt(struct aead_request *req) | |
340 | { | |
341 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
342 | ||
cb3f3817 | 343 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
ad18cc9d TK |
344 | return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); |
345 | } | |
346 | ||
347 | int omap_aes_gcm_decrypt(struct aead_request *req) | |
348 | { | |
349 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
350 | ||
cb3f3817 | 351 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
ad18cc9d TK |
352 | return omap_aes_gcm_crypt(req, FLAGS_GCM); |
353 | } | |
354 | ||
355 | int omap_aes_4106gcm_encrypt(struct aead_request *req) | |
356 | { | |
357 | struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
358 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
359 | ||
360 | memcpy(rctx->iv, ctx->nonce, 4); | |
361 | memcpy(rctx->iv + 4, req->iv, 8); | |
362 | return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | | |
363 | FLAGS_RFC4106_GCM); | |
364 | } | |
365 | ||
366 | int omap_aes_4106gcm_decrypt(struct aead_request *req) | |
367 | { | |
368 | struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
369 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
370 | ||
371 | memcpy(rctx->iv, ctx->nonce, 4); | |
372 | memcpy(rctx->iv + 4, req->iv, 8); | |
373 | return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); | |
374 | } | |
375 | ||
376 | int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, | |
377 | unsigned int keylen) | |
378 | { | |
379 | struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); | |
380 | ||
381 | if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && | |
382 | keylen != AES_KEYSIZE_256) | |
383 | return -EINVAL; | |
384 | ||
385 | memcpy(ctx->key, key, keylen); | |
386 | ctx->keylen = keylen; | |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
391 | int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, | |
392 | unsigned int keylen) | |
393 | { | |
394 | struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm); | |
395 | ||
396 | if (keylen < 4) | |
397 | return -EINVAL; | |
398 | ||
399 | keylen -= 4; | |
400 | if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && | |
401 | keylen != AES_KEYSIZE_256) | |
402 | return -EINVAL; | |
403 | ||
404 | memcpy(ctx->key, key, keylen); | |
405 | memcpy(ctx->nonce, key + keylen, 4); | |
406 | ctx->keylen = keylen; | |
407 | ||
408 | return 0; | |
409 | } |