Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ad18cc9d TK |
2 | /* |
3 | * Cryptographic API. | |
4 | * | |
5 | * Support for OMAP AES GCM HW acceleration. | |
6 | * | |
7 | * Copyright (c) 2016 Texas Instruments Incorporated | |
ad18cc9d TK |
8 | */ |
9 | ||
03906fba HX |
10 | #include <crypto/aes.h> |
11 | #include <crypto/engine.h> | |
12 | #include <crypto/gcm.h> | |
13 | #include <crypto/internal/aead.h> | |
14 | #include <crypto/scatterwalk.h> | |
15 | #include <crypto/skcipher.h> | |
ad18cc9d | 16 | #include <linux/errno.h> |
ad18cc9d TK |
17 | #include <linux/dma-mapping.h> |
18 | #include <linux/dmaengine.h> | |
ad18cc9d | 19 | #include <linux/interrupt.h> |
03906fba HX |
20 | #include <linux/kernel.h> |
21 | #include <linux/omap-dma.h> | |
04a4616e | 22 | #include <linux/pm_runtime.h> |
03906fba HX |
23 | #include <linux/scatterlist.h> |
24 | #include <linux/string.h> | |
ad18cc9d TK |
25 | |
26 | #include "omap-crypto.h" | |
27 | #include "omap-aes.h" | |
28 | ||
29 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, | |
30 | struct aead_request *req); | |
31 | ||
32 | static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) | |
33 | { | |
34 | struct aead_request *req = dd->aead_req; | |
35 | ||
ad18cc9d TK |
36 | dd->in_sg = NULL; |
37 | dd->out_sg = NULL; | |
38 | ||
04a4616e TK |
39 | crypto_finalize_aead_request(dd->engine, req, ret); |
40 | ||
41 | pm_runtime_mark_last_busy(dd->dev); | |
42 | pm_runtime_put_autosuspend(dd->dev); | |
ad18cc9d TK |
43 | } |
44 | ||
45 | static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) | |
46 | { | |
47 | u8 *tag; | |
48 | int alen, clen, i, ret = 0, nsg; | |
49 | struct omap_aes_reqctx *rctx; | |
50 | ||
51 | alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); | |
52 | clen = ALIGN(dd->total, AES_BLOCK_SIZE); | |
53 | rctx = aead_request_ctx(dd->aead_req); | |
54 | ||
55 | nsg = !!(dd->assoc_len && dd->total); | |
56 | ||
57 | dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, | |
58 | DMA_FROM_DEVICE); | |
59 | dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); | |
60 | dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); | |
61 | omap_aes_crypt_dma_stop(dd); | |
62 | ||
63 | omap_crypto_cleanup(dd->out_sg, dd->orig_out, | |
64 | dd->aead_req->assoclen, dd->total, | |
65 | FLAGS_OUT_DATA_ST_SHIFT, dd->flags); | |
66 | ||
67 | if (dd->flags & FLAGS_ENCRYPT) | |
68 | scatterwalk_map_and_copy(rctx->auth_tag, | |
69 | dd->aead_req->dst, | |
70 | dd->total + dd->aead_req->assoclen, | |
71 | dd->authsize, 1); | |
72 | ||
73 | omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen, | |
74 | FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags); | |
75 | ||
76 | omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen, | |
77 | FLAGS_IN_DATA_ST_SHIFT, dd->flags); | |
78 | ||
79 | if (!(dd->flags & FLAGS_ENCRYPT)) { | |
80 | tag = (u8 *)rctx->auth_tag; | |
81 | for (i = 0; i < dd->authsize; i++) { | |
82 | if (tag[i]) { | |
ad18cc9d TK |
83 | ret = -EBADMSG; |
84 | } | |
85 | } | |
86 | } | |
87 | ||
88 | omap_aes_gcm_finish_req(dd, ret); | |
ad18cc9d TK |
89 | } |
90 | ||
91 | static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, | |
92 | struct aead_request *req) | |
93 | { | |
94 | int alen, clen, cryptlen, assoclen, ret; | |
95 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | |
96 | unsigned int authlen = crypto_aead_authsize(aead); | |
97 | struct scatterlist *tmp, sg_arr[2]; | |
98 | int nsg; | |
99 | u16 flags; | |
100 | ||
101 | assoclen = req->assoclen; | |
102 | cryptlen = req->cryptlen; | |
103 | ||
104 | if (dd->flags & FLAGS_RFC4106_GCM) | |
105 | assoclen -= 8; | |
106 | ||
107 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
108 | cryptlen -= authlen; | |
109 | ||
110 | alen = ALIGN(assoclen, AES_BLOCK_SIZE); | |
111 | clen = ALIGN(cryptlen, AES_BLOCK_SIZE); | |
112 | ||
113 | nsg = !!(assoclen && cryptlen); | |
114 | ||
115 | omap_aes_clear_copy_flags(dd); | |
116 | ||
117 | sg_init_table(dd->in_sgl, nsg + 1); | |
118 | if (assoclen) { | |
119 | tmp = req->src; | |
120 | ret = omap_crypto_align_sg(&tmp, assoclen, | |
121 | AES_BLOCK_SIZE, dd->in_sgl, | |
122 | OMAP_CRYPTO_COPY_DATA | | |
123 | OMAP_CRYPTO_ZERO_BUF | | |
124 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, | |
125 | FLAGS_ASSOC_DATA_ST_SHIFT, | |
126 | &dd->flags); | |
46d57443 AB |
127 | if (ret) |
128 | return ret; | |
ad18cc9d TK |
129 | } |
130 | ||
131 | if (cryptlen) { | |
132 | tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen); | |
133 | ||
04a4616e TK |
134 | if (nsg) |
135 | sg_unmark_end(dd->in_sgl); | |
136 | ||
ad18cc9d TK |
137 | ret = omap_crypto_align_sg(&tmp, cryptlen, |
138 | AES_BLOCK_SIZE, &dd->in_sgl[nsg], | |
139 | OMAP_CRYPTO_COPY_DATA | | |
140 | OMAP_CRYPTO_ZERO_BUF | | |
141 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, | |
142 | FLAGS_IN_DATA_ST_SHIFT, | |
143 | &dd->flags); | |
46d57443 AB |
144 | if (ret) |
145 | return ret; | |
ad18cc9d TK |
146 | } |
147 | ||
148 | dd->in_sg = dd->in_sgl; | |
149 | dd->total = cryptlen; | |
150 | dd->assoc_len = assoclen; | |
151 | dd->authsize = authlen; | |
152 | ||
153 | dd->out_sg = req->dst; | |
154 | dd->orig_out = req->dst; | |
155 | ||
04a4616e | 156 | dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen); |
ad18cc9d TK |
157 | |
158 | flags = 0; | |
159 | if (req->src == req->dst || dd->out_sg == sg_arr) | |
160 | flags |= OMAP_CRYPTO_FORCE_COPY; | |
161 | ||
a9befcf4 TK |
162 | if (cryptlen) { |
163 | ret = omap_crypto_align_sg(&dd->out_sg, cryptlen, | |
164 | AES_BLOCK_SIZE, &dd->out_sgl, | |
165 | flags, | |
166 | FLAGS_OUT_DATA_ST_SHIFT, &dd->flags); | |
167 | if (ret) | |
168 | return ret; | |
169 | } | |
ad18cc9d TK |
170 | |
171 | dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen); | |
172 | dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen); | |
173 | ||
174 | return 0; | |
175 | } | |
176 | ||
ad18cc9d TK |
177 | static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) |
178 | { | |
f0956d42 | 179 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
ad18cc9d | 180 | |
f0956d42 AB |
181 | aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv); |
182 | return 0; | |
ad18cc9d TK |
183 | } |
184 | ||
185 | void omap_aes_gcm_dma_out_callback(void *data) | |
186 | { | |
187 | struct omap_aes_dev *dd = data; | |
188 | struct omap_aes_reqctx *rctx; | |
189 | int i, val; | |
190 | u32 *auth_tag, tag[4]; | |
191 | ||
192 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
193 | scatterwalk_map_and_copy(tag, dd->aead_req->src, | |
194 | dd->total + dd->aead_req->assoclen, | |
195 | dd->authsize, 0); | |
196 | ||
197 | rctx = aead_request_ctx(dd->aead_req); | |
198 | auth_tag = (u32 *)rctx->auth_tag; | |
199 | for (i = 0; i < 4; i++) { | |
200 | val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); | |
201 | auth_tag[i] = val ^ auth_tag[i]; | |
202 | if (!(dd->flags & FLAGS_ENCRYPT)) | |
203 | auth_tag[i] = auth_tag[i] ^ tag[i]; | |
204 | } | |
205 | ||
206 | omap_aes_gcm_done_task(dd); | |
207 | } | |
208 | ||
209 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, | |
210 | struct aead_request *req) | |
211 | { | |
ad18cc9d | 212 | if (req) |
04a4616e | 213 | return crypto_transfer_aead_request_to_engine(dd->engine, req); |
ad18cc9d | 214 | |
04a4616e TK |
215 | return 0; |
216 | } | |
ad18cc9d | 217 | |
c752c013 HX |
218 | static int omap_aes_gcm_prepare_req(struct aead_request *req, |
219 | struct omap_aes_dev *dd) | |
04a4616e | 220 | { |
04a4616e | 221 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
04a4616e TK |
222 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
223 | int err; | |
ad18cc9d | 224 | |
ad18cc9d TK |
225 | dd->aead_req = req; |
226 | ||
227 | rctx->mode &= FLAGS_MODE_MASK; | |
228 | dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; | |
229 | ||
230 | err = omap_aes_gcm_copy_buffers(dd, req); | |
231 | if (err) | |
232 | return err; | |
233 | ||
04a4616e | 234 | dd->ctx = &ctx->octx; |
ad18cc9d | 235 | |
04a4616e | 236 | return omap_aes_write_ctrl(dd); |
ad18cc9d TK |
237 | } |
238 | ||
239 | static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) | |
240 | { | |
241 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
242 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | |
243 | unsigned int authlen = crypto_aead_authsize(aead); | |
244 | struct omap_aes_dev *dd; | |
245 | __be32 counter = cpu_to_be32(1); | |
246 | int err, assoclen; | |
247 | ||
248 | memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); | |
cb3f3817 | 249 | memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); |
ad18cc9d TK |
250 | |
251 | err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); | |
252 | if (err) | |
253 | return err; | |
254 | ||
255 | if (mode & FLAGS_RFC4106_GCM) | |
256 | assoclen = req->assoclen - 8; | |
257 | else | |
258 | assoclen = req->assoclen; | |
259 | if (assoclen + req->cryptlen == 0) { | |
260 | scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen, | |
261 | 1); | |
262 | return 0; | |
263 | } | |
264 | ||
265 | dd = omap_aes_find_dev(rctx); | |
266 | if (!dd) | |
267 | return -ENODEV; | |
268 | rctx->mode = mode; | |
269 | ||
270 | return omap_aes_gcm_handle_queue(dd, req); | |
271 | } | |
272 | ||
273 | int omap_aes_gcm_encrypt(struct aead_request *req) | |
274 | { | |
275 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
276 | ||
cb3f3817 | 277 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
ad18cc9d TK |
278 | return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); |
279 | } | |
280 | ||
281 | int omap_aes_gcm_decrypt(struct aead_request *req) | |
282 | { | |
283 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
284 | ||
cb3f3817 | 285 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
ad18cc9d TK |
286 | return omap_aes_gcm_crypt(req, FLAGS_GCM); |
287 | } | |
288 | ||
289 | int omap_aes_4106gcm_encrypt(struct aead_request *req) | |
290 | { | |
f0956d42 | 291 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
ad18cc9d TK |
292 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
293 | ||
f0956d42 | 294 | memcpy(rctx->iv, ctx->octx.nonce, 4); |
ad18cc9d | 295 | memcpy(rctx->iv + 4, req->iv, 8); |
b877ad1a AB |
296 | return crypto_ipsec_check_assoclen(req->assoclen) ?: |
297 | omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | | |
ad18cc9d TK |
298 | FLAGS_RFC4106_GCM); |
299 | } | |
300 | ||
301 | int omap_aes_4106gcm_decrypt(struct aead_request *req) | |
302 | { | |
f0956d42 | 303 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
ad18cc9d TK |
304 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
305 | ||
f0956d42 | 306 | memcpy(rctx->iv, ctx->octx.nonce, 4); |
ad18cc9d | 307 | memcpy(rctx->iv + 4, req->iv, 8); |
b877ad1a AB |
308 | return crypto_ipsec_check_assoclen(req->assoclen) ?: |
309 | omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); | |
ad18cc9d TK |
310 | } |
311 | ||
312 | int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, | |
313 | unsigned int keylen) | |
314 | { | |
f0956d42 AB |
315 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
316 | int ret; | |
ad18cc9d | 317 | |
f0956d42 AB |
318 | ret = aes_expandkey(&ctx->actx, key, keylen); |
319 | if (ret) | |
320 | return ret; | |
ad18cc9d | 321 | |
f0956d42 AB |
322 | memcpy(ctx->octx.key, key, keylen); |
323 | ctx->octx.keylen = keylen; | |
ad18cc9d TK |
324 | |
325 | return 0; | |
326 | } | |
327 | ||
328 | int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, | |
329 | unsigned int keylen) | |
330 | { | |
f0956d42 AB |
331 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
332 | int ret; | |
ad18cc9d TK |
333 | |
334 | if (keylen < 4) | |
335 | return -EINVAL; | |
ad18cc9d | 336 | keylen -= 4; |
ad18cc9d | 337 | |
f0956d42 AB |
338 | ret = aes_expandkey(&ctx->actx, key, keylen); |
339 | if (ret) | |
340 | return ret; | |
341 | ||
342 | memcpy(ctx->octx.key, key, keylen); | |
343 | memcpy(ctx->octx.nonce, key + keylen, 4); | |
344 | ctx->octx.keylen = keylen; | |
ad18cc9d TK |
345 | |
346 | return 0; | |
347 | } | |
12adf9d6 AB |
348 | |
349 | int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) | |
350 | { | |
351 | return crypto_gcm_check_authsize(authsize); | |
352 | } | |
353 | ||
354 | int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, | |
355 | unsigned int authsize) | |
356 | { | |
357 | return crypto_rfc4106_check_authsize(authsize); | |
358 | } | |
04a4616e | 359 | |
03906fba | 360 | int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) |
04a4616e TK |
361 | { |
362 | struct aead_request *req = container_of(areq, struct aead_request, | |
363 | base); | |
364 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); | |
365 | struct omap_aes_dev *dd = rctx->dd; | |
c752c013 | 366 | int ret; |
04a4616e TK |
367 | |
368 | if (!dd) | |
369 | return -ENODEV; | |
370 | ||
c752c013 HX |
371 | ret = omap_aes_gcm_prepare_req(req, dd); |
372 | if (ret) | |
373 | return ret; | |
374 | ||
04a4616e TK |
375 | if (dd->in_sg_len) |
376 | ret = omap_aes_crypt_dma_start(dd); | |
377 | else | |
378 | omap_aes_gcm_dma_out_callback(dd); | |
379 | ||
380 | return ret; | |
381 | } | |
382 | ||
383 | int omap_aes_gcm_cra_init(struct crypto_aead *tfm) | |
384 | { | |
04a4616e TK |
385 | crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx)); |
386 | ||
387 | return 0; | |
388 | } |