Commit | Line | Data |
---|---|---|
f63601fd BB |
1 | /* |
2 | * Hash algorithms supported by the CESA: MD5, SHA1 and SHA256. | |
3 | * | |
4 | * Author: Boris Brezillon <boris.brezillon@free-electrons.com> | |
5 | * Author: Arnaud Ebalard <arno@natisbad.org> | |
6 | * | |
7 | * This work is based on an initial version written by | |
8 | * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > | |
9 | * | |
10 | * This program is free software; you can redistribute it and/or modify it | |
11 | * under the terms of the GNU General Public License version 2 as published | |
12 | * by the Free Software Foundation. | |
13 | */ | |
14 | ||
7aeef693 | 15 | #include <crypto/md5.h> |
f63601fd BB |
16 | #include <crypto/sha.h> |
17 | ||
18 | #include "cesa.h" | |
19 | ||
db509a45 BB |
20 | struct mv_cesa_ahash_dma_iter { |
21 | struct mv_cesa_dma_iter base; | |
22 | struct mv_cesa_sg_dma_iter src; | |
23 | }; | |
24 | ||
25 | static inline void | |
26 | mv_cesa_ahash_req_iter_init(struct mv_cesa_ahash_dma_iter *iter, | |
27 | struct ahash_request *req) | |
28 | { | |
29 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
bd274b10 | 30 | unsigned int len = req->nbytes + creq->cache_ptr; |
db509a45 BB |
31 | |
32 | if (!creq->last_req) | |
bd274b10 | 33 | len &= ~CESA_HASH_BLOCK_SIZE_MSK; |
db509a45 BB |
34 | |
35 | mv_cesa_req_dma_iter_init(&iter->base, len); | |
36 | mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); | |
37 | iter->src.op_offset = creq->cache_ptr; | |
38 | } | |
39 | ||
40 | static inline bool | |
41 | mv_cesa_ahash_req_iter_next_op(struct mv_cesa_ahash_dma_iter *iter) | |
42 | { | |
43 | iter->src.op_offset = 0; | |
44 | ||
45 | return mv_cesa_req_dma_iter_next_op(&iter->base); | |
46 | } | |
47 | ||
48 | static inline int mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_req *creq, | |
49 | gfp_t flags) | |
50 | { | |
51 | struct mv_cesa_ahash_dma_req *dreq = &creq->req.dma; | |
52 | ||
53 | creq->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags, | |
54 | &dreq->cache_dma); | |
55 | if (!creq->cache) | |
56 | return -ENOMEM; | |
57 | ||
58 | return 0; | |
59 | } | |
60 | ||
f63601fd BB |
61 | static inline int mv_cesa_ahash_std_alloc_cache(struct mv_cesa_ahash_req *creq, |
62 | gfp_t flags) | |
63 | { | |
64 | creq->cache = kzalloc(CESA_MAX_HASH_BLOCK_SIZE, flags); | |
65 | if (!creq->cache) | |
66 | return -ENOMEM; | |
67 | ||
68 | return 0; | |
69 | } | |
70 | ||
71 | static int mv_cesa_ahash_alloc_cache(struct ahash_request *req) | |
72 | { | |
73 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
74 | gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? | |
75 | GFP_KERNEL : GFP_ATOMIC; | |
db509a45 | 76 | int ret; |
f63601fd BB |
77 | |
78 | if (creq->cache) | |
79 | return 0; | |
80 | ||
db509a45 BB |
81 | if (creq->req.base.type == CESA_DMA_REQ) |
82 | ret = mv_cesa_ahash_dma_alloc_cache(creq, flags); | |
83 | else | |
84 | ret = mv_cesa_ahash_std_alloc_cache(creq, flags); | |
85 | ||
86 | return ret; | |
87 | } | |
88 | ||
89 | static inline void mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_req *creq) | |
90 | { | |
91 | dma_pool_free(cesa_dev->dma->cache_pool, creq->cache, | |
92 | creq->req.dma.cache_dma); | |
f63601fd BB |
93 | } |
94 | ||
95 | static inline void mv_cesa_ahash_std_free_cache(struct mv_cesa_ahash_req *creq) | |
96 | { | |
97 | kfree(creq->cache); | |
98 | } | |
99 | ||
100 | static void mv_cesa_ahash_free_cache(struct mv_cesa_ahash_req *creq) | |
101 | { | |
102 | if (!creq->cache) | |
103 | return; | |
104 | ||
db509a45 BB |
105 | if (creq->req.base.type == CESA_DMA_REQ) |
106 | mv_cesa_ahash_dma_free_cache(creq); | |
107 | else | |
108 | mv_cesa_ahash_std_free_cache(creq); | |
f63601fd BB |
109 | |
110 | creq->cache = NULL; | |
111 | } | |
112 | ||
db509a45 BB |
113 | static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req, |
114 | gfp_t flags) | |
115 | { | |
116 | if (req->padding) | |
117 | return 0; | |
118 | ||
119 | req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags, | |
120 | &req->padding_dma); | |
121 | if (!req->padding) | |
122 | return -ENOMEM; | |
123 | ||
124 | return 0; | |
125 | } | |
126 | ||
127 | static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req) | |
128 | { | |
129 | if (!req->padding) | |
130 | return; | |
131 | ||
132 | dma_pool_free(cesa_dev->dma->padding_pool, req->padding, | |
133 | req->padding_dma); | |
134 | req->padding = NULL; | |
135 | } | |
136 | ||
137 | static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req) | |
138 | { | |
139 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
140 | ||
141 | mv_cesa_ahash_dma_free_padding(&creq->req.dma); | |
142 | } | |
143 | ||
144 | static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) | |
145 | { | |
146 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
147 | ||
148 | dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); | |
149 | mv_cesa_dma_cleanup(&creq->req.dma.base); | |
150 | } | |
151 | ||
152 | static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) | |
153 | { | |
154 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
155 | ||
156 | if (creq->req.base.type == CESA_DMA_REQ) | |
157 | mv_cesa_ahash_dma_cleanup(req); | |
158 | } | |
159 | ||
f63601fd BB |
160 | static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) |
161 | { | |
162 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
163 | ||
164 | mv_cesa_ahash_free_cache(creq); | |
db509a45 BB |
165 | |
166 | if (creq->req.base.type == CESA_DMA_REQ) | |
167 | mv_cesa_ahash_dma_last_cleanup(req); | |
f63601fd BB |
168 | } |
169 | ||
170 | static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq) | |
171 | { | |
172 | unsigned int index, padlen; | |
173 | ||
174 | index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; | |
175 | padlen = (index < 56) ? (56 - index) : (64 + 56 - index); | |
176 | ||
177 | return padlen; | |
178 | } | |
179 | ||
180 | static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) | |
181 | { | |
f63601fd BB |
182 | unsigned int index, padlen; |
183 | ||
184 | buf[0] = 0x80; | |
185 | /* Pad out to 56 mod 64 */ | |
186 | index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; | |
187 | padlen = mv_cesa_ahash_pad_len(creq); | |
188 | memset(buf + 1, 0, padlen - 1); | |
51954a96 RK |
189 | |
190 | if (creq->algo_le) { | |
191 | __le64 bits = cpu_to_le64(creq->len << 3); | |
192 | memcpy(buf + padlen, &bits, sizeof(bits)); | |
193 | } else { | |
194 | __be64 bits = cpu_to_be64(creq->len << 3); | |
195 | memcpy(buf + padlen, &bits, sizeof(bits)); | |
196 | } | |
f63601fd BB |
197 | |
198 | return padlen + 8; | |
199 | } | |
200 | ||
201 | static void mv_cesa_ahash_std_step(struct ahash_request *req) | |
202 | { | |
203 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
204 | struct mv_cesa_ahash_std_req *sreq = &creq->req.std; | |
205 | struct mv_cesa_engine *engine = sreq->base.engine; | |
206 | struct mv_cesa_op_ctx *op; | |
207 | unsigned int new_cache_ptr = 0; | |
208 | u32 frag_mode; | |
209 | size_t len; | |
210 | ||
211 | if (creq->cache_ptr) | |
0f3304dc RK |
212 | memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, |
213 | creq->cache, creq->cache_ptr); | |
f63601fd BB |
214 | |
215 | len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, | |
216 | CESA_SA_SRAM_PAYLOAD_SIZE); | |
217 | ||
218 | if (!creq->last_req) { | |
219 | new_cache_ptr = len & CESA_HASH_BLOCK_SIZE_MSK; | |
220 | len &= ~CESA_HASH_BLOCK_SIZE_MSK; | |
221 | } | |
222 | ||
223 | if (len - creq->cache_ptr) | |
224 | sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents, | |
225 | engine->sram + | |
226 | CESA_SA_DATA_SRAM_OFFSET + | |
227 | creq->cache_ptr, | |
228 | len - creq->cache_ptr, | |
229 | sreq->offset); | |
230 | ||
231 | op = &creq->op_tmpl; | |
232 | ||
233 | frag_mode = mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK; | |
234 | ||
235 | if (creq->last_req && sreq->offset == req->nbytes && | |
236 | creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { | |
237 | if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) | |
238 | frag_mode = CESA_SA_DESC_CFG_NOT_FRAG; | |
239 | else if (frag_mode == CESA_SA_DESC_CFG_MID_FRAG) | |
240 | frag_mode = CESA_SA_DESC_CFG_LAST_FRAG; | |
241 | } | |
242 | ||
243 | if (frag_mode == CESA_SA_DESC_CFG_NOT_FRAG || | |
244 | frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) { | |
245 | if (len && | |
246 | creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { | |
247 | mv_cesa_set_mac_op_total_len(op, creq->len); | |
248 | } else { | |
249 | int trailerlen = mv_cesa_ahash_pad_len(creq) + 8; | |
250 | ||
251 | if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { | |
252 | len &= CESA_HASH_BLOCK_SIZE_MSK; | |
253 | new_cache_ptr = 64 - trailerlen; | |
0f3304dc RK |
254 | memcpy_fromio(creq->cache, |
255 | engine->sram + | |
256 | CESA_SA_DATA_SRAM_OFFSET + len, | |
257 | new_cache_ptr); | |
f63601fd BB |
258 | } else { |
259 | len += mv_cesa_ahash_pad_req(creq, | |
260 | engine->sram + len + | |
261 | CESA_SA_DATA_SRAM_OFFSET); | |
262 | } | |
263 | ||
264 | if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) | |
265 | frag_mode = CESA_SA_DESC_CFG_MID_FRAG; | |
266 | else | |
267 | frag_mode = CESA_SA_DESC_CFG_FIRST_FRAG; | |
268 | } | |
269 | } | |
270 | ||
271 | mv_cesa_set_mac_op_frag_len(op, len); | |
272 | mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); | |
273 | ||
274 | /* FIXME: only update enc_len field */ | |
0f3304dc | 275 | memcpy_toio(engine->sram, op, sizeof(*op)); |
f63601fd BB |
276 | |
277 | if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) | |
278 | mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, | |
279 | CESA_SA_DESC_CFG_FRAG_MSK); | |
280 | ||
281 | creq->cache_ptr = new_cache_ptr; | |
282 | ||
283 | mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); | |
b1508561 | 284 | writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); |
f63601fd BB |
285 | writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); |
286 | } | |
287 | ||
288 | static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status) | |
289 | { | |
290 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
291 | struct mv_cesa_ahash_std_req *sreq = &creq->req.std; | |
292 | ||
293 | if (sreq->offset < (req->nbytes - creq->cache_ptr)) | |
294 | return -EINPROGRESS; | |
295 | ||
296 | return 0; | |
297 | } | |
298 | ||
db509a45 BB |
299 | static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req) |
300 | { | |
301 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
302 | struct mv_cesa_tdma_req *dreq = &creq->req.dma.base; | |
303 | ||
304 | mv_cesa_dma_prepare(dreq, dreq->base.engine); | |
305 | } | |
306 | ||
f63601fd BB |
307 | static void mv_cesa_ahash_std_prepare(struct ahash_request *req) |
308 | { | |
309 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
310 | struct mv_cesa_ahash_std_req *sreq = &creq->req.std; | |
311 | struct mv_cesa_engine *engine = sreq->base.engine; | |
312 | ||
313 | sreq->offset = 0; | |
314 | mv_cesa_adjust_op(engine, &creq->op_tmpl); | |
0f3304dc | 315 | memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); |
f63601fd BB |
316 | } |
317 | ||
318 | static void mv_cesa_ahash_step(struct crypto_async_request *req) | |
319 | { | |
320 | struct ahash_request *ahashreq = ahash_request_cast(req); | |
db509a45 | 321 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); |
f63601fd | 322 | |
db509a45 BB |
323 | if (creq->req.base.type == CESA_DMA_REQ) |
324 | mv_cesa_dma_step(&creq->req.dma.base); | |
325 | else | |
326 | mv_cesa_ahash_std_step(ahashreq); | |
f63601fd BB |
327 | } |
328 | ||
329 | static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) | |
330 | { | |
331 | struct ahash_request *ahashreq = ahash_request_cast(req); | |
332 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); | |
333 | struct mv_cesa_engine *engine = creq->req.base.engine; | |
334 | unsigned int digsize; | |
335 | int ret, i; | |
336 | ||
db509a45 BB |
337 | if (creq->req.base.type == CESA_DMA_REQ) |
338 | ret = mv_cesa_dma_process(&creq->req.dma.base, status); | |
339 | else | |
340 | ret = mv_cesa_ahash_std_process(ahashreq, status); | |
341 | ||
f63601fd BB |
342 | if (ret == -EINPROGRESS) |
343 | return ret; | |
344 | ||
345 | digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); | |
346 | for (i = 0; i < digsize / 4; i++) | |
b1508561 | 347 | creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i)); |
f63601fd BB |
348 | |
349 | if (creq->cache_ptr) | |
350 | sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, | |
351 | creq->cache, | |
352 | creq->cache_ptr, | |
353 | ahashreq->nbytes - creq->cache_ptr); | |
354 | ||
355 | if (creq->last_req) { | |
4c2b130c RK |
356 | /* |
357 | * Hardware's MD5 digest is in little endian format, but | |
358 | * SHA in big endian format | |
359 | */ | |
a9eb678f | 360 | if (creq->algo_le) { |
4c2b130c RK |
361 | __le32 *result = (void *)ahashreq->result; |
362 | ||
363 | for (i = 0; i < digsize / 4; i++) | |
364 | result[i] = cpu_to_le32(creq->state[i]); | |
365 | } else { | |
366 | __be32 *result = (void *)ahashreq->result; | |
f63601fd | 367 | |
4c2b130c RK |
368 | for (i = 0; i < digsize / 4; i++) |
369 | result[i] = cpu_to_be32(creq->state[i]); | |
370 | } | |
f63601fd BB |
371 | } |
372 | ||
373 | return ret; | |
374 | } | |
375 | ||
376 | static void mv_cesa_ahash_prepare(struct crypto_async_request *req, | |
377 | struct mv_cesa_engine *engine) | |
378 | { | |
379 | struct ahash_request *ahashreq = ahash_request_cast(req); | |
380 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); | |
381 | unsigned int digsize; | |
382 | int i; | |
383 | ||
384 | creq->req.base.engine = engine; | |
385 | ||
db509a45 BB |
386 | if (creq->req.base.type == CESA_DMA_REQ) |
387 | mv_cesa_ahash_dma_prepare(ahashreq); | |
388 | else | |
389 | mv_cesa_ahash_std_prepare(ahashreq); | |
f63601fd BB |
390 | |
391 | digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); | |
392 | for (i = 0; i < digsize / 4; i++) | |
b1508561 | 393 | writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); |
f63601fd BB |
394 | } |
395 | ||
396 | static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) | |
397 | { | |
398 | struct ahash_request *ahashreq = ahash_request_cast(req); | |
399 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); | |
400 | ||
401 | if (creq->last_req) | |
402 | mv_cesa_ahash_last_cleanup(ahashreq); | |
db509a45 BB |
403 | |
404 | mv_cesa_ahash_cleanup(ahashreq); | |
f63601fd BB |
405 | } |
406 | ||
407 | static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { | |
408 | .step = mv_cesa_ahash_step, | |
409 | .process = mv_cesa_ahash_process, | |
410 | .prepare = mv_cesa_ahash_prepare, | |
411 | .cleanup = mv_cesa_ahash_req_cleanup, | |
412 | }; | |
413 | ||
414 | static int mv_cesa_ahash_init(struct ahash_request *req, | |
a9eb678f | 415 | struct mv_cesa_op_ctx *tmpl, bool algo_le) |
f63601fd BB |
416 | { |
417 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
418 | ||
419 | memset(creq, 0, sizeof(*creq)); | |
420 | mv_cesa_update_op_cfg(tmpl, | |
421 | CESA_SA_DESC_CFG_OP_MAC_ONLY | | |
422 | CESA_SA_DESC_CFG_FIRST_FRAG, | |
423 | CESA_SA_DESC_CFG_OP_MSK | | |
424 | CESA_SA_DESC_CFG_FRAG_MSK); | |
425 | mv_cesa_set_mac_op_total_len(tmpl, 0); | |
426 | mv_cesa_set_mac_op_frag_len(tmpl, 0); | |
427 | creq->op_tmpl = *tmpl; | |
428 | creq->len = 0; | |
a9eb678f | 429 | creq->algo_le = algo_le; |
f63601fd BB |
430 | |
431 | return 0; | |
432 | } | |
433 | ||
434 | static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm) | |
435 | { | |
436 | struct mv_cesa_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
437 | ||
438 | ctx->base.ops = &mv_cesa_ahash_req_ops; | |
439 | ||
440 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), | |
441 | sizeof(struct mv_cesa_ahash_req)); | |
442 | return 0; | |
443 | } | |
444 | ||
445 | static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached) | |
446 | { | |
447 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
448 | int ret; | |
449 | ||
450 | if (((creq->cache_ptr + req->nbytes) & CESA_HASH_BLOCK_SIZE_MSK) && | |
451 | !creq->last_req) { | |
452 | ret = mv_cesa_ahash_alloc_cache(req); | |
453 | if (ret) | |
454 | return ret; | |
455 | } | |
456 | ||
457 | if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) { | |
458 | *cached = true; | |
459 | ||
460 | if (!req->nbytes) | |
461 | return 0; | |
462 | ||
463 | sg_pcopy_to_buffer(req->src, creq->src_nents, | |
464 | creq->cache + creq->cache_ptr, | |
465 | req->nbytes, 0); | |
466 | ||
467 | creq->cache_ptr += req->nbytes; | |
468 | } | |
469 | ||
470 | return 0; | |
471 | } | |
472 | ||
db509a45 | 473 | static struct mv_cesa_op_ctx * |
96212886 RK |
474 | mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, |
475 | struct mv_cesa_op_ctx *tmpl, unsigned int frag_len, | |
476 | gfp_t flags) | |
db509a45 | 477 | { |
96212886 | 478 | struct mv_cesa_op_ctx *op; |
db509a45 BB |
479 | int ret; |
480 | ||
96212886 RK |
481 | op = mv_cesa_dma_add_op(chain, tmpl, false, flags); |
482 | if (IS_ERR(op)) | |
483 | return op; | |
db509a45 | 484 | |
96212886 RK |
485 | /* Set the operation block fragment length. */ |
486 | mv_cesa_set_mac_op_frag_len(op, frag_len); | |
487 | ||
488 | /* Append dummy desc to launch operation */ | |
489 | ret = mv_cesa_dma_add_dummy_launch(chain, flags); | |
db509a45 BB |
490 | if (ret) |
491 | return ERR_PTR(ret); | |
492 | ||
2f396a91 RK |
493 | if (mv_cesa_mac_op_is_first_frag(tmpl)) |
494 | mv_cesa_update_op_cfg(tmpl, | |
495 | CESA_SA_DESC_CFG_MID_FRAG, | |
496 | CESA_SA_DESC_CFG_FRAG_MSK); | |
db509a45 BB |
497 | |
498 | return op; | |
499 | } | |
500 | ||
0971d09a | 501 | static int |
db509a45 BB |
502 | mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, |
503 | struct mv_cesa_ahash_dma_iter *dma_iter, | |
504 | struct mv_cesa_ahash_req *creq, | |
505 | gfp_t flags) | |
db509a45 | 506 | { |
db509a45 | 507 | struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; |
db509a45 | 508 | |
db509a45 | 509 | if (!creq->cache_ptr) |
0971d09a | 510 | return 0; |
db509a45 | 511 | |
0971d09a RK |
512 | return mv_cesa_dma_add_data_transfer(chain, |
513 | CESA_SA_DATA_SRAM_OFFSET, | |
514 | ahashdreq->cache_dma, | |
515 | creq->cache_ptr, | |
516 | CESA_TDMA_DST_IN_SRAM, | |
517 | flags); | |
db509a45 BB |
518 | } |
519 | ||
520 | static struct mv_cesa_op_ctx * | |
521 | mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, | |
522 | struct mv_cesa_ahash_dma_iter *dma_iter, | |
523 | struct mv_cesa_ahash_req *creq, | |
58953e15 | 524 | unsigned int frag_len, gfp_t flags) |
db509a45 BB |
525 | { |
526 | struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; | |
527 | unsigned int len, trailerlen, padoff = 0; | |
58953e15 | 528 | struct mv_cesa_op_ctx *op; |
db509a45 BB |
529 | int ret; |
530 | ||
aee84a7e RK |
531 | /* |
532 | * If the transfer is smaller than our maximum length, and we have | |
533 | * some data outstanding, we can ask the engine to finish the hash. | |
534 | */ | |
535 | if (creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX && frag_len) { | |
58953e15 RK |
536 | op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, |
537 | flags); | |
538 | if (IS_ERR(op)) | |
539 | return op; | |
db509a45 | 540 | |
aee84a7e RK |
541 | mv_cesa_set_mac_op_total_len(op, creq->len); |
542 | mv_cesa_update_op_cfg(op, mv_cesa_mac_op_is_first_frag(op) ? | |
543 | CESA_SA_DESC_CFG_NOT_FRAG : | |
544 | CESA_SA_DESC_CFG_LAST_FRAG, | |
545 | CESA_SA_DESC_CFG_FRAG_MSK); | |
db509a45 BB |
546 | |
547 | return op; | |
548 | } | |
549 | ||
aee84a7e RK |
550 | /* |
551 | * The request is longer than the engine can handle, or we have | |
552 | * no data outstanding. Manually generate the padding, adding it | |
553 | * as a "mid" fragment. | |
554 | */ | |
db509a45 BB |
555 | ret = mv_cesa_ahash_dma_alloc_padding(ahashdreq, flags); |
556 | if (ret) | |
557 | return ERR_PTR(ret); | |
558 | ||
559 | trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding); | |
560 | ||
ab270e70 RK |
561 | len = min(CESA_SA_SRAM_PAYLOAD_SIZE - frag_len, trailerlen); |
562 | if (len) { | |
563 | ret = mv_cesa_dma_add_data_transfer(chain, | |
db509a45 | 564 | CESA_SA_DATA_SRAM_OFFSET + |
ab270e70 | 565 | frag_len, |
db509a45 BB |
566 | ahashdreq->padding_dma, |
567 | len, CESA_TDMA_DST_IN_SRAM, | |
568 | flags); | |
ab270e70 RK |
569 | if (ret) |
570 | return ERR_PTR(ret); | |
db509a45 | 571 | |
ab270e70 RK |
572 | op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len + len, |
573 | flags); | |
574 | if (IS_ERR(op)) | |
575 | return op; | |
db509a45 | 576 | |
ab270e70 RK |
577 | if (len == trailerlen) |
578 | return op; | |
db509a45 | 579 | |
ab270e70 RK |
580 | padoff += len; |
581 | } | |
db509a45 BB |
582 | |
583 | ret = mv_cesa_dma_add_data_transfer(chain, | |
584 | CESA_SA_DATA_SRAM_OFFSET, | |
585 | ahashdreq->padding_dma + | |
586 | padoff, | |
587 | trailerlen - padoff, | |
588 | CESA_TDMA_DST_IN_SRAM, | |
589 | flags); | |
590 | if (ret) | |
591 | return ERR_PTR(ret); | |
592 | ||
96212886 RK |
593 | return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, trailerlen - padoff, |
594 | flags); | |
db509a45 BB |
595 | } |
596 | ||
597 | static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) | |
598 | { | |
599 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
600 | gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? | |
601 | GFP_KERNEL : GFP_ATOMIC; | |
602 | struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; | |
603 | struct mv_cesa_tdma_req *dreq = &ahashdreq->base; | |
db509a45 BB |
604 | struct mv_cesa_ahash_dma_iter iter; |
605 | struct mv_cesa_op_ctx *op = NULL; | |
e41bbebd | 606 | unsigned int frag_len; |
db509a45 BB |
607 | int ret; |
608 | ||
609 | dreq->chain.first = NULL; | |
610 | dreq->chain.last = NULL; | |
611 | ||
612 | if (creq->src_nents) { | |
613 | ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, | |
614 | DMA_TO_DEVICE); | |
615 | if (!ret) { | |
616 | ret = -ENOMEM; | |
617 | goto err; | |
618 | } | |
619 | } | |
620 | ||
8c07f3a8 | 621 | mv_cesa_tdma_desc_iter_init(&dreq->chain); |
db509a45 BB |
622 | mv_cesa_ahash_req_iter_init(&iter, req); |
623 | ||
0971d09a RK |
624 | /* |
625 | * Add the cache (left-over data from a previous block) first. | |
626 | * This will never overflow the SRAM size. | |
627 | */ | |
8c07f3a8 | 628 | ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags); |
0971d09a | 629 | if (ret) |
db509a45 | 630 | goto err_free_tdma; |
db509a45 | 631 | |
d9bba4c3 RK |
632 | if (iter.src.sg) { |
633 | /* | |
634 | * Add all the new data, inserting an operation block and | |
635 | * launch command between each full SRAM block-worth of | |
e41bbebd | 636 | * data. We intentionally do not add the final op block. |
d9bba4c3 | 637 | */ |
e41bbebd | 638 | while (true) { |
8c07f3a8 BB |
639 | ret = mv_cesa_dma_add_op_transfers(&dreq->chain, |
640 | &iter.base, | |
d9bba4c3 RK |
641 | &iter.src, flags); |
642 | if (ret) | |
643 | goto err_free_tdma; | |
644 | ||
e41bbebd | 645 | frag_len = iter.base.op_len; |
db509a45 | 646 | |
e41bbebd RK |
647 | if (!mv_cesa_ahash_req_iter_next_op(&iter)) |
648 | break; | |
649 | ||
8c07f3a8 | 650 | op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, |
e41bbebd | 651 | frag_len, flags); |
d9bba4c3 RK |
652 | if (IS_ERR(op)) { |
653 | ret = PTR_ERR(op); | |
654 | goto err_free_tdma; | |
655 | } | |
db509a45 | 656 | } |
e41bbebd | 657 | } else { |
d9bba4c3 | 658 | /* Account for the data that was in the cache. */ |
e41bbebd RK |
659 | frag_len = iter.base.op_len; |
660 | } | |
661 | ||
58953e15 RK |
662 | /* |
663 | * At this point, frag_len indicates whether we have any data | |
664 | * outstanding which needs an operation. Queue up the final | |
665 | * operation, which depends whether this is the final request. | |
666 | */ | |
667 | if (creq->last_req) | |
8c07f3a8 BB |
668 | op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq, |
669 | frag_len, flags); | |
58953e15 | 670 | else if (frag_len) |
8c07f3a8 BB |
671 | op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, |
672 | frag_len, flags); | |
db509a45 | 673 | |
db509a45 BB |
674 | if (IS_ERR(op)) { |
675 | ret = PTR_ERR(op); | |
676 | goto err_free_tdma; | |
677 | } | |
678 | ||
679 | if (op) { | |
680 | /* Add dummy desc to wait for crypto operation end */ | |
8c07f3a8 | 681 | ret = mv_cesa_dma_add_dummy_end(&dreq->chain, flags); |
db509a45 BB |
682 | if (ret) |
683 | goto err_free_tdma; | |
684 | } | |
685 | ||
686 | if (!creq->last_req) | |
687 | creq->cache_ptr = req->nbytes + creq->cache_ptr - | |
688 | iter.base.len; | |
689 | else | |
690 | creq->cache_ptr = 0; | |
691 | ||
db509a45 BB |
692 | return 0; |
693 | ||
694 | err_free_tdma: | |
695 | mv_cesa_dma_cleanup(dreq); | |
696 | dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); | |
697 | ||
698 | err: | |
699 | mv_cesa_ahash_last_cleanup(req); | |
700 | ||
701 | return ret; | |
702 | } | |
703 | ||
f63601fd BB |
704 | static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) |
705 | { | |
706 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
db509a45 BB |
707 | int ret; |
708 | ||
709 | if (cesa_dev->caps->has_tdma) | |
710 | creq->req.base.type = CESA_DMA_REQ; | |
711 | else | |
712 | creq->req.base.type = CESA_STD_REQ; | |
f63601fd | 713 | |
f63601fd | 714 | creq->src_nents = sg_nents_for_len(req->src, req->nbytes); |
c22dafb3 LC |
715 | if (creq->src_nents < 0) { |
716 | dev_err(cesa_dev->dev, "Invalid number of src SG"); | |
717 | return creq->src_nents; | |
718 | } | |
f63601fd | 719 | |
db509a45 BB |
720 | ret = mv_cesa_ahash_cache_req(req, cached); |
721 | if (ret) | |
722 | return ret; | |
723 | ||
724 | if (*cached) | |
725 | return 0; | |
726 | ||
727 | if (creq->req.base.type == CESA_DMA_REQ) | |
728 | ret = mv_cesa_ahash_dma_req_init(req); | |
729 | ||
730 | return ret; | |
f63601fd BB |
731 | } |
732 | ||
733 | static int mv_cesa_ahash_update(struct ahash_request *req) | |
734 | { | |
735 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
736 | bool cached = false; | |
737 | int ret; | |
738 | ||
739 | creq->len += req->nbytes; | |
740 | ret = mv_cesa_ahash_req_init(req, &cached); | |
741 | if (ret) | |
742 | return ret; | |
743 | ||
744 | if (cached) | |
745 | return 0; | |
746 | ||
db509a45 | 747 | ret = mv_cesa_queue_req(&req->base); |
cfcd2271 | 748 | if (mv_cesa_req_needs_cleanup(&req->base, ret)) |
db509a45 | 749 | mv_cesa_ahash_cleanup(req); |
db509a45 BB |
750 | |
751 | return ret; | |
f63601fd BB |
752 | } |
753 | ||
754 | static int mv_cesa_ahash_final(struct ahash_request *req) | |
755 | { | |
756 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
757 | struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; | |
758 | bool cached = false; | |
759 | int ret; | |
760 | ||
761 | mv_cesa_set_mac_op_total_len(tmpl, creq->len); | |
762 | creq->last_req = true; | |
763 | req->nbytes = 0; | |
764 | ||
765 | ret = mv_cesa_ahash_req_init(req, &cached); | |
766 | if (ret) | |
767 | return ret; | |
768 | ||
769 | if (cached) | |
770 | return 0; | |
771 | ||
db509a45 | 772 | ret = mv_cesa_queue_req(&req->base); |
cfcd2271 | 773 | if (mv_cesa_req_needs_cleanup(&req->base, ret)) |
db509a45 BB |
774 | mv_cesa_ahash_cleanup(req); |
775 | ||
776 | return ret; | |
f63601fd BB |
777 | } |
778 | ||
779 | static int mv_cesa_ahash_finup(struct ahash_request *req) | |
780 | { | |
781 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
782 | struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; | |
783 | bool cached = false; | |
784 | int ret; | |
785 | ||
786 | creq->len += req->nbytes; | |
787 | mv_cesa_set_mac_op_total_len(tmpl, creq->len); | |
788 | creq->last_req = true; | |
789 | ||
790 | ret = mv_cesa_ahash_req_init(req, &cached); | |
791 | if (ret) | |
792 | return ret; | |
793 | ||
794 | if (cached) | |
795 | return 0; | |
796 | ||
db509a45 | 797 | ret = mv_cesa_queue_req(&req->base); |
cfcd2271 | 798 | if (mv_cesa_req_needs_cleanup(&req->base, ret)) |
db509a45 BB |
799 | mv_cesa_ahash_cleanup(req); |
800 | ||
801 | return ret; | |
f63601fd BB |
802 | } |
803 | ||
a6479ea4 RK |
804 | static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, |
805 | u64 *len, void *cache) | |
7aeef693 | 806 | { |
7aeef693 AE |
807 | struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); |
808 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
809 | unsigned int digsize = crypto_ahash_digestsize(ahash); | |
a6479ea4 | 810 | unsigned int blocksize; |
7aeef693 | 811 | |
80754539 | 812 | blocksize = crypto_ahash_blocksize(ahash); |
7aeef693 | 813 | |
a6479ea4 RK |
814 | *len = creq->len; |
815 | memcpy(hash, creq->state, digsize); | |
816 | memset(cache, 0, blocksize); | |
7aeef693 | 817 | if (creq->cache) |
a6479ea4 | 818 | memcpy(cache, creq->cache, creq->cache_ptr); |
7aeef693 AE |
819 | |
820 | return 0; | |
821 | } | |
822 | ||
a6479ea4 RK |
823 | static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, |
824 | u64 len, const void *cache) | |
7aeef693 | 825 | { |
7aeef693 AE |
826 | struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); |
827 | struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); | |
828 | unsigned int digsize = crypto_ahash_digestsize(ahash); | |
a6479ea4 | 829 | unsigned int blocksize; |
7aeef693 AE |
830 | unsigned int cache_ptr; |
831 | int ret; | |
832 | ||
e72f407e RK |
833 | ret = crypto_ahash_init(req); |
834 | if (ret) | |
835 | return ret; | |
836 | ||
80754539 | 837 | blocksize = crypto_ahash_blocksize(ahash); |
a6479ea4 | 838 | if (len >= blocksize) |
c3bf02a2 RK |
839 | mv_cesa_update_op_cfg(&creq->op_tmpl, |
840 | CESA_SA_DESC_CFG_MID_FRAG, | |
841 | CESA_SA_DESC_CFG_FRAG_MSK); | |
842 | ||
a6479ea4 RK |
843 | creq->len = len; |
844 | memcpy(creq->state, hash, digsize); | |
7aeef693 AE |
845 | creq->cache_ptr = 0; |
846 | ||
a6479ea4 | 847 | cache_ptr = do_div(len, blocksize); |
7aeef693 AE |
848 | if (!cache_ptr) |
849 | return 0; | |
850 | ||
851 | ret = mv_cesa_ahash_alloc_cache(req); | |
852 | if (ret) | |
853 | return ret; | |
854 | ||
a6479ea4 | 855 | memcpy(creq->cache, cache, cache_ptr); |
7aeef693 AE |
856 | creq->cache_ptr = cache_ptr; |
857 | ||
858 | return 0; | |
859 | } | |
860 | ||
a6479ea4 RK |
861 | static int mv_cesa_md5_init(struct ahash_request *req) |
862 | { | |
d30cb2fa | 863 | struct mv_cesa_op_ctx tmpl = { }; |
a6479ea4 RK |
864 | |
865 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); | |
866 | ||
a9eb678f | 867 | mv_cesa_ahash_init(req, &tmpl, true); |
a6479ea4 RK |
868 | |
869 | return 0; | |
870 | } | |
871 | ||
872 | static int mv_cesa_md5_export(struct ahash_request *req, void *out) | |
873 | { | |
874 | struct md5_state *out_state = out; | |
875 | ||
876 | return mv_cesa_ahash_export(req, out_state->hash, | |
877 | &out_state->byte_count, out_state->block); | |
878 | } | |
879 | ||
880 | static int mv_cesa_md5_import(struct ahash_request *req, const void *in) | |
881 | { | |
882 | const struct md5_state *in_state = in; | |
883 | ||
884 | return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count, | |
885 | in_state->block); | |
886 | } | |
887 | ||
7aeef693 AE |
888 | static int mv_cesa_md5_digest(struct ahash_request *req) |
889 | { | |
890 | int ret; | |
891 | ||
892 | ret = mv_cesa_md5_init(req); | |
893 | if (ret) | |
894 | return ret; | |
895 | ||
896 | return mv_cesa_ahash_finup(req); | |
897 | } | |
898 | ||
899 | struct ahash_alg mv_md5_alg = { | |
900 | .init = mv_cesa_md5_init, | |
901 | .update = mv_cesa_ahash_update, | |
902 | .final = mv_cesa_ahash_final, | |
903 | .finup = mv_cesa_ahash_finup, | |
904 | .digest = mv_cesa_md5_digest, | |
905 | .export = mv_cesa_md5_export, | |
906 | .import = mv_cesa_md5_import, | |
907 | .halg = { | |
908 | .digestsize = MD5_DIGEST_SIZE, | |
9f5594c9 | 909 | .statesize = sizeof(struct md5_state), |
7aeef693 AE |
910 | .base = { |
911 | .cra_name = "md5", | |
912 | .cra_driver_name = "mv-md5", | |
913 | .cra_priority = 300, | |
914 | .cra_flags = CRYPTO_ALG_ASYNC | | |
915 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
916 | .cra_blocksize = MD5_HMAC_BLOCK_SIZE, | |
917 | .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), | |
918 | .cra_init = mv_cesa_ahash_cra_init, | |
919 | .cra_module = THIS_MODULE, | |
920 | } | |
921 | } | |
922 | }; | |
923 | ||
f63601fd BB |
924 | static int mv_cesa_sha1_init(struct ahash_request *req) |
925 | { | |
d30cb2fa | 926 | struct mv_cesa_op_ctx tmpl = { }; |
f63601fd BB |
927 | |
928 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); | |
929 | ||
a9eb678f | 930 | mv_cesa_ahash_init(req, &tmpl, false); |
f63601fd BB |
931 | |
932 | return 0; | |
933 | } | |
934 | ||
935 | static int mv_cesa_sha1_export(struct ahash_request *req, void *out) | |
936 | { | |
937 | struct sha1_state *out_state = out; | |
f63601fd | 938 | |
a6479ea4 RK |
939 | return mv_cesa_ahash_export(req, out_state->state, &out_state->count, |
940 | out_state->buffer); | |
f63601fd BB |
941 | } |
942 | ||
943 | static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) | |
944 | { | |
945 | const struct sha1_state *in_state = in; | |
f63601fd | 946 | |
a6479ea4 RK |
947 | return mv_cesa_ahash_import(req, in_state->state, in_state->count, |
948 | in_state->buffer); | |
f63601fd BB |
949 | } |
950 | ||
951 | static int mv_cesa_sha1_digest(struct ahash_request *req) | |
952 | { | |
953 | int ret; | |
954 | ||
955 | ret = mv_cesa_sha1_init(req); | |
956 | if (ret) | |
957 | return ret; | |
958 | ||
959 | return mv_cesa_ahash_finup(req); | |
960 | } | |
961 | ||
962 | struct ahash_alg mv_sha1_alg = { | |
963 | .init = mv_cesa_sha1_init, | |
964 | .update = mv_cesa_ahash_update, | |
965 | .final = mv_cesa_ahash_final, | |
966 | .finup = mv_cesa_ahash_finup, | |
967 | .digest = mv_cesa_sha1_digest, | |
968 | .export = mv_cesa_sha1_export, | |
969 | .import = mv_cesa_sha1_import, | |
970 | .halg = { | |
971 | .digestsize = SHA1_DIGEST_SIZE, | |
9f5594c9 | 972 | .statesize = sizeof(struct sha1_state), |
f63601fd BB |
973 | .base = { |
974 | .cra_name = "sha1", | |
975 | .cra_driver_name = "mv-sha1", | |
976 | .cra_priority = 300, | |
977 | .cra_flags = CRYPTO_ALG_ASYNC | | |
978 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
979 | .cra_blocksize = SHA1_BLOCK_SIZE, | |
980 | .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), | |
981 | .cra_init = mv_cesa_ahash_cra_init, | |
982 | .cra_module = THIS_MODULE, | |
983 | } | |
984 | } | |
985 | }; | |
986 | ||
f85a762e AE |
987 | static int mv_cesa_sha256_init(struct ahash_request *req) |
988 | { | |
d30cb2fa | 989 | struct mv_cesa_op_ctx tmpl = { }; |
f85a762e AE |
990 | |
991 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); | |
992 | ||
a9eb678f | 993 | mv_cesa_ahash_init(req, &tmpl, false); |
f85a762e AE |
994 | |
995 | return 0; | |
996 | } | |
997 | ||
998 | static int mv_cesa_sha256_digest(struct ahash_request *req) | |
999 | { | |
1000 | int ret; | |
1001 | ||
1002 | ret = mv_cesa_sha256_init(req); | |
1003 | if (ret) | |
1004 | return ret; | |
1005 | ||
1006 | return mv_cesa_ahash_finup(req); | |
1007 | } | |
1008 | ||
1009 | static int mv_cesa_sha256_export(struct ahash_request *req, void *out) | |
1010 | { | |
1011 | struct sha256_state *out_state = out; | |
f85a762e | 1012 | |
a6479ea4 RK |
1013 | return mv_cesa_ahash_export(req, out_state->state, &out_state->count, |
1014 | out_state->buf); | |
f85a762e AE |
1015 | } |
1016 | ||
1017 | static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) | |
1018 | { | |
1019 | const struct sha256_state *in_state = in; | |
f85a762e | 1020 | |
a6479ea4 RK |
1021 | return mv_cesa_ahash_import(req, in_state->state, in_state->count, |
1022 | in_state->buf); | |
f85a762e AE |
1023 | } |
1024 | ||
1025 | struct ahash_alg mv_sha256_alg = { | |
1026 | .init = mv_cesa_sha256_init, | |
1027 | .update = mv_cesa_ahash_update, | |
1028 | .final = mv_cesa_ahash_final, | |
1029 | .finup = mv_cesa_ahash_finup, | |
1030 | .digest = mv_cesa_sha256_digest, | |
1031 | .export = mv_cesa_sha256_export, | |
1032 | .import = mv_cesa_sha256_import, | |
1033 | .halg = { | |
1034 | .digestsize = SHA256_DIGEST_SIZE, | |
9f5594c9 | 1035 | .statesize = sizeof(struct sha256_state), |
f85a762e AE |
1036 | .base = { |
1037 | .cra_name = "sha256", | |
1038 | .cra_driver_name = "mv-sha256", | |
1039 | .cra_priority = 300, | |
1040 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1041 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1042 | .cra_blocksize = SHA256_BLOCK_SIZE, | |
1043 | .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), | |
1044 | .cra_init = mv_cesa_ahash_cra_init, | |
1045 | .cra_module = THIS_MODULE, | |
1046 | } | |
1047 | } | |
1048 | }; | |
1049 | ||
f63601fd BB |
1050 | struct mv_cesa_ahash_result { |
1051 | struct completion completion; | |
1052 | int error; | |
1053 | }; | |
1054 | ||
1055 | static void mv_cesa_hmac_ahash_complete(struct crypto_async_request *req, | |
1056 | int error) | |
1057 | { | |
1058 | struct mv_cesa_ahash_result *result = req->data; | |
1059 | ||
1060 | if (error == -EINPROGRESS) | |
1061 | return; | |
1062 | ||
1063 | result->error = error; | |
1064 | complete(&result->completion); | |
1065 | } | |
1066 | ||
1067 | static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad, | |
1068 | void *state, unsigned int blocksize) | |
1069 | { | |
1070 | struct mv_cesa_ahash_result result; | |
1071 | struct scatterlist sg; | |
1072 | int ret; | |
1073 | ||
1074 | ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, | |
1075 | mv_cesa_hmac_ahash_complete, &result); | |
1076 | sg_init_one(&sg, pad, blocksize); | |
1077 | ahash_request_set_crypt(req, &sg, pad, blocksize); | |
1078 | init_completion(&result.completion); | |
1079 | ||
1080 | ret = crypto_ahash_init(req); | |
1081 | if (ret) | |
1082 | return ret; | |
1083 | ||
1084 | ret = crypto_ahash_update(req); | |
1085 | if (ret && ret != -EINPROGRESS) | |
1086 | return ret; | |
1087 | ||
1088 | wait_for_completion_interruptible(&result.completion); | |
1089 | if (result.error) | |
1090 | return result.error; | |
1091 | ||
1092 | ret = crypto_ahash_export(req, state); | |
1093 | if (ret) | |
1094 | return ret; | |
1095 | ||
1096 | return 0; | |
1097 | } | |
1098 | ||
1099 | static int mv_cesa_ahmac_pad_init(struct ahash_request *req, | |
1100 | const u8 *key, unsigned int keylen, | |
1101 | u8 *ipad, u8 *opad, | |
1102 | unsigned int blocksize) | |
1103 | { | |
1104 | struct mv_cesa_ahash_result result; | |
1105 | struct scatterlist sg; | |
1106 | int ret; | |
1107 | int i; | |
1108 | ||
1109 | if (keylen <= blocksize) { | |
1110 | memcpy(ipad, key, keylen); | |
1111 | } else { | |
1112 | u8 *keydup = kmemdup(key, keylen, GFP_KERNEL); | |
1113 | ||
1114 | if (!keydup) | |
1115 | return -ENOMEM; | |
1116 | ||
1117 | ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, | |
1118 | mv_cesa_hmac_ahash_complete, | |
1119 | &result); | |
1120 | sg_init_one(&sg, keydup, keylen); | |
1121 | ahash_request_set_crypt(req, &sg, ipad, keylen); | |
1122 | init_completion(&result.completion); | |
1123 | ||
1124 | ret = crypto_ahash_digest(req); | |
1125 | if (ret == -EINPROGRESS) { | |
1126 | wait_for_completion_interruptible(&result.completion); | |
1127 | ret = result.error; | |
1128 | } | |
1129 | ||
1130 | /* Set the memory region to 0 to avoid any leak. */ | |
1131 | memset(keydup, 0, keylen); | |
1132 | kfree(keydup); | |
1133 | ||
1134 | if (ret) | |
1135 | return ret; | |
1136 | ||
1137 | keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); | |
1138 | } | |
1139 | ||
1140 | memset(ipad + keylen, 0, blocksize - keylen); | |
1141 | memcpy(opad, ipad, blocksize); | |
1142 | ||
1143 | for (i = 0; i < blocksize; i++) { | |
1144 | ipad[i] ^= 0x36; | |
1145 | opad[i] ^= 0x5c; | |
1146 | } | |
1147 | ||
1148 | return 0; | |
1149 | } | |
1150 | ||
1151 | static int mv_cesa_ahmac_setkey(const char *hash_alg_name, | |
1152 | const u8 *key, unsigned int keylen, | |
1153 | void *istate, void *ostate) | |
1154 | { | |
1155 | struct ahash_request *req; | |
1156 | struct crypto_ahash *tfm; | |
1157 | unsigned int blocksize; | |
1158 | u8 *ipad = NULL; | |
1159 | u8 *opad; | |
1160 | int ret; | |
1161 | ||
1162 | tfm = crypto_alloc_ahash(hash_alg_name, CRYPTO_ALG_TYPE_AHASH, | |
1163 | CRYPTO_ALG_TYPE_AHASH_MASK); | |
1164 | if (IS_ERR(tfm)) | |
1165 | return PTR_ERR(tfm); | |
1166 | ||
1167 | req = ahash_request_alloc(tfm, GFP_KERNEL); | |
1168 | if (!req) { | |
1169 | ret = -ENOMEM; | |
1170 | goto free_ahash; | |
1171 | } | |
1172 | ||
1173 | crypto_ahash_clear_flags(tfm, ~0); | |
1174 | ||
1175 | blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); | |
1176 | ||
1177 | ipad = kzalloc(2 * blocksize, GFP_KERNEL); | |
1178 | if (!ipad) { | |
1179 | ret = -ENOMEM; | |
1180 | goto free_req; | |
1181 | } | |
1182 | ||
1183 | opad = ipad + blocksize; | |
1184 | ||
1185 | ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize); | |
1186 | if (ret) | |
1187 | goto free_ipad; | |
1188 | ||
1189 | ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize); | |
1190 | if (ret) | |
1191 | goto free_ipad; | |
1192 | ||
1193 | ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize); | |
1194 | ||
1195 | free_ipad: | |
1196 | kfree(ipad); | |
1197 | free_req: | |
1198 | ahash_request_free(req); | |
1199 | free_ahash: | |
1200 | crypto_free_ahash(tfm); | |
1201 | ||
1202 | return ret; | |
1203 | } | |
1204 | ||
1205 | static int mv_cesa_ahmac_cra_init(struct crypto_tfm *tfm) | |
1206 | { | |
1207 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(tfm); | |
1208 | ||
1209 | ctx->base.ops = &mv_cesa_ahash_req_ops; | |
1210 | ||
1211 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), | |
1212 | sizeof(struct mv_cesa_ahash_req)); | |
1213 | return 0; | |
1214 | } | |
1215 | ||
7aeef693 AE |
1216 | static int mv_cesa_ahmac_md5_init(struct ahash_request *req) |
1217 | { | |
1218 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); | |
d30cb2fa | 1219 | struct mv_cesa_op_ctx tmpl = { }; |
7aeef693 AE |
1220 | |
1221 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_MD5); | |
1222 | memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); | |
1223 | ||
a9eb678f | 1224 | mv_cesa_ahash_init(req, &tmpl, true); |
7aeef693 AE |
1225 | |
1226 | return 0; | |
1227 | } | |
1228 | ||
1229 | static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, | |
1230 | unsigned int keylen) | |
1231 | { | |
1232 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); | |
1233 | struct md5_state istate, ostate; | |
1234 | int ret, i; | |
1235 | ||
1236 | ret = mv_cesa_ahmac_setkey("mv-md5", key, keylen, &istate, &ostate); | |
1237 | if (ret) | |
1238 | return ret; | |
1239 | ||
1240 | for (i = 0; i < ARRAY_SIZE(istate.hash); i++) | |
1241 | ctx->iv[i] = be32_to_cpu(istate.hash[i]); | |
1242 | ||
1243 | for (i = 0; i < ARRAY_SIZE(ostate.hash); i++) | |
1244 | ctx->iv[i + 8] = be32_to_cpu(ostate.hash[i]); | |
1245 | ||
1246 | return 0; | |
1247 | } | |
1248 | ||
1249 | static int mv_cesa_ahmac_md5_digest(struct ahash_request *req) | |
1250 | { | |
1251 | int ret; | |
1252 | ||
1253 | ret = mv_cesa_ahmac_md5_init(req); | |
1254 | if (ret) | |
1255 | return ret; | |
1256 | ||
1257 | return mv_cesa_ahash_finup(req); | |
1258 | } | |
1259 | ||
1260 | struct ahash_alg mv_ahmac_md5_alg = { | |
1261 | .init = mv_cesa_ahmac_md5_init, | |
1262 | .update = mv_cesa_ahash_update, | |
1263 | .final = mv_cesa_ahash_final, | |
1264 | .finup = mv_cesa_ahash_finup, | |
1265 | .digest = mv_cesa_ahmac_md5_digest, | |
1266 | .setkey = mv_cesa_ahmac_md5_setkey, | |
1267 | .export = mv_cesa_md5_export, | |
1268 | .import = mv_cesa_md5_import, | |
1269 | .halg = { | |
1270 | .digestsize = MD5_DIGEST_SIZE, | |
1271 | .statesize = sizeof(struct md5_state), | |
1272 | .base = { | |
1273 | .cra_name = "hmac(md5)", | |
1274 | .cra_driver_name = "mv-hmac-md5", | |
1275 | .cra_priority = 300, | |
1276 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1277 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1278 | .cra_blocksize = MD5_HMAC_BLOCK_SIZE, | |
1279 | .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), | |
1280 | .cra_init = mv_cesa_ahmac_cra_init, | |
1281 | .cra_module = THIS_MODULE, | |
1282 | } | |
1283 | } | |
1284 | }; | |
1285 | ||
f63601fd BB |
1286 | static int mv_cesa_ahmac_sha1_init(struct ahash_request *req) |
1287 | { | |
1288 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); | |
d30cb2fa | 1289 | struct mv_cesa_op_ctx tmpl = { }; |
f63601fd BB |
1290 | |
1291 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA1); | |
1292 | memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); | |
1293 | ||
a9eb678f | 1294 | mv_cesa_ahash_init(req, &tmpl, false); |
f63601fd BB |
1295 | |
1296 | return 0; | |
1297 | } | |
1298 | ||
1299 | static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, | |
1300 | unsigned int keylen) | |
1301 | { | |
1302 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); | |
1303 | struct sha1_state istate, ostate; | |
1304 | int ret, i; | |
1305 | ||
1306 | ret = mv_cesa_ahmac_setkey("mv-sha1", key, keylen, &istate, &ostate); | |
1307 | if (ret) | |
1308 | return ret; | |
1309 | ||
1310 | for (i = 0; i < ARRAY_SIZE(istate.state); i++) | |
1311 | ctx->iv[i] = be32_to_cpu(istate.state[i]); | |
1312 | ||
1313 | for (i = 0; i < ARRAY_SIZE(ostate.state); i++) | |
1314 | ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); | |
1315 | ||
1316 | return 0; | |
1317 | } | |
1318 | ||
1319 | static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req) | |
1320 | { | |
1321 | int ret; | |
1322 | ||
1323 | ret = mv_cesa_ahmac_sha1_init(req); | |
1324 | if (ret) | |
1325 | return ret; | |
1326 | ||
1327 | return mv_cesa_ahash_finup(req); | |
1328 | } | |
1329 | ||
1330 | struct ahash_alg mv_ahmac_sha1_alg = { | |
1331 | .init = mv_cesa_ahmac_sha1_init, | |
1332 | .update = mv_cesa_ahash_update, | |
1333 | .final = mv_cesa_ahash_final, | |
1334 | .finup = mv_cesa_ahash_finup, | |
1335 | .digest = mv_cesa_ahmac_sha1_digest, | |
1336 | .setkey = mv_cesa_ahmac_sha1_setkey, | |
1337 | .export = mv_cesa_sha1_export, | |
1338 | .import = mv_cesa_sha1_import, | |
1339 | .halg = { | |
1340 | .digestsize = SHA1_DIGEST_SIZE, | |
1341 | .statesize = sizeof(struct sha1_state), | |
1342 | .base = { | |
1343 | .cra_name = "hmac(sha1)", | |
1344 | .cra_driver_name = "mv-hmac-sha1", | |
1345 | .cra_priority = 300, | |
1346 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1347 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1348 | .cra_blocksize = SHA1_BLOCK_SIZE, | |
1349 | .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), | |
1350 | .cra_init = mv_cesa_ahmac_cra_init, | |
1351 | .cra_module = THIS_MODULE, | |
1352 | } | |
1353 | } | |
1354 | }; | |
f85a762e AE |
1355 | |
1356 | static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, | |
1357 | unsigned int keylen) | |
1358 | { | |
1359 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); | |
1360 | struct sha256_state istate, ostate; | |
1361 | int ret, i; | |
1362 | ||
1363 | ret = mv_cesa_ahmac_setkey("mv-sha256", key, keylen, &istate, &ostate); | |
1364 | if (ret) | |
1365 | return ret; | |
1366 | ||
1367 | for (i = 0; i < ARRAY_SIZE(istate.state); i++) | |
1368 | ctx->iv[i] = be32_to_cpu(istate.state[i]); | |
1369 | ||
1370 | for (i = 0; i < ARRAY_SIZE(ostate.state); i++) | |
1371 | ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); | |
1372 | ||
1373 | return 0; | |
1374 | } | |
1375 | ||
1376 | static int mv_cesa_ahmac_sha256_init(struct ahash_request *req) | |
1377 | { | |
1378 | struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); | |
d30cb2fa | 1379 | struct mv_cesa_op_ctx tmpl = { }; |
f85a762e AE |
1380 | |
1381 | mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA256); | |
1382 | memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); | |
1383 | ||
a9eb678f | 1384 | mv_cesa_ahash_init(req, &tmpl, false); |
f85a762e AE |
1385 | |
1386 | return 0; | |
1387 | } | |
1388 | ||
1389 | static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req) | |
1390 | { | |
1391 | int ret; | |
1392 | ||
1393 | ret = mv_cesa_ahmac_sha256_init(req); | |
1394 | if (ret) | |
1395 | return ret; | |
1396 | ||
1397 | return mv_cesa_ahash_finup(req); | |
1398 | } | |
1399 | ||
1400 | struct ahash_alg mv_ahmac_sha256_alg = { | |
1401 | .init = mv_cesa_ahmac_sha256_init, | |
1402 | .update = mv_cesa_ahash_update, | |
1403 | .final = mv_cesa_ahash_final, | |
1404 | .finup = mv_cesa_ahash_finup, | |
1405 | .digest = mv_cesa_ahmac_sha256_digest, | |
1406 | .setkey = mv_cesa_ahmac_sha256_setkey, | |
1407 | .export = mv_cesa_sha256_export, | |
1408 | .import = mv_cesa_sha256_import, | |
1409 | .halg = { | |
1410 | .digestsize = SHA256_DIGEST_SIZE, | |
1411 | .statesize = sizeof(struct sha256_state), | |
1412 | .base = { | |
1413 | .cra_name = "hmac(sha256)", | |
1414 | .cra_driver_name = "mv-hmac-sha256", | |
1415 | .cra_priority = 300, | |
1416 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1417 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1418 | .cra_blocksize = SHA256_BLOCK_SIZE, | |
1419 | .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), | |
1420 | .cra_init = mv_cesa_ahmac_cra_init, | |
1421 | .cra_module = THIS_MODULE, | |
1422 | } | |
1423 | } | |
1424 | }; |