Commit | Line | Data |
---|---|---|
c942fddf | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
049359d6 JH |
2 | /** |
3 | * AMCC SoC PPC4xx Crypto Driver | |
4 | * | |
5 | * Copyright (c) 2008 Applied Micro Circuits Corporation. | |
6 | * All rights reserved. James Hsiao <jhsiao@amcc.com> | |
7 | * | |
049359d6 JH |
8 | * This file implements AMCC crypto offload Linux device driver for use with |
9 | * Linux CryptoAPI. | |
10 | */ | |
11 | ||
12 | #include <linux/kernel.h> | |
13 | #include <linux/interrupt.h> | |
14 | #include <linux/spinlock_types.h> | |
15 | #include <linux/random.h> | |
16 | #include <linux/scatterlist.h> | |
17 | #include <linux/crypto.h> | |
18 | #include <linux/dma-mapping.h> | |
19 | #include <linux/platform_device.h> | |
20 | #include <linux/init.h> | |
c11eede6 RH |
21 | #include <linux/module.h> |
22 | #include <linux/of_address.h> | |
23 | #include <linux/of_irq.h> | |
049359d6 | 24 | #include <linux/of_platform.h> |
5a0e3ad6 | 25 | #include <linux/slab.h> |
049359d6 JH |
26 | #include <asm/dcr.h> |
27 | #include <asm/dcr-regs.h> | |
28 | #include <asm/cacheflush.h> | |
a0aae821 | 29 | #include <crypto/aead.h> |
049359d6 | 30 | #include <crypto/aes.h> |
f2a13e7c | 31 | #include <crypto/ctr.h> |
59231368 | 32 | #include <crypto/gcm.h> |
a24d22b2 | 33 | #include <crypto/sha1.h> |
d072bfa4 | 34 | #include <crypto/rng.h> |
5c727f92 | 35 | #include <crypto/scatterwalk.h> |
ce05ffe1 | 36 | #include <crypto/skcipher.h> |
a0aae821 | 37 | #include <crypto/internal/aead.h> |
d072bfa4 | 38 | #include <crypto/internal/rng.h> |
8ef8d195 | 39 | #include <crypto/internal/skcipher.h> |
049359d6 JH |
40 | #include "crypto4xx_reg_def.h" |
41 | #include "crypto4xx_core.h" | |
42 | #include "crypto4xx_sa.h" | |
5343e674 | 43 | #include "crypto4xx_trng.h" |
049359d6 JH |
44 | |
45 | #define PPC4XX_SEC_VERSION_STR "0.5" | |
46 | ||
47 | /** | |
48 | * PPC4xx Crypto Engine Initialization Routine | |
49 | */ | |
50 | static void crypto4xx_hw_init(struct crypto4xx_device *dev) | |
51 | { | |
52 | union ce_ring_size ring_size; | |
7c6c0dc7 | 53 | union ce_ring_control ring_ctrl; |
049359d6 JH |
54 | union ce_part_ring_size part_ring_size; |
55 | union ce_io_threshold io_threshold; | |
56 | u32 rand_num; | |
57 | union ce_pe_dma_cfg pe_dma_cfg; | |
c5697462 | 58 | u32 device_ctrl; |
049359d6 JH |
59 | |
60 | writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG); | |
61 | /* setup pe dma, include reset sg, pdr and pe, then release reset */ | |
62 | pe_dma_cfg.w = 0; | |
63 | pe_dma_cfg.bf.bo_sgpd_en = 1; | |
64 | pe_dma_cfg.bf.bo_data_en = 0; | |
65 | pe_dma_cfg.bf.bo_sa_en = 1; | |
66 | pe_dma_cfg.bf.bo_pd_en = 1; | |
67 | pe_dma_cfg.bf.dynamic_sa_en = 1; | |
68 | pe_dma_cfg.bf.reset_sg = 1; | |
69 | pe_dma_cfg.bf.reset_pdr = 1; | |
70 | pe_dma_cfg.bf.reset_pe = 1; | |
71 | writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); | |
72 | /* un reset pe,sg and pdr */ | |
73 | pe_dma_cfg.bf.pe_mode = 0; | |
74 | pe_dma_cfg.bf.reset_sg = 0; | |
75 | pe_dma_cfg.bf.reset_pdr = 0; | |
76 | pe_dma_cfg.bf.reset_pe = 0; | |
77 | pe_dma_cfg.bf.bo_td_en = 0; | |
78 | writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); | |
79 | writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_PDR_BASE); | |
80 | writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_RDR_BASE); | |
81 | writel(PPC4XX_PRNG_CTRL_AUTO_EN, dev->ce_base + CRYPTO4XX_PRNG_CTRL); | |
82 | get_random_bytes(&rand_num, sizeof(rand_num)); | |
83 | writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_L); | |
84 | get_random_bytes(&rand_num, sizeof(rand_num)); | |
85 | writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_H); | |
86 | ring_size.w = 0; | |
87 | ring_size.bf.ring_offset = PPC4XX_PD_SIZE; | |
88 | ring_size.bf.ring_size = PPC4XX_NUM_PD; | |
89 | writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE); | |
90 | ring_ctrl.w = 0; | |
91 | writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL); | |
c5697462 JB |
92 | device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL); |
93 | device_ctrl |= PPC4XX_DC_3DES_EN; | |
94 | writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); | |
049359d6 JH |
95 | writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE); |
96 | writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE); | |
97 | part_ring_size.w = 0; | |
98 | part_ring_size.bf.sdr_size = PPC4XX_SDR_SIZE; | |
99 | part_ring_size.bf.gdr_size = PPC4XX_GDR_SIZE; | |
100 | writel(part_ring_size.w, dev->ce_base + CRYPTO4XX_PART_RING_SIZE); | |
101 | writel(PPC4XX_SD_BUFFER_SIZE, dev->ce_base + CRYPTO4XX_PART_RING_CFG); | |
102 | io_threshold.w = 0; | |
103 | io_threshold.bf.output_threshold = PPC4XX_OUTPUT_THRESHOLD; | |
104 | io_threshold.bf.input_threshold = PPC4XX_INPUT_THRESHOLD; | |
105 | writel(io_threshold.w, dev->ce_base + CRYPTO4XX_IO_THRESHOLD); | |
106 | writel(0, dev->ce_base + CRYPTO4XX_PDR_BASE_UADDR); | |
107 | writel(0, dev->ce_base + CRYPTO4XX_RDR_BASE_UADDR); | |
108 | writel(0, dev->ce_base + CRYPTO4XX_PKT_SRC_UADDR); | |
109 | writel(0, dev->ce_base + CRYPTO4XX_PKT_DEST_UADDR); | |
110 | writel(0, dev->ce_base + CRYPTO4XX_SA_UADDR); | |
111 | writel(0, dev->ce_base + CRYPTO4XX_GATH_RING_BASE_UADDR); | |
112 | writel(0, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE_UADDR); | |
113 | /* un reset pe,sg and pdr */ | |
114 | pe_dma_cfg.bf.pe_mode = 1; | |
115 | pe_dma_cfg.bf.reset_sg = 0; | |
116 | pe_dma_cfg.bf.reset_pdr = 0; | |
117 | pe_dma_cfg.bf.reset_pe = 0; | |
118 | pe_dma_cfg.bf.bo_td_en = 0; | |
119 | writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG); | |
120 | /*clear all pending interrupt*/ | |
121 | writel(PPC4XX_INTERRUPT_CLR, dev->ce_base + CRYPTO4XX_INT_CLR); | |
122 | writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); | |
123 | writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT); | |
124 | writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG); | |
b66c685a CL |
125 | if (dev->is_revb) { |
126 | writel(PPC4XX_INT_TIMEOUT_CNT_REVB << 10, | |
127 | dev->ce_base + CRYPTO4XX_INT_TIMEOUT_CNT); | |
128 | writel(PPC4XX_PD_DONE_INT | PPC4XX_TMO_ERR_INT, | |
129 | dev->ce_base + CRYPTO4XX_INT_EN); | |
130 | } else { | |
131 | writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN); | |
132 | } | |
049359d6 JH |
133 | } |
134 | ||
135 | int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) | |
136 | { | |
6396bb22 | 137 | ctx->sa_in = kcalloc(size, 4, GFP_ATOMIC); |
049359d6 JH |
138 | if (ctx->sa_in == NULL) |
139 | return -ENOMEM; | |
140 | ||
6396bb22 | 141 | ctx->sa_out = kcalloc(size, 4, GFP_ATOMIC); |
049359d6 | 142 | if (ctx->sa_out == NULL) { |
2f77690d CL |
143 | kfree(ctx->sa_in); |
144 | ctx->sa_in = NULL; | |
049359d6 JH |
145 | return -ENOMEM; |
146 | } | |
147 | ||
049359d6 JH |
148 | ctx->sa_len = size; |
149 | ||
150 | return 0; | |
151 | } | |
152 | ||
153 | void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) | |
154 | { | |
2f77690d CL |
155 | kfree(ctx->sa_in); |
156 | ctx->sa_in = NULL; | |
157 | kfree(ctx->sa_out); | |
158 | ctx->sa_out = NULL; | |
049359d6 JH |
159 | ctx->sa_len = 0; |
160 | } | |
161 | ||
049359d6 JH |
162 | /** |
163 | * alloc memory for the gather ring | |
164 | * no need to alloc buf for the ring | |
165 | * gdr_tail, gdr_head and gdr_count are initialized by this function | |
166 | */ | |
167 | static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) | |
168 | { | |
169 | int i; | |
049359d6 JH |
170 | dev->pdr = dma_alloc_coherent(dev->core_dev->device, |
171 | sizeof(struct ce_pd) * PPC4XX_NUM_PD, | |
30a50e44 | 172 | &dev->pdr_pa, GFP_KERNEL); |
049359d6 JH |
173 | if (!dev->pdr) |
174 | return -ENOMEM; | |
175 | ||
6396bb22 KC |
176 | dev->pdr_uinfo = kcalloc(PPC4XX_NUM_PD, sizeof(struct pd_uinfo), |
177 | GFP_KERNEL); | |
049359d6 JH |
178 | if (!dev->pdr_uinfo) { |
179 | dma_free_coherent(dev->core_dev->device, | |
180 | sizeof(struct ce_pd) * PPC4XX_NUM_PD, | |
181 | dev->pdr, | |
182 | dev->pdr_pa); | |
183 | return -ENOMEM; | |
184 | } | |
049359d6 | 185 | dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, |
9e0a0b3a | 186 | sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, |
049359d6 | 187 | &dev->shadow_sa_pool_pa, |
30a50e44 | 188 | GFP_KERNEL); |
049359d6 JH |
189 | if (!dev->shadow_sa_pool) |
190 | return -ENOMEM; | |
191 | ||
192 | dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device, | |
193 | sizeof(struct sa_state_record) * PPC4XX_NUM_PD, | |
30a50e44 | 194 | &dev->shadow_sr_pool_pa, GFP_KERNEL); |
049359d6 JH |
195 | if (!dev->shadow_sr_pool) |
196 | return -ENOMEM; | |
197 | for (i = 0; i < PPC4XX_NUM_PD; i++) { | |
cd4dcd6d CL |
198 | struct ce_pd *pd = &dev->pdr[i]; |
199 | struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[i]; | |
200 | ||
201 | pd->sa = dev->shadow_sa_pool_pa + | |
202 | sizeof(union shadow_sa_buf) * i; | |
049359d6 JH |
203 | |
204 | /* alloc 256 bytes which is enough for any kind of dynamic sa */ | |
9e0a0b3a | 205 | pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; |
049359d6 JH |
206 | |
207 | /* alloc state record */ | |
9e0a0b3a | 208 | pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; |
049359d6 JH |
209 | pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + |
210 | sizeof(struct sa_state_record) * i; | |
211 | } | |
212 | ||
213 | return 0; | |
214 | } | |
215 | ||
216 | static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) | |
217 | { | |
5d59ad6e | 218 | if (dev->pdr) |
049359d6 JH |
219 | dma_free_coherent(dev->core_dev->device, |
220 | sizeof(struct ce_pd) * PPC4XX_NUM_PD, | |
221 | dev->pdr, dev->pdr_pa); | |
5d59ad6e | 222 | |
049359d6 | 223 | if (dev->shadow_sa_pool) |
9e0a0b3a CL |
224 | dma_free_coherent(dev->core_dev->device, |
225 | sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, | |
226 | dev->shadow_sa_pool, dev->shadow_sa_pool_pa); | |
5d59ad6e | 227 | |
049359d6 JH |
228 | if (dev->shadow_sr_pool) |
229 | dma_free_coherent(dev->core_dev->device, | |
230 | sizeof(struct sa_state_record) * PPC4XX_NUM_PD, | |
231 | dev->shadow_sr_pool, dev->shadow_sr_pool_pa); | |
232 | ||
233 | kfree(dev->pdr_uinfo); | |
234 | } | |
235 | ||
236 | static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) | |
237 | { | |
238 | u32 retval; | |
239 | u32 tmp; | |
240 | ||
241 | retval = dev->pdr_head; | |
242 | tmp = (dev->pdr_head + 1) % PPC4XX_NUM_PD; | |
243 | ||
244 | if (tmp == dev->pdr_tail) | |
245 | return ERING_WAS_FULL; | |
246 | ||
247 | dev->pdr_head = tmp; | |
248 | ||
249 | return retval; | |
250 | } | |
251 | ||
252 | static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) | |
253 | { | |
9e0a0b3a | 254 | struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; |
4b5b7999 | 255 | u32 tail; |
049359d6 JH |
256 | unsigned long flags; |
257 | ||
049359d6 | 258 | spin_lock_irqsave(&dev->core_dev->lock, flags); |
4b5b7999 CL |
259 | pd_uinfo->state = PD_ENTRY_FREE; |
260 | ||
049359d6 JH |
261 | if (dev->pdr_tail != PPC4XX_LAST_PD) |
262 | dev->pdr_tail++; | |
263 | else | |
264 | dev->pdr_tail = 0; | |
4b5b7999 | 265 | tail = dev->pdr_tail; |
049359d6 JH |
266 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); |
267 | ||
4b5b7999 | 268 | return tail; |
049359d6 JH |
269 | } |
270 | ||
049359d6 JH |
271 | /** |
272 | * alloc memory for the gather ring | |
273 | * no need to alloc buf for the ring | |
274 | * gdr_tail, gdr_head and gdr_count are initialized by this function | |
275 | */ | |
276 | static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev) | |
277 | { | |
750afb08 LC |
278 | dev->gdr = dma_alloc_coherent(dev->core_dev->device, |
279 | sizeof(struct ce_gd) * PPC4XX_NUM_GD, | |
30a50e44 | 280 | &dev->gdr_pa, GFP_KERNEL); |
049359d6 JH |
281 | if (!dev->gdr) |
282 | return -ENOMEM; | |
283 | ||
049359d6 JH |
284 | return 0; |
285 | } | |
286 | ||
287 | static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) | |
288 | { | |
b87b2c4d CL |
289 | if (dev->gdr) |
290 | dma_free_coherent(dev->core_dev->device, | |
049359d6 JH |
291 | sizeof(struct ce_gd) * PPC4XX_NUM_GD, |
292 | dev->gdr, dev->gdr_pa); | |
293 | } | |
294 | ||
295 | /* | |
296 | * when this function is called. | |
297 | * preemption or interrupt must be disabled | |
298 | */ | |
64e1062b | 299 | static u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) |
049359d6 JH |
300 | { |
301 | u32 retval; | |
302 | u32 tmp; | |
64e1062b | 303 | |
049359d6 JH |
304 | if (n >= PPC4XX_NUM_GD) |
305 | return ERING_WAS_FULL; | |
306 | ||
307 | retval = dev->gdr_head; | |
308 | tmp = (dev->gdr_head + n) % PPC4XX_NUM_GD; | |
309 | if (dev->gdr_head > dev->gdr_tail) { | |
310 | if (tmp < dev->gdr_head && tmp >= dev->gdr_tail) | |
311 | return ERING_WAS_FULL; | |
312 | } else if (dev->gdr_head < dev->gdr_tail) { | |
313 | if (tmp < dev->gdr_head || tmp >= dev->gdr_tail) | |
314 | return ERING_WAS_FULL; | |
315 | } | |
316 | dev->gdr_head = tmp; | |
317 | ||
318 | return retval; | |
319 | } | |
320 | ||
321 | static u32 crypto4xx_put_gd_to_gdr(struct crypto4xx_device *dev) | |
322 | { | |
323 | unsigned long flags; | |
324 | ||
325 | spin_lock_irqsave(&dev->core_dev->lock, flags); | |
326 | if (dev->gdr_tail == dev->gdr_head) { | |
327 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
328 | return 0; | |
329 | } | |
330 | ||
331 | if (dev->gdr_tail != PPC4XX_LAST_GD) | |
332 | dev->gdr_tail++; | |
333 | else | |
334 | dev->gdr_tail = 0; | |
335 | ||
336 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
337 | ||
338 | return 0; | |
339 | } | |
340 | ||
341 | static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, | |
342 | dma_addr_t *gd_dma, u32 idx) | |
343 | { | |
344 | *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; | |
345 | ||
9e0a0b3a | 346 | return &dev->gdr[idx]; |
049359d6 JH |
347 | } |
348 | ||
349 | /** | |
350 | * alloc memory for the scatter ring | |
351 | * need to alloc buf for the ring | |
352 | * sdr_tail, sdr_head and sdr_count are initialized by this function | |
353 | */ | |
354 | static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) | |
355 | { | |
356 | int i; | |
049359d6 | 357 | |
049359d6 JH |
358 | dev->scatter_buffer_va = |
359 | dma_alloc_coherent(dev->core_dev->device, | |
40e3b847 | 360 | PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, |
30a50e44 | 361 | &dev->scatter_buffer_pa, GFP_KERNEL); |
746c908c | 362 | if (!dev->scatter_buffer_va) |
049359d6 | 363 | return -ENOMEM; |
049359d6 | 364 | |
b87b2c4d CL |
365 | /* alloc memory for scatter descriptor ring */ |
366 | dev->sdr = dma_alloc_coherent(dev->core_dev->device, | |
367 | sizeof(struct ce_sd) * PPC4XX_NUM_SD, | |
30a50e44 | 368 | &dev->sdr_pa, GFP_KERNEL); |
b87b2c4d CL |
369 | if (!dev->sdr) |
370 | return -ENOMEM; | |
371 | ||
049359d6 | 372 | for (i = 0; i < PPC4XX_NUM_SD; i++) { |
9e0a0b3a | 373 | dev->sdr[i].ptr = dev->scatter_buffer_pa + |
40e3b847 | 374 | PPC4XX_SD_BUFFER_SIZE * i; |
049359d6 JH |
375 | } |
376 | ||
377 | return 0; | |
378 | } | |
379 | ||
380 | static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) | |
381 | { | |
5d59ad6e | 382 | if (dev->sdr) |
049359d6 JH |
383 | dma_free_coherent(dev->core_dev->device, |
384 | sizeof(struct ce_sd) * PPC4XX_NUM_SD, | |
385 | dev->sdr, dev->sdr_pa); | |
386 | ||
5d59ad6e | 387 | if (dev->scatter_buffer_va) |
049359d6 | 388 | dma_free_coherent(dev->core_dev->device, |
40e3b847 | 389 | PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, |
049359d6 JH |
390 | dev->scatter_buffer_va, |
391 | dev->scatter_buffer_pa); | |
392 | } | |
393 | ||
394 | /* | |
395 | * when this function is called. | |
396 | * preemption or interrupt must be disabled | |
397 | */ | |
398 | static u32 crypto4xx_get_n_sd(struct crypto4xx_device *dev, int n) | |
399 | { | |
400 | u32 retval; | |
401 | u32 tmp; | |
402 | ||
403 | if (n >= PPC4XX_NUM_SD) | |
404 | return ERING_WAS_FULL; | |
405 | ||
406 | retval = dev->sdr_head; | |
407 | tmp = (dev->sdr_head + n) % PPC4XX_NUM_SD; | |
408 | if (dev->sdr_head > dev->gdr_tail) { | |
409 | if (tmp < dev->sdr_head && tmp >= dev->sdr_tail) | |
410 | return ERING_WAS_FULL; | |
411 | } else if (dev->sdr_head < dev->sdr_tail) { | |
412 | if (tmp < dev->sdr_head || tmp >= dev->sdr_tail) | |
413 | return ERING_WAS_FULL; | |
414 | } /* the head = tail, or empty case is already take cared */ | |
415 | dev->sdr_head = tmp; | |
416 | ||
417 | return retval; | |
418 | } | |
419 | ||
420 | static u32 crypto4xx_put_sd_to_sdr(struct crypto4xx_device *dev) | |
421 | { | |
422 | unsigned long flags; | |
423 | ||
424 | spin_lock_irqsave(&dev->core_dev->lock, flags); | |
425 | if (dev->sdr_tail == dev->sdr_head) { | |
426 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
427 | return 0; | |
428 | } | |
429 | if (dev->sdr_tail != PPC4XX_LAST_SD) | |
430 | dev->sdr_tail++; | |
431 | else | |
432 | dev->sdr_tail = 0; | |
433 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
434 | ||
435 | return 0; | |
436 | } | |
437 | ||
438 | static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, | |
439 | dma_addr_t *sd_dma, u32 idx) | |
440 | { | |
441 | *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; | |
442 | ||
9e0a0b3a | 443 | return &dev->sdr[idx]; |
049359d6 JH |
444 | } |
445 | ||
049359d6 JH |
446 | static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, |
447 | struct ce_pd *pd, | |
448 | struct pd_uinfo *pd_uinfo, | |
449 | u32 nbytes, | |
450 | struct scatterlist *dst) | |
451 | { | |
5c727f92 CL |
452 | unsigned int first_sd = pd_uinfo->first_sd; |
453 | unsigned int last_sd; | |
454 | unsigned int overflow = 0; | |
455 | unsigned int to_copy; | |
456 | unsigned int dst_start = 0; | |
457 | ||
458 | /* | |
459 | * Because the scatter buffers are all neatly organized in one | |
460 | * big continuous ringbuffer; scatterwalk_map_and_copy() can | |
461 | * be instructed to copy a range of buffers in one go. | |
462 | */ | |
463 | ||
464 | last_sd = (first_sd + pd_uinfo->num_sd); | |
465 | if (last_sd > PPC4XX_LAST_SD) { | |
466 | last_sd = PPC4XX_LAST_SD; | |
467 | overflow = last_sd % PPC4XX_NUM_SD; | |
468 | } | |
049359d6 JH |
469 | |
470 | while (nbytes) { | |
5c727f92 CL |
471 | void *buf = dev->scatter_buffer_va + |
472 | first_sd * PPC4XX_SD_BUFFER_SIZE; | |
473 | ||
474 | to_copy = min(nbytes, PPC4XX_SD_BUFFER_SIZE * | |
475 | (1 + last_sd - first_sd)); | |
476 | scatterwalk_map_and_copy(buf, dst, dst_start, to_copy, 1); | |
477 | nbytes -= to_copy; | |
478 | ||
479 | if (overflow) { | |
480 | first_sd = 0; | |
481 | last_sd = overflow; | |
482 | dst_start += to_copy; | |
483 | overflow = 0; | |
049359d6 JH |
484 | } |
485 | } | |
486 | } | |
487 | ||
cd4dcd6d CL |
488 | static void crypto4xx_copy_digest_to_dst(void *dst, |
489 | struct pd_uinfo *pd_uinfo, | |
049359d6 JH |
490 | struct crypto4xx_ctx *ctx) |
491 | { | |
492 | struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; | |
049359d6 JH |
493 | |
494 | if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { | |
cd4dcd6d | 495 | memcpy(dst, pd_uinfo->sr_va->save_digest, |
049359d6 JH |
496 | SA_HASH_ALG_SHA1_DIGEST_SIZE); |
497 | } | |
049359d6 JH |
498 | } |
499 | ||
500 | static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, | |
501 | struct pd_uinfo *pd_uinfo) | |
502 | { | |
503 | int i; | |
504 | if (pd_uinfo->num_gd) { | |
505 | for (i = 0; i < pd_uinfo->num_gd; i++) | |
506 | crypto4xx_put_gd_to_gdr(dev); | |
507 | pd_uinfo->first_gd = 0xffffffff; | |
508 | pd_uinfo->num_gd = 0; | |
509 | } | |
510 | if (pd_uinfo->num_sd) { | |
511 | for (i = 0; i < pd_uinfo->num_sd; i++) | |
512 | crypto4xx_put_sd_to_sdr(dev); | |
513 | ||
514 | pd_uinfo->first_sd = 0xffffffff; | |
515 | pd_uinfo->num_sd = 0; | |
516 | } | |
517 | } | |
518 | ||
ce05ffe1 | 519 | static void crypto4xx_cipher_done(struct crypto4xx_device *dev, |
049359d6 JH |
520 | struct pd_uinfo *pd_uinfo, |
521 | struct ce_pd *pd) | |
522 | { | |
ce05ffe1 | 523 | struct skcipher_request *req; |
049359d6 JH |
524 | struct scatterlist *dst; |
525 | dma_addr_t addr; | |
526 | ||
ce05ffe1 | 527 | req = skcipher_request_cast(pd_uinfo->async_req); |
049359d6 | 528 | |
38cf5533 | 529 | if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { |
ce05ffe1 CL |
530 | crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, |
531 | req->cryptlen, req->dst); | |
049359d6 JH |
532 | } else { |
533 | dst = pd_uinfo->dest_va; | |
534 | addr = dma_map_page(dev->core_dev->device, sg_page(dst), | |
535 | dst->offset, dst->length, DMA_FROM_DEVICE); | |
536 | } | |
fc340115 CL |
537 | |
538 | if (pd_uinfo->sa_va->sa_command_0.bf.save_iv == SA_SAVE_IV) { | |
539 | struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); | |
540 | ||
541 | crypto4xx_memcpy_from_le32((u32 *)req->iv, | |
542 | pd_uinfo->sr_va->save_iv, | |
543 | crypto_skcipher_ivsize(skcipher)); | |
544 | } | |
545 | ||
049359d6 | 546 | crypto4xx_ret_sg_desc(dev, pd_uinfo); |
8ef8d195 CL |
547 | |
548 | if (pd_uinfo->state & PD_ENTRY_BUSY) | |
ce05ffe1 CL |
549 | skcipher_request_complete(req, -EINPROGRESS); |
550 | skcipher_request_complete(req, 0); | |
049359d6 JH |
551 | } |
552 | ||
a0aae821 | 553 | static void crypto4xx_ahash_done(struct crypto4xx_device *dev, |
049359d6 JH |
554 | struct pd_uinfo *pd_uinfo) |
555 | { | |
556 | struct crypto4xx_ctx *ctx; | |
557 | struct ahash_request *ahash_req; | |
558 | ||
559 | ahash_req = ahash_request_cast(pd_uinfo->async_req); | |
560 | ctx = crypto_tfm_ctx(ahash_req->base.tfm); | |
561 | ||
cd4dcd6d | 562 | crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, |
049359d6 JH |
563 | crypto_tfm_ctx(ahash_req->base.tfm)); |
564 | crypto4xx_ret_sg_desc(dev, pd_uinfo); | |
8ef8d195 CL |
565 | |
566 | if (pd_uinfo->state & PD_ENTRY_BUSY) | |
567 | ahash_request_complete(ahash_req, -EINPROGRESS); | |
568 | ahash_request_complete(ahash_req, 0); | |
a0aae821 | 569 | } |
049359d6 | 570 | |
a0aae821 CL |
571 | static void crypto4xx_aead_done(struct crypto4xx_device *dev, |
572 | struct pd_uinfo *pd_uinfo, | |
573 | struct ce_pd *pd) | |
574 | { | |
0b5a7f71 CL |
575 | struct aead_request *aead_req = container_of(pd_uinfo->async_req, |
576 | struct aead_request, base); | |
a0aae821 | 577 | struct scatterlist *dst = pd_uinfo->dest_va; |
0b5a7f71 CL |
578 | size_t cp_len = crypto_aead_authsize( |
579 | crypto_aead_reqtfm(aead_req)); | |
a8d79d7b | 580 | u32 icv[AES_BLOCK_SIZE]; |
a0aae821 CL |
581 | int err = 0; |
582 | ||
38cf5533 | 583 | if (pd_uinfo->sa_va->sa_command_0.bf.scatter) { |
a0aae821 CL |
584 | crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, |
585 | pd->pd_ctl_len.bf.pkt_len, | |
586 | dst); | |
587 | } else { | |
67d8208f | 588 | dma_unmap_page(dev->core_dev->device, pd->dest, dst->length, |
a0aae821 CL |
589 | DMA_FROM_DEVICE); |
590 | } | |
591 | ||
592 | if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { | |
593 | /* append icv at the end */ | |
a0aae821 | 594 | crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, |
a8d79d7b | 595 | sizeof(icv)); |
a0aae821 CL |
596 | |
597 | scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, | |
598 | cp_len, 1); | |
0b5a7f71 CL |
599 | } else { |
600 | /* check icv at the end */ | |
601 | scatterwalk_map_and_copy(icv, aead_req->src, | |
602 | aead_req->assoclen + aead_req->cryptlen - | |
603 | cp_len, cp_len, 0); | |
604 | ||
a8d79d7b | 605 | crypto4xx_memcpy_from_le32(icv, icv, sizeof(icv)); |
0b5a7f71 CL |
606 | |
607 | if (crypto_memneq(icv, pd_uinfo->sr_va->save_digest, cp_len)) | |
608 | err = -EBADMSG; | |
a0aae821 CL |
609 | } |
610 | ||
611 | crypto4xx_ret_sg_desc(dev, pd_uinfo); | |
612 | ||
613 | if (pd->pd_ctl.bf.status & 0xff) { | |
0b5a7f71 CL |
614 | if (!__ratelimit(&dev->aead_ratelimit)) { |
615 | if (pd->pd_ctl.bf.status & 2) | |
616 | pr_err("pad fail error\n"); | |
617 | if (pd->pd_ctl.bf.status & 4) | |
618 | pr_err("seqnum fail\n"); | |
619 | if (pd->pd_ctl.bf.status & 8) | |
620 | pr_err("error _notify\n"); | |
621 | pr_err("aead return err status = 0x%02x\n", | |
622 | pd->pd_ctl.bf.status & 0xff); | |
623 | pr_err("pd pad_ctl = 0x%08x\n", | |
624 | pd->pd_ctl.bf.pd_pad_ctl); | |
a0aae821 | 625 | } |
0b5a7f71 | 626 | err = -EINVAL; |
a0aae821 CL |
627 | } |
628 | ||
629 | if (pd_uinfo->state & PD_ENTRY_BUSY) | |
630 | aead_request_complete(aead_req, -EINPROGRESS); | |
631 | ||
632 | aead_request_complete(aead_req, err); | |
049359d6 JH |
633 | } |
634 | ||
a0aae821 | 635 | static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) |
049359d6 | 636 | { |
9e0a0b3a CL |
637 | struct ce_pd *pd = &dev->pdr[idx]; |
638 | struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; | |
049359d6 | 639 | |
a0aae821 | 640 | switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) { |
ce05ffe1 CL |
641 | case CRYPTO_ALG_TYPE_SKCIPHER: |
642 | crypto4xx_cipher_done(dev, pd_uinfo, pd); | |
a0aae821 CL |
643 | break; |
644 | case CRYPTO_ALG_TYPE_AEAD: | |
645 | crypto4xx_aead_done(dev, pd_uinfo, pd); | |
646 | break; | |
647 | case CRYPTO_ALG_TYPE_AHASH: | |
648 | crypto4xx_ahash_done(dev, pd_uinfo); | |
649 | break; | |
650 | } | |
049359d6 JH |
651 | } |
652 | ||
049359d6 JH |
653 | static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) |
654 | { | |
655 | crypto4xx_destroy_pdr(core_dev->dev); | |
656 | crypto4xx_destroy_gdr(core_dev->dev); | |
657 | crypto4xx_destroy_sdr(core_dev->dev); | |
049359d6 JH |
658 | iounmap(core_dev->dev->ce_base); |
659 | kfree(core_dev->dev); | |
660 | kfree(core_dev); | |
661 | } | |
662 | ||
049359d6 JH |
663 | static u32 get_next_gd(u32 current) |
664 | { | |
665 | if (current != PPC4XX_LAST_GD) | |
666 | return current + 1; | |
667 | else | |
668 | return 0; | |
669 | } | |
670 | ||
671 | static u32 get_next_sd(u32 current) | |
672 | { | |
673 | if (current != PPC4XX_LAST_SD) | |
674 | return current + 1; | |
675 | else | |
676 | return 0; | |
677 | } | |
678 | ||
cd4dcd6d | 679 | int crypto4xx_build_pd(struct crypto_async_request *req, |
049359d6 JH |
680 | struct crypto4xx_ctx *ctx, |
681 | struct scatterlist *src, | |
682 | struct scatterlist *dst, | |
cd4dcd6d CL |
683 | const unsigned int datalen, |
684 | const __le32 *iv, const u32 iv_len, | |
685 | const struct dynamic_sa_ctl *req_sa, | |
a0aae821 | 686 | const unsigned int sa_len, |
658c9d2b CL |
687 | const unsigned int assoclen, |
688 | struct scatterlist *_dst) | |
049359d6 JH |
689 | { |
690 | struct crypto4xx_device *dev = ctx->dev; | |
049359d6 | 691 | struct dynamic_sa_ctl *sa; |
049359d6 JH |
692 | struct ce_gd *gd; |
693 | struct ce_pd *pd; | |
694 | u32 num_gd, num_sd; | |
695 | u32 fst_gd = 0xffffffff; | |
696 | u32 fst_sd = 0xffffffff; | |
697 | u32 pd_entry; | |
698 | unsigned long flags; | |
cd4dcd6d CL |
699 | struct pd_uinfo *pd_uinfo; |
700 | unsigned int nbytes = datalen; | |
701 | size_t offset_to_sr_ptr; | |
049359d6 | 702 | u32 gd_idx = 0; |
a0aae821 | 703 | int tmp; |
7e92e171 CL |
704 | bool is_busy, force_sd; |
705 | ||
706 | /* | |
707 | * There's a very subtile/disguised "bug" in the hardware that | |
708 | * gets indirectly mentioned in 18.1.3.5 Encryption/Decryption | |
709 | * of the hardware spec: | |
710 | * *drum roll* the AES/(T)DES OFB and CFB modes are listed as | |
711 | * operation modes for >>> "Block ciphers" <<<. | |
712 | * | |
713 | * To workaround this issue and stop the hardware from causing | |
714 | * "overran dst buffer" on crypttexts that are not a multiple | |
715 | * of 16 (AES_BLOCK_SIZE), we force the driver to use the | |
716 | * scatter buffers. | |
717 | */ | |
718 | force_sd = (req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_CFB | |
719 | || req_sa->sa_command_1.bf.crypto_mode9_8 == CRYPTO_MODE_OFB) | |
720 | && (datalen % AES_BLOCK_SIZE); | |
049359d6 | 721 | |
a0aae821 CL |
722 | /* figure how many gd are needed */ |
723 | tmp = sg_nents_for_len(src, assoclen + datalen); | |
724 | if (tmp < 0) { | |
7aff7d0a | 725 | dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); |
a0aae821 CL |
726 | return tmp; |
727 | } | |
728 | if (tmp == 1) | |
729 | tmp = 0; | |
730 | num_gd = tmp; | |
731 | ||
732 | if (assoclen) { | |
733 | nbytes += assoclen; | |
734 | dst = scatterwalk_ffwd(_dst, dst, assoclen); | |
7aff7d0a | 735 | } |
049359d6 | 736 | |
a0aae821 | 737 | /* figure how many sd are needed */ |
7e92e171 | 738 | if (sg_is_last(dst) && force_sd == false) { |
049359d6 JH |
739 | num_sd = 0; |
740 | } else { | |
741 | if (datalen > PPC4XX_SD_BUFFER_SIZE) { | |
742 | num_sd = datalen / PPC4XX_SD_BUFFER_SIZE; | |
743 | if (datalen % PPC4XX_SD_BUFFER_SIZE) | |
744 | num_sd++; | |
745 | } else { | |
746 | num_sd = 1; | |
747 | } | |
748 | } | |
749 | ||
750 | /* | |
751 | * The follow section of code needs to be protected | |
752 | * The gather ring and scatter ring needs to be consecutive | |
753 | * In case of run out of any kind of descriptor, the descriptor | |
754 | * already got must be return the original place. | |
755 | */ | |
756 | spin_lock_irqsave(&dev->core_dev->lock, flags); | |
8ef8d195 CL |
757 | /* |
758 | * Let the caller know to slow down, once more than 13/16ths = 81% | |
759 | * of the available data contexts are being used simultaneously. | |
760 | * | |
761 | * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for | |
762 | * 31 more contexts. Before new requests have to be rejected. | |
763 | */ | |
764 | if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { | |
765 | is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= | |
766 | ((PPC4XX_NUM_PD * 13) / 16); | |
767 | } else { | |
768 | /* | |
769 | * To fix contention issues between ipsec (no blacklog) and | |
770 | * dm-crypto (backlog) reserve 32 entries for "no backlog" | |
771 | * data contexts. | |
772 | */ | |
773 | is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= | |
774 | ((PPC4XX_NUM_PD * 15) / 16); | |
775 | ||
776 | if (is_busy) { | |
777 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
778 | return -EBUSY; | |
779 | } | |
780 | } | |
781 | ||
049359d6 JH |
782 | if (num_gd) { |
783 | fst_gd = crypto4xx_get_n_gd(dev, num_gd); | |
784 | if (fst_gd == ERING_WAS_FULL) { | |
785 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
786 | return -EAGAIN; | |
787 | } | |
788 | } | |
789 | if (num_sd) { | |
790 | fst_sd = crypto4xx_get_n_sd(dev, num_sd); | |
791 | if (fst_sd == ERING_WAS_FULL) { | |
792 | if (num_gd) | |
793 | dev->gdr_head = fst_gd; | |
794 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
795 | return -EAGAIN; | |
796 | } | |
797 | } | |
798 | pd_entry = crypto4xx_get_pd_from_pdr_nolock(dev); | |
799 | if (pd_entry == ERING_WAS_FULL) { | |
800 | if (num_gd) | |
801 | dev->gdr_head = fst_gd; | |
802 | if (num_sd) | |
803 | dev->sdr_head = fst_sd; | |
804 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
805 | return -EAGAIN; | |
806 | } | |
807 | spin_unlock_irqrestore(&dev->core_dev->lock, flags); | |
808 | ||
cd4dcd6d CL |
809 | pd = &dev->pdr[pd_entry]; |
810 | pd->sa_len = sa_len; | |
811 | ||
9e0a0b3a | 812 | pd_uinfo = &dev->pdr_uinfo[pd_entry]; |
049359d6 JH |
813 | pd_uinfo->num_gd = num_gd; |
814 | pd_uinfo->num_sd = num_sd; | |
7e92e171 CL |
815 | pd_uinfo->dest_va = dst; |
816 | pd_uinfo->async_req = req; | |
049359d6 | 817 | |
cd4dcd6d CL |
818 | if (iv_len) |
819 | memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); | |
049359d6 | 820 | |
cd4dcd6d CL |
821 | sa = pd_uinfo->sa_va; |
822 | memcpy(sa, req_sa, sa_len * 4); | |
823 | ||
a0aae821 | 824 | sa->sa_command_1.bf.hash_crypto_offset = (assoclen >> 2); |
cd4dcd6d CL |
825 | offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); |
826 | *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; | |
049359d6 | 827 | |
049359d6 | 828 | if (num_gd) { |
cd4dcd6d CL |
829 | dma_addr_t gd_dma; |
830 | struct scatterlist *sg; | |
831 | ||
049359d6 JH |
832 | /* get first gd we are going to use */ |
833 | gd_idx = fst_gd; | |
834 | pd_uinfo->first_gd = fst_gd; | |
049359d6 JH |
835 | gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); |
836 | pd->src = gd_dma; | |
837 | /* enable gather */ | |
838 | sa->sa_command_0.bf.gather = 1; | |
049359d6 | 839 | /* walk the sg, and setup gather array */ |
cd4dcd6d CL |
840 | |
841 | sg = src; | |
049359d6 | 842 | while (nbytes) { |
cd4dcd6d CL |
843 | size_t len; |
844 | ||
845 | len = min(sg->length, nbytes); | |
846 | gd->ptr = dma_map_page(dev->core_dev->device, | |
847 | sg_page(sg), sg->offset, len, DMA_TO_DEVICE); | |
848 | gd->ctl_len.len = len; | |
049359d6 JH |
849 | gd->ctl_len.done = 0; |
850 | gd->ctl_len.ready = 1; | |
cd4dcd6d | 851 | if (len >= nbytes) |
049359d6 | 852 | break; |
cd4dcd6d | 853 | |
049359d6 JH |
854 | nbytes -= sg->length; |
855 | gd_idx = get_next_gd(gd_idx); | |
856 | gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); | |
cd4dcd6d | 857 | sg = sg_next(sg); |
049359d6 JH |
858 | } |
859 | } else { | |
860 | pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), | |
cd4dcd6d CL |
861 | src->offset, min(nbytes, src->length), |
862 | DMA_TO_DEVICE); | |
049359d6 JH |
863 | /* |
864 | * Disable gather in sa command | |
865 | */ | |
866 | sa->sa_command_0.bf.gather = 0; | |
867 | /* | |
868 | * Indicate gather array is not used | |
869 | */ | |
870 | pd_uinfo->first_gd = 0xffffffff; | |
049359d6 | 871 | } |
7e92e171 | 872 | if (!num_sd) { |
049359d6 JH |
873 | /* |
874 | * we know application give us dst a whole piece of memory | |
875 | * no need to use scatter ring. | |
049359d6 | 876 | */ |
049359d6 | 877 | pd_uinfo->first_sd = 0xffffffff; |
049359d6 | 878 | sa->sa_command_0.bf.scatter = 0; |
cd4dcd6d CL |
879 | pd->dest = (u32)dma_map_page(dev->core_dev->device, |
880 | sg_page(dst), dst->offset, | |
881 | min(datalen, dst->length), | |
882 | DMA_TO_DEVICE); | |
049359d6 | 883 | } else { |
cd4dcd6d | 884 | dma_addr_t sd_dma; |
049359d6 | 885 | struct ce_sd *sd = NULL; |
cd4dcd6d | 886 | |
049359d6 JH |
887 | u32 sd_idx = fst_sd; |
888 | nbytes = datalen; | |
889 | sa->sa_command_0.bf.scatter = 1; | |
049359d6 | 890 | pd_uinfo->first_sd = fst_sd; |
049359d6 JH |
891 | sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); |
892 | pd->dest = sd_dma; | |
893 | /* setup scatter descriptor */ | |
894 | sd->ctl.done = 0; | |
895 | sd->ctl.rdy = 1; | |
896 | /* sd->ptr should be setup by sd_init routine*/ | |
049359d6 JH |
897 | if (nbytes >= PPC4XX_SD_BUFFER_SIZE) |
898 | nbytes -= PPC4XX_SD_BUFFER_SIZE; | |
899 | else | |
900 | nbytes = 0; | |
901 | while (nbytes) { | |
902 | sd_idx = get_next_sd(sd_idx); | |
903 | sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx); | |
904 | /* setup scatter descriptor */ | |
905 | sd->ctl.done = 0; | |
906 | sd->ctl.rdy = 1; | |
cd4dcd6d | 907 | if (nbytes >= PPC4XX_SD_BUFFER_SIZE) { |
049359d6 | 908 | nbytes -= PPC4XX_SD_BUFFER_SIZE; |
cd4dcd6d | 909 | } else { |
049359d6 JH |
910 | /* |
911 | * SD entry can hold PPC4XX_SD_BUFFER_SIZE, | |
912 | * which is more than nbytes, so done. | |
913 | */ | |
914 | nbytes = 0; | |
cd4dcd6d | 915 | } |
049359d6 JH |
916 | } |
917 | } | |
918 | ||
4b5b7999 | 919 | pd->pd_ctl.w = PD_CTL_HOST_READY | |
5bdad829 | 920 | ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || |
4b5b7999 CL |
921 | (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? |
922 | PD_CTL_HASH_FINAL : 0); | |
a0aae821 | 923 | pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); |
8ef8d195 CL |
924 | pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); |
925 | ||
049359d6 JH |
926 | wmb(); |
927 | /* write any value to push engine to read a pd */ | |
4b5b7999 | 928 | writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); |
049359d6 | 929 | writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); |
8ef8d195 | 930 | return is_busy ? -EBUSY : -EINPROGRESS; |
049359d6 JH |
931 | } |
932 | ||
933 | /** | |
934 | * Algorithm Registration Functions | |
935 | */ | |
a0aae821 CL |
936 | static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg, |
937 | struct crypto4xx_ctx *ctx) | |
049359d6 | 938 | { |
049359d6 JH |
939 | ctx->dev = amcc_alg->dev; |
940 | ctx->sa_in = NULL; | |
941 | ctx->sa_out = NULL; | |
049359d6 | 942 | ctx->sa_len = 0; |
a0aae821 | 943 | } |
049359d6 | 944 | |
ce05ffe1 | 945 | static int crypto4xx_sk_init(struct crypto_skcipher *sk) |
a0aae821 | 946 | { |
ce05ffe1 | 947 | struct skcipher_alg *alg = crypto_skcipher_alg(sk); |
a0aae821 | 948 | struct crypto4xx_alg *amcc_alg; |
ce05ffe1 | 949 | struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(sk); |
049359d6 | 950 | |
98e87e3d CL |
951 | if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { |
952 | ctx->sw_cipher.cipher = | |
9848e4c8 CL |
953 | crypto_alloc_sync_skcipher(alg->base.cra_name, 0, |
954 | CRYPTO_ALG_NEED_FALLBACK); | |
98e87e3d CL |
955 | if (IS_ERR(ctx->sw_cipher.cipher)) |
956 | return PTR_ERR(ctx->sw_cipher.cipher); | |
98e87e3d CL |
957 | } |
958 | ||
a0aae821 CL |
959 | amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); |
960 | crypto4xx_ctx_init(amcc_alg, ctx); | |
049359d6 JH |
961 | return 0; |
962 | } | |
963 | ||
a0aae821 | 964 | static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx) |
049359d6 | 965 | { |
049359d6 | 966 | crypto4xx_free_sa(ctx); |
049359d6 JH |
967 | } |
968 | ||
ce05ffe1 | 969 | static void crypto4xx_sk_exit(struct crypto_skcipher *sk) |
a0aae821 | 970 | { |
ce05ffe1 CL |
971 | struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(sk); |
972 | ||
973 | crypto4xx_common_exit(ctx); | |
98e87e3d | 974 | if (ctx->sw_cipher.cipher) |
9848e4c8 | 975 | crypto_free_sync_skcipher(ctx->sw_cipher.cipher); |
a0aae821 CL |
976 | } |
977 | ||
978 | static int crypto4xx_aead_init(struct crypto_aead *tfm) | |
979 | { | |
980 | struct aead_alg *alg = crypto_aead_alg(tfm); | |
981 | struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); | |
982 | struct crypto4xx_alg *amcc_alg; | |
983 | ||
984 | ctx->sw_cipher.aead = crypto_alloc_aead(alg->base.cra_name, 0, | |
985 | CRYPTO_ALG_NEED_FALLBACK | | |
986 | CRYPTO_ALG_ASYNC); | |
987 | if (IS_ERR(ctx->sw_cipher.aead)) | |
988 | return PTR_ERR(ctx->sw_cipher.aead); | |
989 | ||
990 | amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead); | |
991 | crypto4xx_ctx_init(amcc_alg, ctx); | |
658c9d2b CL |
992 | crypto_aead_set_reqsize(tfm, max(sizeof(struct aead_request) + 32 + |
993 | crypto_aead_reqsize(ctx->sw_cipher.aead), | |
994 | sizeof(struct crypto4xx_aead_reqctx))); | |
a0aae821 CL |
995 | return 0; |
996 | } | |
997 | ||
998 | static void crypto4xx_aead_exit(struct crypto_aead *tfm) | |
999 | { | |
1000 | struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); | |
1001 | ||
1002 | crypto4xx_common_exit(ctx); | |
1003 | crypto_free_aead(ctx->sw_cipher.aead); | |
1004 | } | |
1005 | ||
1006 | static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, | |
1007 | struct crypto4xx_alg_common *crypto_alg, | |
1008 | int array_size) | |
049359d6 JH |
1009 | { |
1010 | struct crypto4xx_alg *alg; | |
1011 | int i; | |
1012 | int rc = 0; | |
1013 | ||
1014 | for (i = 0; i < array_size; i++) { | |
1015 | alg = kzalloc(sizeof(struct crypto4xx_alg), GFP_KERNEL); | |
1016 | if (!alg) | |
1017 | return -ENOMEM; | |
1018 | ||
1019 | alg->alg = crypto_alg[i]; | |
049359d6 | 1020 | alg->dev = sec_dev; |
4dc10c01 HX |
1021 | |
1022 | switch (alg->alg.type) { | |
a0aae821 CL |
1023 | case CRYPTO_ALG_TYPE_AEAD: |
1024 | rc = crypto_register_aead(&alg->alg.u.aead); | |
1025 | break; | |
1026 | ||
4dc10c01 HX |
1027 | case CRYPTO_ALG_TYPE_AHASH: |
1028 | rc = crypto_register_ahash(&alg->alg.u.hash); | |
1029 | break; | |
1030 | ||
d072bfa4 CL |
1031 | case CRYPTO_ALG_TYPE_RNG: |
1032 | rc = crypto_register_rng(&alg->alg.u.rng); | |
1033 | break; | |
1034 | ||
4dc10c01 | 1035 | default: |
ce05ffe1 | 1036 | rc = crypto_register_skcipher(&alg->alg.u.cipher); |
4dc10c01 HX |
1037 | break; |
1038 | } | |
1039 | ||
a728a196 | 1040 | if (rc) |
049359d6 | 1041 | kfree(alg); |
a728a196 | 1042 | else |
049359d6 | 1043 | list_add_tail(&alg->entry, &sec_dev->alg_list); |
049359d6 JH |
1044 | } |
1045 | ||
1046 | return 0; | |
1047 | } | |
1048 | ||
1049 | static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) | |
1050 | { | |
1051 | struct crypto4xx_alg *alg, *tmp; | |
1052 | ||
1053 | list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { | |
1054 | list_del(&alg->entry); | |
4dc10c01 HX |
1055 | switch (alg->alg.type) { |
1056 | case CRYPTO_ALG_TYPE_AHASH: | |
1057 | crypto_unregister_ahash(&alg->alg.u.hash); | |
1058 | break; | |
1059 | ||
a0aae821 CL |
1060 | case CRYPTO_ALG_TYPE_AEAD: |
1061 | crypto_unregister_aead(&alg->alg.u.aead); | |
1062 | break; | |
1063 | ||
d072bfa4 CL |
1064 | case CRYPTO_ALG_TYPE_RNG: |
1065 | crypto_unregister_rng(&alg->alg.u.rng); | |
1066 | break; | |
1067 | ||
4dc10c01 | 1068 | default: |
ce05ffe1 | 1069 | crypto_unregister_skcipher(&alg->alg.u.cipher); |
4dc10c01 | 1070 | } |
049359d6 JH |
1071 | kfree(alg); |
1072 | } | |
1073 | } | |
1074 | ||
1075 | static void crypto4xx_bh_tasklet_cb(unsigned long data) | |
1076 | { | |
1077 | struct device *dev = (struct device *)data; | |
1078 | struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); | |
1079 | struct pd_uinfo *pd_uinfo; | |
1080 | struct ce_pd *pd; | |
4b5b7999 CL |
1081 | u32 tail = core_dev->dev->pdr_tail; |
1082 | u32 head = core_dev->dev->pdr_head; | |
049359d6 | 1083 | |
4b5b7999 | 1084 | do { |
9e0a0b3a CL |
1085 | pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; |
1086 | pd = &core_dev->dev->pdr[tail]; | |
8ef8d195 | 1087 | if ((pd_uinfo->state & PD_ENTRY_INUSE) && |
4b5b7999 CL |
1088 | ((READ_ONCE(pd->pd_ctl.w) & |
1089 | (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) == | |
1090 | PD_CTL_PE_DONE)) { | |
049359d6 | 1091 | crypto4xx_pd_done(core_dev->dev, tail); |
4b5b7999 | 1092 | tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail); |
049359d6 JH |
1093 | } else { |
1094 | /* if tail not done, break */ | |
1095 | break; | |
1096 | } | |
4b5b7999 | 1097 | } while (head != tail); |
049359d6 JH |
1098 | } |
1099 | ||
1100 | /** | |
1101 | * Top Half of isr. | |
1102 | */ | |
b66c685a CL |
1103 | static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data, |
1104 | u32 clr_val) | |
049359d6 JH |
1105 | { |
1106 | struct device *dev = (struct device *)data; | |
1107 | struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); | |
1108 | ||
b66c685a | 1109 | writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR); |
049359d6 JH |
1110 | tasklet_schedule(&core_dev->tasklet); |
1111 | ||
1112 | return IRQ_HANDLED; | |
1113 | } | |
1114 | ||
b66c685a CL |
1115 | static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) |
1116 | { | |
1117 | return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR); | |
1118 | } | |
1119 | ||
1120 | static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data) | |
1121 | { | |
1122 | return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR | | |
1123 | PPC4XX_TMO_ERR_INT); | |
1124 | } | |
1125 | ||
d072bfa4 CL |
1126 | static int ppc4xx_prng_data_read(struct crypto4xx_device *dev, |
1127 | u8 *data, unsigned int max) | |
1128 | { | |
1129 | unsigned int i, curr = 0; | |
1130 | u32 val[2]; | |
1131 | ||
1132 | do { | |
1133 | /* trigger PRN generation */ | |
1134 | writel(PPC4XX_PRNG_CTRL_AUTO_EN, | |
1135 | dev->ce_base + CRYPTO4XX_PRNG_CTRL); | |
1136 | ||
1137 | for (i = 0; i < 1024; i++) { | |
1138 | /* usually 19 iterations are enough */ | |
1139 | if ((readl(dev->ce_base + CRYPTO4XX_PRNG_STAT) & | |
1140 | CRYPTO4XX_PRNG_STAT_BUSY)) | |
1141 | continue; | |
1142 | ||
1143 | val[0] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_0); | |
1144 | val[1] = readl_be(dev->ce_base + CRYPTO4XX_PRNG_RES_1); | |
1145 | break; | |
1146 | } | |
1147 | if (i == 1024) | |
1148 | return -ETIMEDOUT; | |
1149 | ||
1150 | if ((max - curr) >= 8) { | |
1151 | memcpy(data, &val, 8); | |
1152 | data += 8; | |
1153 | curr += 8; | |
1154 | } else { | |
1155 | /* copy only remaining bytes */ | |
1156 | memcpy(data, &val, max - curr); | |
1157 | break; | |
1158 | } | |
1159 | } while (curr < max); | |
1160 | ||
1161 | return curr; | |
1162 | } | |
1163 | ||
1164 | static int crypto4xx_prng_generate(struct crypto_rng *tfm, | |
1165 | const u8 *src, unsigned int slen, | |
1166 | u8 *dstn, unsigned int dlen) | |
1167 | { | |
1168 | struct rng_alg *alg = crypto_rng_alg(tfm); | |
1169 | struct crypto4xx_alg *amcc_alg; | |
1170 | struct crypto4xx_device *dev; | |
1171 | int ret; | |
1172 | ||
1173 | amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.rng); | |
1174 | dev = amcc_alg->dev; | |
1175 | ||
1176 | mutex_lock(&dev->core_dev->rng_lock); | |
1177 | ret = ppc4xx_prng_data_read(dev, dstn, dlen); | |
1178 | mutex_unlock(&dev->core_dev->rng_lock); | |
1179 | return ret; | |
1180 | } | |
1181 | ||
1182 | ||
1183 | static int crypto4xx_prng_seed(struct crypto_rng *tfm, const u8 *seed, | |
1184 | unsigned int slen) | |
1185 | { | |
1186 | return 0; | |
1187 | } | |
1188 | ||
049359d6 JH |
1189 | /** |
1190 | * Supported Crypto Algorithms | |
1191 | */ | |
a0aae821 | 1192 | static struct crypto4xx_alg_common crypto4xx_alg[] = { |
049359d6 | 1193 | /* Crypto AES modes */ |
ce05ffe1 CL |
1194 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { |
1195 | .base = { | |
1196 | .cra_name = "cbc(aes)", | |
1197 | .cra_driver_name = "cbc-aes-ppc4xx", | |
1198 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1199 | .cra_flags = CRYPTO_ALG_ASYNC | |
ce05ffe1 CL |
1200 | CRYPTO_ALG_KERN_DRIVER_ONLY, |
1201 | .cra_blocksize = AES_BLOCK_SIZE, | |
1202 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), | |
1203 | .cra_module = THIS_MODULE, | |
1204 | }, | |
1205 | .min_keysize = AES_MIN_KEY_SIZE, | |
1206 | .max_keysize = AES_MAX_KEY_SIZE, | |
1207 | .ivsize = AES_IV_SIZE, | |
1208 | .setkey = crypto4xx_setkey_aes_cbc, | |
0f7a8137 CL |
1209 | .encrypt = crypto4xx_encrypt_iv_block, |
1210 | .decrypt = crypto4xx_decrypt_iv_block, | |
ce05ffe1 CL |
1211 | .init = crypto4xx_sk_init, |
1212 | .exit = crypto4xx_sk_exit, | |
f2a13e7c | 1213 | } }, |
ce05ffe1 CL |
1214 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { |
1215 | .base = { | |
1216 | .cra_name = "cfb(aes)", | |
1217 | .cra_driver_name = "cfb-aes-ppc4xx", | |
1218 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1219 | .cra_flags = CRYPTO_ALG_ASYNC | |
ce05ffe1 | 1220 | CRYPTO_ALG_KERN_DRIVER_ONLY, |
70c4997f | 1221 | .cra_blocksize = 1, |
ce05ffe1 CL |
1222 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), |
1223 | .cra_module = THIS_MODULE, | |
1224 | }, | |
1225 | .min_keysize = AES_MIN_KEY_SIZE, | |
1226 | .max_keysize = AES_MAX_KEY_SIZE, | |
1227 | .ivsize = AES_IV_SIZE, | |
1228 | .setkey = crypto4xx_setkey_aes_cfb, | |
0f7a8137 CL |
1229 | .encrypt = crypto4xx_encrypt_iv_stream, |
1230 | .decrypt = crypto4xx_decrypt_iv_stream, | |
ce05ffe1 CL |
1231 | .init = crypto4xx_sk_init, |
1232 | .exit = crypto4xx_sk_exit, | |
f2a13e7c | 1233 | } }, |
98e87e3d CL |
1234 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { |
1235 | .base = { | |
1236 | .cra_name = "ctr(aes)", | |
1237 | .cra_driver_name = "ctr-aes-ppc4xx", | |
1238 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1239 | .cra_flags = CRYPTO_ALG_NEED_FALLBACK | |
98e87e3d CL |
1240 | CRYPTO_ALG_ASYNC | |
1241 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
bfa2ba7d | 1242 | .cra_blocksize = 1, |
98e87e3d CL |
1243 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), |
1244 | .cra_module = THIS_MODULE, | |
1245 | }, | |
1246 | .min_keysize = AES_MIN_KEY_SIZE, | |
1247 | .max_keysize = AES_MAX_KEY_SIZE, | |
1248 | .ivsize = AES_IV_SIZE, | |
1249 | .setkey = crypto4xx_setkey_aes_ctr, | |
1250 | .encrypt = crypto4xx_encrypt_ctr, | |
1251 | .decrypt = crypto4xx_decrypt_ctr, | |
1252 | .init = crypto4xx_sk_init, | |
1253 | .exit = crypto4xx_sk_exit, | |
1254 | } }, | |
ce05ffe1 CL |
1255 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { |
1256 | .base = { | |
1257 | .cra_name = "rfc3686(ctr(aes))", | |
1258 | .cra_driver_name = "rfc3686-ctr-aes-ppc4xx", | |
1259 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1260 | .cra_flags = CRYPTO_ALG_ASYNC | |
ce05ffe1 | 1261 | CRYPTO_ALG_KERN_DRIVER_ONLY, |
bfa2ba7d | 1262 | .cra_blocksize = 1, |
ce05ffe1 CL |
1263 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), |
1264 | .cra_module = THIS_MODULE, | |
1265 | }, | |
1266 | .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, | |
1267 | .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, | |
1268 | .ivsize = CTR_RFC3686_IV_SIZE, | |
1269 | .setkey = crypto4xx_setkey_rfc3686, | |
1270 | .encrypt = crypto4xx_rfc3686_encrypt, | |
1271 | .decrypt = crypto4xx_rfc3686_decrypt, | |
1272 | .init = crypto4xx_sk_init, | |
1273 | .exit = crypto4xx_sk_exit, | |
f2a13e7c | 1274 | } }, |
ce05ffe1 CL |
1275 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { |
1276 | .base = { | |
1277 | .cra_name = "ecb(aes)", | |
1278 | .cra_driver_name = "ecb-aes-ppc4xx", | |
1279 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1280 | .cra_flags = CRYPTO_ALG_ASYNC | |
ce05ffe1 CL |
1281 | CRYPTO_ALG_KERN_DRIVER_ONLY, |
1282 | .cra_blocksize = AES_BLOCK_SIZE, | |
1283 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), | |
1284 | .cra_module = THIS_MODULE, | |
1285 | }, | |
1286 | .min_keysize = AES_MIN_KEY_SIZE, | |
1287 | .max_keysize = AES_MAX_KEY_SIZE, | |
1288 | .setkey = crypto4xx_setkey_aes_ecb, | |
0f7a8137 CL |
1289 | .encrypt = crypto4xx_encrypt_noiv_block, |
1290 | .decrypt = crypto4xx_decrypt_noiv_block, | |
ce05ffe1 CL |
1291 | .init = crypto4xx_sk_init, |
1292 | .exit = crypto4xx_sk_exit, | |
1293 | } }, | |
1294 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { | |
1295 | .base = { | |
1296 | .cra_name = "ofb(aes)", | |
1297 | .cra_driver_name = "ofb-aes-ppc4xx", | |
1298 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
2c95e6d9 | 1299 | .cra_flags = CRYPTO_ALG_ASYNC | |
ce05ffe1 | 1300 | CRYPTO_ALG_KERN_DRIVER_ONLY, |
70c4997f | 1301 | .cra_blocksize = 1, |
ce05ffe1 CL |
1302 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), |
1303 | .cra_module = THIS_MODULE, | |
1304 | }, | |
1305 | .min_keysize = AES_MIN_KEY_SIZE, | |
1306 | .max_keysize = AES_MAX_KEY_SIZE, | |
1307 | .ivsize = AES_IV_SIZE, | |
1308 | .setkey = crypto4xx_setkey_aes_ofb, | |
0f7a8137 CL |
1309 | .encrypt = crypto4xx_encrypt_iv_stream, |
1310 | .decrypt = crypto4xx_decrypt_iv_stream, | |
ce05ffe1 CL |
1311 | .init = crypto4xx_sk_init, |
1312 | .exit = crypto4xx_sk_exit, | |
f2a13e7c | 1313 | } }, |
65ea8b67 CL |
1314 | |
1315 | /* AEAD */ | |
1316 | { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { | |
1317 | .setkey = crypto4xx_setkey_aes_ccm, | |
1318 | .setauthsize = crypto4xx_setauthsize_aead, | |
1319 | .encrypt = crypto4xx_encrypt_aes_ccm, | |
1320 | .decrypt = crypto4xx_decrypt_aes_ccm, | |
1321 | .init = crypto4xx_aead_init, | |
1322 | .exit = crypto4xx_aead_exit, | |
1323 | .ivsize = AES_BLOCK_SIZE, | |
1324 | .maxauthsize = 16, | |
1325 | .base = { | |
1326 | .cra_name = "ccm(aes)", | |
1327 | .cra_driver_name = "ccm-aes-ppc4xx", | |
1328 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
1329 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1330 | CRYPTO_ALG_NEED_FALLBACK | | |
1331 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1332 | .cra_blocksize = 1, | |
1333 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), | |
1334 | .cra_module = THIS_MODULE, | |
1335 | }, | |
1336 | } }, | |
59231368 CL |
1337 | { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { |
1338 | .setkey = crypto4xx_setkey_aes_gcm, | |
1339 | .setauthsize = crypto4xx_setauthsize_aead, | |
1340 | .encrypt = crypto4xx_encrypt_aes_gcm, | |
1341 | .decrypt = crypto4xx_decrypt_aes_gcm, | |
1342 | .init = crypto4xx_aead_init, | |
1343 | .exit = crypto4xx_aead_exit, | |
1344 | .ivsize = GCM_AES_IV_SIZE, | |
1345 | .maxauthsize = 16, | |
1346 | .base = { | |
1347 | .cra_name = "gcm(aes)", | |
1348 | .cra_driver_name = "gcm-aes-ppc4xx", | |
1349 | .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, | |
1350 | .cra_flags = CRYPTO_ALG_ASYNC | | |
1351 | CRYPTO_ALG_NEED_FALLBACK | | |
1352 | CRYPTO_ALG_KERN_DRIVER_ONLY, | |
1353 | .cra_blocksize = 1, | |
1354 | .cra_ctxsize = sizeof(struct crypto4xx_ctx), | |
1355 | .cra_module = THIS_MODULE, | |
1356 | }, | |
1357 | } }, | |
d072bfa4 CL |
1358 | { .type = CRYPTO_ALG_TYPE_RNG, .u.rng = { |
1359 | .base = { | |
1360 | .cra_name = "stdrng", | |
1361 | .cra_driver_name = "crypto4xx_rng", | |
1362 | .cra_priority = 300, | |
1363 | .cra_ctxsize = 0, | |
1364 | .cra_module = THIS_MODULE, | |
1365 | }, | |
1366 | .generate = crypto4xx_prng_generate, | |
1367 | .seed = crypto4xx_prng_seed, | |
1368 | .seedsize = 0, | |
1369 | } }, | |
049359d6 JH |
1370 | }; |
1371 | ||
1372 | /** | |
1373 | * Module Initialization Routine | |
1374 | */ | |
1eb8a1b3 | 1375 | static int crypto4xx_probe(struct platform_device *ofdev) |
049359d6 JH |
1376 | { |
1377 | int rc; | |
1378 | struct resource res; | |
1379 | struct device *dev = &ofdev->dev; | |
1380 | struct crypto4xx_core_device *core_dev; | |
b66c685a CL |
1381 | u32 pvr; |
1382 | bool is_revb = true; | |
049359d6 | 1383 | |
7cea8cc4 | 1384 | rc = of_address_to_resource(ofdev->dev.of_node, 0, &res); |
049359d6 JH |
1385 | if (rc) |
1386 | return -ENODEV; | |
1387 | ||
1388 | if (of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto")) { | |
1389 | mtdcri(SDR0, PPC460EX_SDR0_SRST, | |
1390 | mfdcri(SDR0, PPC460EX_SDR0_SRST) | PPC460EX_CE_RESET); | |
1391 | mtdcri(SDR0, PPC460EX_SDR0_SRST, | |
1392 | mfdcri(SDR0, PPC460EX_SDR0_SRST) & ~PPC460EX_CE_RESET); | |
1393 | } else if (of_find_compatible_node(NULL, NULL, | |
1394 | "amcc,ppc405ex-crypto")) { | |
1395 | mtdcri(SDR0, PPC405EX_SDR0_SRST, | |
1396 | mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET); | |
1397 | mtdcri(SDR0, PPC405EX_SDR0_SRST, | |
1398 | mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET); | |
b66c685a | 1399 | is_revb = false; |
049359d6 JH |
1400 | } else if (of_find_compatible_node(NULL, NULL, |
1401 | "amcc,ppc460sx-crypto")) { | |
1402 | mtdcri(SDR0, PPC460SX_SDR0_SRST, | |
1403 | mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET); | |
1404 | mtdcri(SDR0, PPC460SX_SDR0_SRST, | |
1405 | mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET); | |
1406 | } else { | |
1407 | printk(KERN_ERR "Crypto Function Not supported!\n"); | |
1408 | return -EINVAL; | |
1409 | } | |
1410 | ||
1411 | core_dev = kzalloc(sizeof(struct crypto4xx_core_device), GFP_KERNEL); | |
1412 | if (!core_dev) | |
1413 | return -ENOMEM; | |
1414 | ||
1415 | dev_set_drvdata(dev, core_dev); | |
1416 | core_dev->ofdev = ofdev; | |
1417 | core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL); | |
ffe55266 | 1418 | rc = -ENOMEM; |
049359d6 JH |
1419 | if (!core_dev->dev) |
1420 | goto err_alloc_dev; | |
1421 | ||
b66c685a CL |
1422 | /* |
1423 | * Older version of 460EX/GT have a hardware bug. | |
1424 | * Hence they do not support H/W based security intr coalescing | |
1425 | */ | |
1426 | pvr = mfspr(SPRN_PVR); | |
1427 | if (is_revb && ((pvr >> 4) == 0x130218A)) { | |
1428 | u32 min = PVR_MIN(pvr); | |
1429 | ||
1430 | if (min < 4) { | |
1431 | dev_info(dev, "RevA detected - disable interrupt coalescing\n"); | |
1432 | is_revb = false; | |
1433 | } | |
1434 | } | |
1435 | ||
049359d6 | 1436 | core_dev->dev->core_dev = core_dev; |
b66c685a | 1437 | core_dev->dev->is_revb = is_revb; |
049359d6 | 1438 | core_dev->device = dev; |
d072bfa4 | 1439 | mutex_init(&core_dev->rng_lock); |
049359d6 JH |
1440 | spin_lock_init(&core_dev->lock); |
1441 | INIT_LIST_HEAD(&core_dev->dev->alg_list); | |
a0aae821 | 1442 | ratelimit_default_init(&core_dev->dev->aead_ratelimit); |
b87b2c4d CL |
1443 | rc = crypto4xx_build_sdr(core_dev->dev); |
1444 | if (rc) | |
1445 | goto err_build_sdr; | |
049359d6 JH |
1446 | rc = crypto4xx_build_pdr(core_dev->dev); |
1447 | if (rc) | |
b87b2c4d | 1448 | goto err_build_sdr; |
049359d6 JH |
1449 | |
1450 | rc = crypto4xx_build_gdr(core_dev->dev); | |
049359d6 JH |
1451 | if (rc) |
1452 | goto err_build_sdr; | |
1453 | ||
1454 | /* Init tasklet for bottom half processing */ | |
1455 | tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb, | |
1456 | (unsigned long) dev); | |
1457 | ||
7cea8cc4 | 1458 | core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0); |
049359d6 JH |
1459 | if (!core_dev->dev->ce_base) { |
1460 | dev_err(dev, "failed to of_iomap\n"); | |
b48ae1df | 1461 | rc = -ENOMEM; |
049359d6 JH |
1462 | goto err_iomap; |
1463 | } | |
1464 | ||
b0a191ce CL |
1465 | /* Register for Crypto isr, Crypto Engine IRQ */ |
1466 | core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); | |
b66c685a CL |
1467 | rc = request_irq(core_dev->irq, is_revb ? |
1468 | crypto4xx_ce_interrupt_handler_revb : | |
1469 | crypto4xx_ce_interrupt_handler, 0, | |
57268aba | 1470 | KBUILD_MODNAME, dev); |
b0a191ce CL |
1471 | if (rc) |
1472 | goto err_request_irq; | |
1473 | ||
049359d6 JH |
1474 | /* need to setup pdr, rdr, gdr and sdr before this */ |
1475 | crypto4xx_hw_init(core_dev->dev); | |
1476 | ||
1477 | /* Register security algorithms with Linux CryptoAPI */ | |
1478 | rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg, | |
1479 | ARRAY_SIZE(crypto4xx_alg)); | |
1480 | if (rc) | |
1481 | goto err_start_dev; | |
1482 | ||
5343e674 | 1483 | ppc4xx_trng_probe(core_dev); |
049359d6 JH |
1484 | return 0; |
1485 | ||
1486 | err_start_dev: | |
049359d6 | 1487 | free_irq(core_dev->irq, dev); |
bdd3f7fa | 1488 | err_request_irq: |
049359d6 | 1489 | irq_dispose_mapping(core_dev->irq); |
b0a191ce CL |
1490 | iounmap(core_dev->dev->ce_base); |
1491 | err_iomap: | |
049359d6 | 1492 | tasklet_kill(&core_dev->tasklet); |
049359d6 | 1493 | err_build_sdr: |
5d59ad6e | 1494 | crypto4xx_destroy_sdr(core_dev->dev); |
049359d6 | 1495 | crypto4xx_destroy_gdr(core_dev->dev); |
5d59ad6e | 1496 | crypto4xx_destroy_pdr(core_dev->dev); |
049359d6 JH |
1497 | kfree(core_dev->dev); |
1498 | err_alloc_dev: | |
1499 | kfree(core_dev); | |
1500 | ||
1501 | return rc; | |
1502 | } | |
1503 | ||
1eb8a1b3 | 1504 | static int crypto4xx_remove(struct platform_device *ofdev) |
049359d6 JH |
1505 | { |
1506 | struct device *dev = &ofdev->dev; | |
1507 | struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); | |
1508 | ||
5343e674 CL |
1509 | ppc4xx_trng_remove(core_dev); |
1510 | ||
049359d6 JH |
1511 | free_irq(core_dev->irq, dev); |
1512 | irq_dispose_mapping(core_dev->irq); | |
1513 | ||
1514 | tasklet_kill(&core_dev->tasklet); | |
1515 | /* Un-register with Linux CryptoAPI */ | |
1516 | crypto4xx_unregister_alg(core_dev->dev); | |
d072bfa4 | 1517 | mutex_destroy(&core_dev->rng_lock); |
049359d6 JH |
1518 | /* Free all allocated memory */ |
1519 | crypto4xx_stop_all(core_dev); | |
1520 | ||
1521 | return 0; | |
1522 | } | |
1523 | ||
6c3f975a | 1524 | static const struct of_device_id crypto4xx_match[] = { |
049359d6 JH |
1525 | { .compatible = "amcc,ppc4xx-crypto",}, |
1526 | { }, | |
1527 | }; | |
3dc1597b | 1528 | MODULE_DEVICE_TABLE(of, crypto4xx_match); |
049359d6 | 1529 | |
00006124 | 1530 | static struct platform_driver crypto4xx_driver = { |
4018294b | 1531 | .driver = { |
333b1928 | 1532 | .name = KBUILD_MODNAME, |
4018294b GL |
1533 | .of_match_table = crypto4xx_match, |
1534 | }, | |
049359d6 | 1535 | .probe = crypto4xx_probe, |
1eb8a1b3 | 1536 | .remove = crypto4xx_remove, |
049359d6 JH |
1537 | }; |
1538 | ||
741e8c2d | 1539 | module_platform_driver(crypto4xx_driver); |
049359d6 JH |
1540 | |
1541 | MODULE_LICENSE("GPL"); | |
1542 | MODULE_AUTHOR("James Hsiao <jhsiao@amcc.com>"); | |
1543 | MODULE_DESCRIPTION("Driver for AMCC PPC4xx crypto accelerator"); |