Commit | Line | Data |
---|---|---|
a61127c2 | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2492c845 DW |
2 | /* |
3 | * Copyright © 2006, Intel Corporation. | |
2492c845 DW |
4 | */ |
5 | #ifndef _ADMA_H | |
6 | #define _ADMA_H | |
7 | #include <linux/types.h> | |
8 | #include <linux/io.h> | |
aad7ad2a | 9 | #include <linux/platform_data/dma-iop32x.h> |
2492c845 DW |
10 | |
11 | /* Memory copy units */ | |
12 | #define DMA_CCR(chan) (chan->mmr_base + 0x0) | |
13 | #define DMA_CSR(chan) (chan->mmr_base + 0x4) | |
14 | #define DMA_DAR(chan) (chan->mmr_base + 0xc) | |
15 | #define DMA_NDAR(chan) (chan->mmr_base + 0x10) | |
16 | #define DMA_PADR(chan) (chan->mmr_base + 0x14) | |
17 | #define DMA_PUADR(chan) (chan->mmr_base + 0x18) | |
18 | #define DMA_LADR(chan) (chan->mmr_base + 0x1c) | |
19 | #define DMA_BCR(chan) (chan->mmr_base + 0x20) | |
20 | #define DMA_DCR(chan) (chan->mmr_base + 0x24) | |
21 | ||
22 | /* Application accelerator unit */ | |
23 | #define AAU_ACR(chan) (chan->mmr_base + 0x0) | |
24 | #define AAU_ASR(chan) (chan->mmr_base + 0x4) | |
25 | #define AAU_ADAR(chan) (chan->mmr_base + 0x8) | |
26 | #define AAU_ANDAR(chan) (chan->mmr_base + 0xc) | |
27 | #define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2))) | |
28 | #define AAU_DAR(chan) (chan->mmr_base + 0x20) | |
29 | #define AAU_ABCR(chan) (chan->mmr_base + 0x24) | |
30 | #define AAU_ADCR(chan) (chan->mmr_base + 0x28) | |
31 | #define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2))) | |
32 | #define AAU_EDCR0_IDX 8 | |
33 | #define AAU_EDCR1_IDX 17 | |
34 | #define AAU_EDCR2_IDX 26 | |
35 | ||
2492c845 DW |
36 | struct iop3xx_aau_desc_ctrl { |
37 | unsigned int int_en:1; | |
38 | unsigned int blk1_cmd_ctrl:3; | |
39 | unsigned int blk2_cmd_ctrl:3; | |
40 | unsigned int blk3_cmd_ctrl:3; | |
41 | unsigned int blk4_cmd_ctrl:3; | |
42 | unsigned int blk5_cmd_ctrl:3; | |
43 | unsigned int blk6_cmd_ctrl:3; | |
44 | unsigned int blk7_cmd_ctrl:3; | |
45 | unsigned int blk8_cmd_ctrl:3; | |
46 | unsigned int blk_ctrl:2; | |
47 | unsigned int dual_xor_en:1; | |
48 | unsigned int tx_complete:1; | |
49 | unsigned int zero_result_err:1; | |
50 | unsigned int zero_result_en:1; | |
51 | unsigned int dest_write_en:1; | |
52 | }; | |
53 | ||
54 | struct iop3xx_aau_e_desc_ctrl { | |
55 | unsigned int reserved:1; | |
56 | unsigned int blk1_cmd_ctrl:3; | |
57 | unsigned int blk2_cmd_ctrl:3; | |
58 | unsigned int blk3_cmd_ctrl:3; | |
59 | unsigned int blk4_cmd_ctrl:3; | |
60 | unsigned int blk5_cmd_ctrl:3; | |
61 | unsigned int blk6_cmd_ctrl:3; | |
62 | unsigned int blk7_cmd_ctrl:3; | |
63 | unsigned int blk8_cmd_ctrl:3; | |
64 | unsigned int reserved2:7; | |
65 | }; | |
66 | ||
67 | struct iop3xx_dma_desc_ctrl { | |
68 | unsigned int pci_transaction:4; | |
69 | unsigned int int_en:1; | |
70 | unsigned int dac_cycle_en:1; | |
71 | unsigned int mem_to_mem_en:1; | |
72 | unsigned int crc_data_tx_en:1; | |
73 | unsigned int crc_gen_en:1; | |
74 | unsigned int crc_seed_dis:1; | |
75 | unsigned int reserved:21; | |
76 | unsigned int crc_tx_complete:1; | |
77 | }; | |
78 | ||
79 | struct iop3xx_desc_dma { | |
80 | u32 next_desc; | |
81 | union { | |
82 | u32 pci_src_addr; | |
83 | u32 pci_dest_addr; | |
84 | u32 src_addr; | |
85 | }; | |
86 | union { | |
87 | u32 upper_pci_src_addr; | |
88 | u32 upper_pci_dest_addr; | |
89 | }; | |
90 | union { | |
91 | u32 local_pci_src_addr; | |
92 | u32 local_pci_dest_addr; | |
93 | u32 dest_addr; | |
94 | }; | |
95 | u32 byte_count; | |
96 | union { | |
97 | u32 desc_ctrl; | |
98 | struct iop3xx_dma_desc_ctrl desc_ctrl_field; | |
99 | }; | |
100 | u32 crc_addr; | |
101 | }; | |
102 | ||
103 | struct iop3xx_desc_aau { | |
104 | u32 next_desc; | |
105 | u32 src[4]; | |
106 | u32 dest_addr; | |
107 | u32 byte_count; | |
108 | union { | |
109 | u32 desc_ctrl; | |
110 | struct iop3xx_aau_desc_ctrl desc_ctrl_field; | |
111 | }; | |
112 | union { | |
113 | u32 src_addr; | |
114 | u32 e_desc_ctrl; | |
115 | struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; | |
116 | } src_edc[31]; | |
117 | }; | |
118 | ||
119 | struct iop3xx_aau_gfmr { | |
120 | unsigned int gfmr1:8; | |
121 | unsigned int gfmr2:8; | |
122 | unsigned int gfmr3:8; | |
123 | unsigned int gfmr4:8; | |
124 | }; | |
125 | ||
126 | struct iop3xx_desc_pq_xor { | |
127 | u32 next_desc; | |
128 | u32 src[3]; | |
129 | union { | |
130 | u32 data_mult1; | |
131 | struct iop3xx_aau_gfmr data_mult1_field; | |
132 | }; | |
133 | u32 dest_addr; | |
134 | u32 byte_count; | |
135 | union { | |
136 | u32 desc_ctrl; | |
137 | struct iop3xx_aau_desc_ctrl desc_ctrl_field; | |
138 | }; | |
139 | union { | |
140 | u32 src_addr; | |
141 | u32 e_desc_ctrl; | |
142 | struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; | |
143 | u32 data_multiplier; | |
144 | struct iop3xx_aau_gfmr data_mult_field; | |
145 | u32 reserved; | |
146 | } src_edc_gfmr[19]; | |
147 | }; | |
148 | ||
149 | struct iop3xx_desc_dual_xor { | |
150 | u32 next_desc; | |
151 | u32 src0_addr; | |
152 | u32 src1_addr; | |
153 | u32 h_src_addr; | |
154 | u32 d_src_addr; | |
155 | u32 h_dest_addr; | |
156 | u32 byte_count; | |
157 | union { | |
158 | u32 desc_ctrl; | |
159 | struct iop3xx_aau_desc_ctrl desc_ctrl_field; | |
160 | }; | |
161 | u32 d_dest_addr; | |
162 | }; | |
163 | ||
164 | union iop3xx_desc { | |
165 | struct iop3xx_desc_aau *aau; | |
166 | struct iop3xx_desc_dma *dma; | |
167 | struct iop3xx_desc_pq_xor *pq_xor; | |
168 | struct iop3xx_desc_dual_xor *dual_xor; | |
169 | void *ptr; | |
170 | }; | |
171 | ||
7bf649ae DW |
172 | /* No support for p+q operations */ |
173 | static inline int | |
174 | iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op) | |
175 | { | |
176 | BUG(); | |
177 | return 0; | |
178 | } | |
179 | ||
180 | static inline void | |
181 | iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt, | |
182 | unsigned long flags) | |
183 | { | |
184 | BUG(); | |
185 | } | |
186 | ||
187 | static inline void | |
188 | iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr) | |
189 | { | |
190 | BUG(); | |
191 | } | |
192 | ||
193 | static inline void | |
194 | iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx, | |
195 | dma_addr_t addr, unsigned char coef) | |
196 | { | |
197 | BUG(); | |
198 | } | |
199 | ||
200 | static inline int | |
201 | iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op) | |
202 | { | |
203 | BUG(); | |
204 | return 0; | |
205 | } | |
206 | ||
207 | static inline void | |
208 | iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, | |
209 | unsigned long flags) | |
210 | { | |
211 | BUG(); | |
212 | } | |
213 | ||
214 | static inline void | |
215 | iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) | |
216 | { | |
217 | BUG(); | |
218 | } | |
219 | ||
220 | #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr | |
221 | ||
222 | static inline void | |
223 | iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx, | |
224 | dma_addr_t *src) | |
225 | { | |
226 | BUG(); | |
227 | } | |
228 | ||
2492c845 DW |
229 | static inline int iop_adma_get_max_xor(void) |
230 | { | |
231 | return 32; | |
232 | } | |
233 | ||
7bf649ae DW |
234 | static inline int iop_adma_get_max_pq(void) |
235 | { | |
236 | BUG(); | |
237 | return 0; | |
238 | } | |
239 | ||
2492c845 DW |
240 | static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan) |
241 | { | |
242 | int id = chan->device->id; | |
243 | ||
244 | switch (id) { | |
245 | case DMA0_ID: | |
246 | case DMA1_ID: | |
247 | return __raw_readl(DMA_DAR(chan)); | |
248 | case AAU_ID: | |
249 | return __raw_readl(AAU_ADAR(chan)); | |
250 | default: | |
251 | BUG(); | |
252 | } | |
253 | return 0; | |
254 | } | |
255 | ||
256 | static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan, | |
257 | u32 next_desc_addr) | |
258 | { | |
259 | int id = chan->device->id; | |
260 | ||
261 | switch (id) { | |
262 | case DMA0_ID: | |
263 | case DMA1_ID: | |
264 | __raw_writel(next_desc_addr, DMA_NDAR(chan)); | |
265 | break; | |
266 | case AAU_ID: | |
267 | __raw_writel(next_desc_addr, AAU_ANDAR(chan)); | |
268 | break; | |
269 | } | |
270 | ||
271 | } | |
272 | ||
273 | #define IOP_ADMA_STATUS_BUSY (1 << 10) | |
274 | #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024) | |
275 | #define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024) | |
276 | #define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024) | |
277 | ||
278 | static inline int iop_chan_is_busy(struct iop_adma_chan *chan) | |
279 | { | |
280 | u32 status = __raw_readl(DMA_CSR(chan)); | |
281 | return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0; | |
282 | } | |
283 | ||
284 | static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc, | |
285 | int num_slots) | |
286 | { | |
287 | /* num_slots will only ever be 1, 2, 4, or 8 */ | |
288 | return (desc->idx & (num_slots - 1)) ? 0 : 1; | |
289 | } | |
290 | ||
291 | /* to do: support large (i.e. > hw max) buffer sizes */ | |
292 | static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op) | |
293 | { | |
294 | *slots_per_op = 1; | |
295 | return 1; | |
296 | } | |
297 | ||
298 | /* to do: support large (i.e. > hw max) buffer sizes */ | |
299 | static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op) | |
300 | { | |
301 | *slots_per_op = 1; | |
302 | return 1; | |
303 | } | |
304 | ||
305 | static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, | |
306 | int *slots_per_op) | |
307 | { | |
2a46fa13 | 308 | static const char slot_count_table[] = { |
2492c845 DW |
309 | 1, 1, 1, 1, /* 01 - 04 */ |
310 | 2, 2, 2, 2, /* 05 - 08 */ | |
311 | 4, 4, 4, 4, /* 09 - 12 */ | |
312 | 4, 4, 4, 4, /* 13 - 16 */ | |
313 | 8, 8, 8, 8, /* 17 - 20 */ | |
314 | 8, 8, 8, 8, /* 21 - 24 */ | |
315 | 8, 8, 8, 8, /* 25 - 28 */ | |
316 | 8, 8, 8, 8, /* 29 - 32 */ | |
317 | }; | |
2a46fa13 | 318 | *slots_per_op = slot_count_table[src_cnt - 1]; |
2492c845 DW |
319 | return *slots_per_op; |
320 | } | |
321 | ||
322 | static inline int | |
323 | iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan) | |
324 | { | |
325 | switch (chan->device->id) { | |
326 | case DMA0_ID: | |
327 | case DMA1_ID: | |
328 | return iop_chan_memcpy_slot_count(0, slots_per_op); | |
329 | case AAU_ID: | |
330 | return iop3xx_aau_xor_slot_count(0, 2, slots_per_op); | |
331 | default: | |
332 | BUG(); | |
333 | } | |
334 | return 0; | |
335 | } | |
336 | ||
337 | static inline int iop_chan_xor_slot_count(size_t len, int src_cnt, | |
338 | int *slots_per_op) | |
339 | { | |
340 | int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); | |
341 | ||
342 | if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT) | |
343 | return slot_cnt; | |
344 | ||
345 | len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; | |
346 | while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) { | |
347 | len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; | |
348 | slot_cnt += *slots_per_op; | |
349 | } | |
350 | ||
f1acb878 | 351 | slot_cnt += *slots_per_op; |
2492c845 DW |
352 | |
353 | return slot_cnt; | |
354 | } | |
355 | ||
356 | /* zero sum on iop3xx is limited to 1k at a time so it requires multiple | |
357 | * descriptors | |
358 | */ | |
359 | static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt, | |
360 | int *slots_per_op) | |
361 | { | |
362 | int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); | |
363 | ||
364 | if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) | |
365 | return slot_cnt; | |
366 | ||
367 | len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; | |
368 | while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { | |
369 | len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; | |
370 | slot_cnt += *slots_per_op; | |
371 | } | |
372 | ||
f1acb878 | 373 | slot_cnt += *slots_per_op; |
2492c845 DW |
374 | |
375 | return slot_cnt; | |
376 | } | |
377 | ||
2492c845 DW |
378 | static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, |
379 | struct iop_adma_chan *chan) | |
380 | { | |
381 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
382 | ||
383 | switch (chan->device->id) { | |
384 | case DMA0_ID: | |
385 | case DMA1_ID: | |
386 | return hw_desc.dma->byte_count; | |
387 | case AAU_ID: | |
388 | return hw_desc.aau->byte_count; | |
389 | default: | |
390 | BUG(); | |
391 | } | |
392 | return 0; | |
393 | } | |
394 | ||
395 | /* translate the src_idx to a descriptor word index */ | |
396 | static inline int __desc_idx(int src_idx) | |
397 | { | |
b1d18dc0 | 398 | static const int desc_idx_table[] = { 0, 0, 0, 0, |
2492c845 DW |
399 | 0, 1, 2, 3, |
400 | 5, 6, 7, 8, | |
401 | 9, 10, 11, 12, | |
402 | 14, 15, 16, 17, | |
403 | 18, 19, 20, 21, | |
404 | 23, 24, 25, 26, | |
405 | 27, 28, 29, 30, | |
406 | }; | |
407 | ||
408 | return desc_idx_table[src_idx]; | |
409 | } | |
410 | ||
411 | static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc, | |
412 | struct iop_adma_chan *chan, | |
413 | int src_idx) | |
414 | { | |
415 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
416 | ||
417 | switch (chan->device->id) { | |
418 | case DMA0_ID: | |
419 | case DMA1_ID: | |
420 | return hw_desc.dma->src_addr; | |
421 | case AAU_ID: | |
422 | break; | |
423 | default: | |
424 | BUG(); | |
425 | } | |
426 | ||
427 | if (src_idx < 4) | |
428 | return hw_desc.aau->src[src_idx]; | |
429 | else | |
430 | return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr; | |
431 | } | |
432 | ||
433 | static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc, | |
434 | int src_idx, dma_addr_t addr) | |
435 | { | |
436 | if (src_idx < 4) | |
437 | hw_desc->src[src_idx] = addr; | |
438 | else | |
439 | hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr; | |
440 | } | |
441 | ||
442 | static inline void | |
d4c56f97 | 443 | iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags) |
2492c845 DW |
444 | { |
445 | struct iop3xx_desc_dma *hw_desc = desc->hw_desc; | |
446 | union { | |
447 | u32 value; | |
448 | struct iop3xx_dma_desc_ctrl field; | |
449 | } u_desc_ctrl; | |
450 | ||
451 | u_desc_ctrl.value = 0; | |
452 | u_desc_ctrl.field.mem_to_mem_en = 1; | |
453 | u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */ | |
d4c56f97 | 454 | u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; |
2492c845 DW |
455 | hw_desc->desc_ctrl = u_desc_ctrl.value; |
456 | hw_desc->upper_pci_src_addr = 0; | |
457 | hw_desc->crc_addr = 0; | |
458 | } | |
459 | ||
460 | static inline void | |
d4c56f97 | 461 | iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags) |
2492c845 DW |
462 | { |
463 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc; | |
464 | union { | |
465 | u32 value; | |
466 | struct iop3xx_aau_desc_ctrl field; | |
467 | } u_desc_ctrl; | |
468 | ||
469 | u_desc_ctrl.value = 0; | |
470 | u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */ | |
471 | u_desc_ctrl.field.dest_write_en = 1; | |
d4c56f97 | 472 | u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; |
2492c845 DW |
473 | hw_desc->desc_ctrl = u_desc_ctrl.value; |
474 | } | |
475 | ||
476 | static inline u32 | |
d4c56f97 DW |
477 | iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt, |
478 | unsigned long flags) | |
2492c845 DW |
479 | { |
480 | int i, shift; | |
481 | u32 edcr; | |
482 | union { | |
483 | u32 value; | |
484 | struct iop3xx_aau_desc_ctrl field; | |
485 | } u_desc_ctrl; | |
486 | ||
487 | u_desc_ctrl.value = 0; | |
488 | switch (src_cnt) { | |
489 | case 25 ... 32: | |
490 | u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ | |
491 | edcr = 0; | |
492 | shift = 1; | |
493 | for (i = 24; i < src_cnt; i++) { | |
494 | edcr |= (1 << shift); | |
495 | shift += 3; | |
496 | } | |
497 | hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr; | |
498 | src_cnt = 24; | |
df561f66 | 499 | fallthrough; |
2492c845 DW |
500 | case 17 ... 24: |
501 | if (!u_desc_ctrl.field.blk_ctrl) { | |
502 | hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; | |
503 | u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ | |
504 | } | |
505 | edcr = 0; | |
506 | shift = 1; | |
507 | for (i = 16; i < src_cnt; i++) { | |
508 | edcr |= (1 << shift); | |
509 | shift += 3; | |
510 | } | |
511 | hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr; | |
512 | src_cnt = 16; | |
df561f66 | 513 | fallthrough; |
2492c845 DW |
514 | case 9 ... 16: |
515 | if (!u_desc_ctrl.field.blk_ctrl) | |
516 | u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ | |
517 | edcr = 0; | |
518 | shift = 1; | |
519 | for (i = 8; i < src_cnt; i++) { | |
520 | edcr |= (1 << shift); | |
521 | shift += 3; | |
522 | } | |
523 | hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr; | |
524 | src_cnt = 8; | |
df561f66 | 525 | fallthrough; |
2492c845 DW |
526 | case 2 ... 8: |
527 | shift = 1; | |
528 | for (i = 0; i < src_cnt; i++) { | |
529 | u_desc_ctrl.value |= (1 << shift); | |
530 | shift += 3; | |
531 | } | |
532 | ||
533 | if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) | |
534 | u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ | |
535 | } | |
536 | ||
537 | u_desc_ctrl.field.dest_write_en = 1; | |
538 | u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */ | |
d4c56f97 | 539 | u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; |
2492c845 DW |
540 | hw_desc->desc_ctrl = u_desc_ctrl.value; |
541 | ||
542 | return u_desc_ctrl.value; | |
543 | } | |
544 | ||
545 | static inline void | |
d4c56f97 DW |
546 | iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt, |
547 | unsigned long flags) | |
2492c845 | 548 | { |
d4c56f97 | 549 | iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags); |
2492c845 DW |
550 | } |
551 | ||
552 | /* return the number of operations */ | |
553 | static inline int | |
d4c56f97 DW |
554 | iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, |
555 | unsigned long flags) | |
2492c845 DW |
556 | { |
557 | int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; | |
558 | struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter; | |
559 | union { | |
560 | u32 value; | |
561 | struct iop3xx_aau_desc_ctrl field; | |
562 | } u_desc_ctrl; | |
563 | int i, j; | |
564 | ||
565 | hw_desc = desc->hw_desc; | |
566 | ||
567 | for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0; | |
568 | i += slots_per_op, j++) { | |
569 | iter = iop_hw_desc_slot_idx(hw_desc, i); | |
d4c56f97 | 570 | u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags); |
2492c845 DW |
571 | u_desc_ctrl.field.dest_write_en = 0; |
572 | u_desc_ctrl.field.zero_result_en = 1; | |
d4c56f97 | 573 | u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; |
2492c845 DW |
574 | iter->desc_ctrl = u_desc_ctrl.value; |
575 | ||
576 | /* for the subsequent descriptors preserve the store queue | |
577 | * and chain them together | |
578 | */ | |
579 | if (i) { | |
580 | prev_hw_desc = | |
581 | iop_hw_desc_slot_idx(hw_desc, i - slots_per_op); | |
582 | prev_hw_desc->next_desc = | |
583 | (u32) (desc->async_tx.phys + (i << 5)); | |
584 | } | |
585 | } | |
586 | ||
587 | return j; | |
588 | } | |
589 | ||
590 | static inline void | |
d4c56f97 DW |
591 | iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt, |
592 | unsigned long flags) | |
2492c845 DW |
593 | { |
594 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc; | |
595 | union { | |
596 | u32 value; | |
597 | struct iop3xx_aau_desc_ctrl field; | |
598 | } u_desc_ctrl; | |
599 | ||
600 | u_desc_ctrl.value = 0; | |
601 | switch (src_cnt) { | |
602 | case 25 ... 32: | |
603 | u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ | |
604 | hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; | |
df561f66 | 605 | fallthrough; |
2492c845 DW |
606 | case 17 ... 24: |
607 | if (!u_desc_ctrl.field.blk_ctrl) { | |
608 | hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; | |
609 | u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ | |
610 | } | |
611 | hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0; | |
df561f66 | 612 | fallthrough; |
2492c845 DW |
613 | case 9 ... 16: |
614 | if (!u_desc_ctrl.field.blk_ctrl) | |
615 | u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ | |
616 | hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0; | |
df561f66 | 617 | fallthrough; |
2492c845 DW |
618 | case 1 ... 8: |
619 | if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) | |
620 | u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ | |
621 | } | |
622 | ||
623 | u_desc_ctrl.field.dest_write_en = 0; | |
d4c56f97 | 624 | u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; |
2492c845 DW |
625 | hw_desc->desc_ctrl = u_desc_ctrl.value; |
626 | } | |
627 | ||
628 | static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc, | |
629 | struct iop_adma_chan *chan, | |
630 | u32 byte_count) | |
631 | { | |
632 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
633 | ||
634 | switch (chan->device->id) { | |
635 | case DMA0_ID: | |
636 | case DMA1_ID: | |
637 | hw_desc.dma->byte_count = byte_count; | |
638 | break; | |
639 | case AAU_ID: | |
640 | hw_desc.aau->byte_count = byte_count; | |
641 | break; | |
642 | default: | |
643 | BUG(); | |
644 | } | |
645 | } | |
646 | ||
647 | static inline void | |
648 | iop_desc_init_interrupt(struct iop_adma_desc_slot *desc, | |
649 | struct iop_adma_chan *chan) | |
650 | { | |
651 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
652 | ||
653 | switch (chan->device->id) { | |
654 | case DMA0_ID: | |
655 | case DMA1_ID: | |
656 | iop_desc_init_memcpy(desc, 1); | |
657 | hw_desc.dma->byte_count = 0; | |
658 | hw_desc.dma->dest_addr = 0; | |
659 | hw_desc.dma->src_addr = 0; | |
660 | break; | |
661 | case AAU_ID: | |
662 | iop_desc_init_null_xor(desc, 2, 1); | |
663 | hw_desc.aau->byte_count = 0; | |
664 | hw_desc.aau->dest_addr = 0; | |
665 | hw_desc.aau->src[0] = 0; | |
666 | hw_desc.aau->src[1] = 0; | |
667 | break; | |
668 | default: | |
669 | BUG(); | |
670 | } | |
671 | } | |
672 | ||
673 | static inline void | |
674 | iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) | |
675 | { | |
676 | int slots_per_op = desc->slots_per_op; | |
677 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; | |
678 | int i = 0; | |
679 | ||
680 | if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { | |
681 | hw_desc->byte_count = len; | |
682 | } else { | |
683 | do { | |
684 | iter = iop_hw_desc_slot_idx(hw_desc, i); | |
685 | iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; | |
686 | len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; | |
687 | i += slots_per_op; | |
688 | } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT); | |
689 | ||
f1acb878 RK |
690 | iter = iop_hw_desc_slot_idx(hw_desc, i); |
691 | iter->byte_count = len; | |
2492c845 DW |
692 | } |
693 | } | |
694 | ||
695 | static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc, | |
696 | struct iop_adma_chan *chan, | |
697 | dma_addr_t addr) | |
698 | { | |
699 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
700 | ||
701 | switch (chan->device->id) { | |
702 | case DMA0_ID: | |
703 | case DMA1_ID: | |
704 | hw_desc.dma->dest_addr = addr; | |
705 | break; | |
706 | case AAU_ID: | |
707 | hw_desc.aau->dest_addr = addr; | |
708 | break; | |
709 | default: | |
710 | BUG(); | |
711 | } | |
712 | } | |
713 | ||
714 | static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc, | |
715 | dma_addr_t addr) | |
716 | { | |
717 | struct iop3xx_desc_dma *hw_desc = desc->hw_desc; | |
718 | hw_desc->src_addr = addr; | |
719 | } | |
720 | ||
721 | static inline void | |
722 | iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx, | |
723 | dma_addr_t addr) | |
724 | { | |
725 | ||
726 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; | |
727 | int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; | |
728 | int i; | |
729 | ||
730 | for (i = 0; (slot_cnt -= slots_per_op) >= 0; | |
731 | i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { | |
732 | iter = iop_hw_desc_slot_idx(hw_desc, i); | |
733 | iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); | |
734 | } | |
735 | } | |
736 | ||
737 | static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc, | |
738 | int src_idx, dma_addr_t addr) | |
739 | { | |
740 | ||
741 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; | |
742 | int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; | |
743 | int i; | |
744 | ||
745 | for (i = 0; (slot_cnt -= slots_per_op) >= 0; | |
746 | i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) { | |
747 | iter = iop_hw_desc_slot_idx(hw_desc, i); | |
748 | iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); | |
749 | } | |
750 | } | |
751 | ||
752 | static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc, | |
753 | u32 next_desc_addr) | |
754 | { | |
755 | /* hw_desc->next_desc is the same location for all channels */ | |
756 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
65e50381 DW |
757 | |
758 | iop_paranoia(hw_desc.dma->next_desc); | |
2492c845 DW |
759 | hw_desc.dma->next_desc = next_desc_addr; |
760 | } | |
761 | ||
762 | static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc) | |
763 | { | |
764 | /* hw_desc->next_desc is the same location for all channels */ | |
765 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
766 | return hw_desc.dma->next_desc; | |
767 | } | |
768 | ||
769 | static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc) | |
770 | { | |
771 | /* hw_desc->next_desc is the same location for all channels */ | |
772 | union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; | |
773 | hw_desc.dma->next_desc = 0; | |
774 | } | |
775 | ||
776 | static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc, | |
777 | u32 val) | |
778 | { | |
779 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc; | |
780 | hw_desc->src[0] = val; | |
781 | } | |
782 | ||
ad283ea4 DW |
783 | static inline enum sum_check_flags |
784 | iop_desc_get_zero_result(struct iop_adma_desc_slot *desc) | |
2492c845 DW |
785 | { |
786 | struct iop3xx_desc_aau *hw_desc = desc->hw_desc; | |
787 | struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field; | |
788 | ||
65e50381 | 789 | iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en)); |
ad283ea4 | 790 | return desc_ctrl.zero_result_err << SUM_CHECK_P; |
2492c845 DW |
791 | } |
792 | ||
793 | static inline void iop_chan_append(struct iop_adma_chan *chan) | |
794 | { | |
795 | u32 dma_chan_ctrl; | |
2492c845 DW |
796 | |
797 | dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); | |
798 | dma_chan_ctrl |= 0x2; | |
799 | __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); | |
800 | } | |
801 | ||
2492c845 DW |
802 | static inline u32 iop_chan_get_status(struct iop_adma_chan *chan) |
803 | { | |
804 | return __raw_readl(DMA_CSR(chan)); | |
805 | } | |
806 | ||
807 | static inline void iop_chan_disable(struct iop_adma_chan *chan) | |
808 | { | |
809 | u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); | |
810 | dma_chan_ctrl &= ~1; | |
811 | __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); | |
812 | } | |
813 | ||
814 | static inline void iop_chan_enable(struct iop_adma_chan *chan) | |
815 | { | |
816 | u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); | |
817 | ||
818 | dma_chan_ctrl |= 1; | |
819 | __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); | |
820 | } | |
821 | ||
822 | static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan) | |
823 | { | |
824 | u32 status = __raw_readl(DMA_CSR(chan)); | |
825 | status &= (1 << 9); | |
826 | __raw_writel(status, DMA_CSR(chan)); | |
827 | } | |
828 | ||
829 | static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan) | |
830 | { | |
831 | u32 status = __raw_readl(DMA_CSR(chan)); | |
832 | status &= (1 << 8); | |
833 | __raw_writel(status, DMA_CSR(chan)); | |
834 | } | |
835 | ||
836 | static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan) | |
837 | { | |
838 | u32 status = __raw_readl(DMA_CSR(chan)); | |
839 | ||
840 | switch (chan->device->id) { | |
841 | case DMA0_ID: | |
842 | case DMA1_ID: | |
843 | status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1); | |
844 | break; | |
845 | case AAU_ID: | |
846 | status &= (1 << 5); | |
847 | break; | |
848 | default: | |
849 | BUG(); | |
850 | } | |
851 | ||
852 | __raw_writel(status, DMA_CSR(chan)); | |
853 | } | |
854 | ||
855 | static inline int | |
856 | iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan) | |
857 | { | |
858 | return 0; | |
859 | } | |
860 | ||
861 | static inline int | |
862 | iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan) | |
863 | { | |
864 | return 0; | |
865 | } | |
866 | ||
867 | static inline int | |
868 | iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan) | |
869 | { | |
870 | return 0; | |
871 | } | |
872 | ||
873 | static inline int | |
874 | iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan) | |
875 | { | |
876 | return test_bit(5, &status); | |
877 | } | |
878 | ||
879 | static inline int | |
880 | iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan) | |
881 | { | |
882 | switch (chan->device->id) { | |
883 | case DMA0_ID: | |
884 | case DMA1_ID: | |
885 | return test_bit(2, &status); | |
886 | default: | |
887 | return 0; | |
888 | } | |
889 | } | |
890 | ||
891 | static inline int | |
892 | iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan) | |
893 | { | |
894 | switch (chan->device->id) { | |
895 | case DMA0_ID: | |
896 | case DMA1_ID: | |
897 | return test_bit(3, &status); | |
898 | default: | |
899 | return 0; | |
900 | } | |
901 | } | |
902 | ||
903 | static inline int | |
904 | iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan) | |
905 | { | |
906 | switch (chan->device->id) { | |
907 | case DMA0_ID: | |
908 | case DMA1_ID: | |
909 | return test_bit(1, &status); | |
910 | default: | |
911 | return 0; | |
912 | } | |
913 | } | |
914 | #endif /* _ADMA_H */ |