Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
d894fc60 AS |
2 | /* |
3 | * Ingenic JZ4780 DMA controller | |
4 | * | |
5 | * Copyright (c) 2015 Imagination Technologies | |
6 | * Author: Alex Smith <alex@alex-smith.me.uk> | |
d894fc60 AS |
7 | */ |
8 | ||
9 | #include <linux/clk.h> | |
10 | #include <linux/dmapool.h> | |
2128565a | 11 | #include <linux/dma-mapping.h> |
d894fc60 AS |
12 | #include <linux/init.h> |
13 | #include <linux/interrupt.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/of.h> | |
6147b032 | 16 | #include <linux/of_device.h> |
d894fc60 AS |
17 | #include <linux/of_dma.h> |
18 | #include <linux/platform_device.h> | |
19 | #include <linux/slab.h> | |
20 | ||
21 | #include "dmaengine.h" | |
22 | #include "virt-dma.h" | |
23 | ||
d894fc60 | 24 | /* Global registers. */ |
33633583 PC |
25 | #define JZ_DMA_REG_DMAC 0x00 |
26 | #define JZ_DMA_REG_DIRQP 0x04 | |
27 | #define JZ_DMA_REG_DDR 0x08 | |
28 | #define JZ_DMA_REG_DDRS 0x0c | |
29870eb7 PC |
29 | #define JZ_DMA_REG_DCKE 0x10 |
30 | #define JZ_DMA_REG_DCKES 0x14 | |
31 | #define JZ_DMA_REG_DCKEC 0x18 | |
33633583 PC |
32 | #define JZ_DMA_REG_DMACP 0x1c |
33 | #define JZ_DMA_REG_DSIRQP 0x20 | |
34 | #define JZ_DMA_REG_DSIRQM 0x24 | |
35 | #define JZ_DMA_REG_DCIRQP 0x28 | |
36 | #define JZ_DMA_REG_DCIRQM 0x2c | |
d894fc60 AS |
37 | |
38 | /* Per-channel registers. */ | |
39 | #define JZ_DMA_REG_CHAN(n) (n * 0x20) | |
33633583 PC |
40 | #define JZ_DMA_REG_DSA 0x00 |
41 | #define JZ_DMA_REG_DTA 0x04 | |
42 | #define JZ_DMA_REG_DTC 0x08 | |
43 | #define JZ_DMA_REG_DRT 0x0c | |
44 | #define JZ_DMA_REG_DCS 0x10 | |
45 | #define JZ_DMA_REG_DCM 0x14 | |
46 | #define JZ_DMA_REG_DDA 0x18 | |
47 | #define JZ_DMA_REG_DSD 0x1c | |
d894fc60 AS |
48 | |
49 | #define JZ_DMA_DMAC_DMAE BIT(0) | |
50 | #define JZ_DMA_DMAC_AR BIT(2) | |
51 | #define JZ_DMA_DMAC_HLT BIT(3) | |
17a8e30e | 52 | #define JZ_DMA_DMAC_FAIC BIT(27) |
d894fc60 AS |
53 | #define JZ_DMA_DMAC_FMSC BIT(31) |
54 | ||
55 | #define JZ_DMA_DRT_AUTO 0x8 | |
56 | ||
57 | #define JZ_DMA_DCS_CTE BIT(0) | |
58 | #define JZ_DMA_DCS_HLT BIT(2) | |
59 | #define JZ_DMA_DCS_TT BIT(3) | |
60 | #define JZ_DMA_DCS_AR BIT(4) | |
61 | #define JZ_DMA_DCS_DES8 BIT(30) | |
62 | ||
63 | #define JZ_DMA_DCM_LINK BIT(0) | |
64 | #define JZ_DMA_DCM_TIE BIT(1) | |
65 | #define JZ_DMA_DCM_STDE BIT(2) | |
66 | #define JZ_DMA_DCM_TSZ_SHIFT 8 | |
67 | #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) | |
68 | #define JZ_DMA_DCM_DP_SHIFT 12 | |
69 | #define JZ_DMA_DCM_SP_SHIFT 14 | |
70 | #define JZ_DMA_DCM_DAI BIT(22) | |
71 | #define JZ_DMA_DCM_SAI BIT(23) | |
72 | ||
73 | #define JZ_DMA_SIZE_4_BYTE 0x0 | |
74 | #define JZ_DMA_SIZE_1_BYTE 0x1 | |
75 | #define JZ_DMA_SIZE_2_BYTE 0x2 | |
76 | #define JZ_DMA_SIZE_16_BYTE 0x3 | |
77 | #define JZ_DMA_SIZE_32_BYTE 0x4 | |
78 | #define JZ_DMA_SIZE_64_BYTE 0x5 | |
79 | #define JZ_DMA_SIZE_128_BYTE 0x6 | |
80 | ||
81 | #define JZ_DMA_WIDTH_32_BIT 0x0 | |
82 | #define JZ_DMA_WIDTH_8_BIT 0x1 | |
83 | #define JZ_DMA_WIDTH_16_BIT 0x2 | |
84 | ||
85 | #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ | |
86 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ | |
87 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) | |
88 | ||
33633583 PC |
89 | #define JZ4780_DMA_CTRL_OFFSET 0x1000 |
90 | ||
29870eb7 PC |
91 | /* macros for use with jz4780_dma_soc_data.flags */ |
92 | #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0) | |
93 | #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1) | |
94 | #define JZ_SOC_DATA_PER_CHAN_PM BIT(2) | |
ae9156b6 | 95 | #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3) |
f4c255f1 | 96 | #define JZ_SOC_DATA_BREAK_LINKS BIT(4) |
29870eb7 | 97 | |
d894fc60 AS |
98 | /** |
99 | * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. | |
100 | * @dcm: value for the DCM (channel command) register | |
101 | * @dsa: source address | |
102 | * @dta: target address | |
103 | * @dtc: transfer count (number of blocks of the transfer size specified in DCM | |
104 | * to transfer) in the low 24 bits, offset of the next descriptor from the | |
105 | * descriptor base address in the upper 8 bits. | |
d894fc60 AS |
106 | */ |
107 | struct jz4780_dma_hwdesc { | |
c8c0cda8 PC |
108 | u32 dcm; |
109 | u32 dsa; | |
110 | u32 dta; | |
111 | u32 dtc; | |
d894fc60 AS |
112 | }; |
113 | ||
114 | /* Size of allocations for hardware descriptor blocks. */ | |
115 | #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE | |
116 | #define JZ_DMA_MAX_DESC \ | |
117 | (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) | |
118 | ||
119 | struct jz4780_dma_desc { | |
120 | struct virt_dma_desc vdesc; | |
121 | ||
122 | struct jz4780_dma_hwdesc *desc; | |
123 | dma_addr_t desc_phys; | |
124 | unsigned int count; | |
125 | enum dma_transaction_type type; | |
76a09663 | 126 | u32 transfer_type; |
c8c0cda8 | 127 | u32 status; |
d894fc60 AS |
128 | }; |
129 | ||
130 | struct jz4780_dma_chan { | |
131 | struct virt_dma_chan vchan; | |
132 | unsigned int id; | |
133 | struct dma_pool *desc_pool; | |
134 | ||
76a09663 | 135 | u32 transfer_type_tx, transfer_type_rx; |
c8c0cda8 | 136 | u32 transfer_shift; |
d894fc60 AS |
137 | struct dma_slave_config config; |
138 | ||
139 | struct jz4780_dma_desc *desc; | |
140 | unsigned int curr_hwdesc; | |
141 | }; | |
142 | ||
6147b032 PC |
143 | struct jz4780_dma_soc_data { |
144 | unsigned int nb_channels; | |
29870eb7 PC |
145 | unsigned int transfer_ord_max; |
146 | unsigned long flags; | |
6147b032 PC |
147 | }; |
148 | ||
d894fc60 AS |
149 | struct jz4780_dma_dev { |
150 | struct dma_device dma_device; | |
33633583 PC |
151 | void __iomem *chn_base; |
152 | void __iomem *ctrl_base; | |
d894fc60 AS |
153 | struct clk *clk; |
154 | unsigned int irq; | |
6147b032 | 155 | const struct jz4780_dma_soc_data *soc_data; |
d894fc60 | 156 | |
c8c0cda8 | 157 | u32 chan_reserved; |
6147b032 | 158 | struct jz4780_dma_chan chan[]; |
d894fc60 AS |
159 | }; |
160 | ||
026fd406 | 161 | struct jz4780_dma_filter_data { |
76a09663 | 162 | u32 transfer_type_tx, transfer_type_rx; |
d894fc60 AS |
163 | int channel; |
164 | }; | |
165 | ||
166 | static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) | |
167 | { | |
168 | return container_of(chan, struct jz4780_dma_chan, vchan.chan); | |
169 | } | |
170 | ||
171 | static inline struct jz4780_dma_desc *to_jz4780_dma_desc( | |
172 | struct virt_dma_desc *vdesc) | |
173 | { | |
174 | return container_of(vdesc, struct jz4780_dma_desc, vdesc); | |
175 | } | |
176 | ||
177 | static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( | |
178 | struct jz4780_dma_chan *jzchan) | |
179 | { | |
180 | return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, | |
181 | dma_device); | |
182 | } | |
183 | ||
c8c0cda8 | 184 | static inline u32 jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, |
33633583 PC |
185 | unsigned int chn, unsigned int reg) |
186 | { | |
187 | return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); | |
188 | } | |
189 | ||
190 | static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, | |
c8c0cda8 | 191 | unsigned int chn, unsigned int reg, u32 val) |
33633583 PC |
192 | { |
193 | writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); | |
194 | } | |
195 | ||
c8c0cda8 | 196 | static inline u32 jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, |
d894fc60 AS |
197 | unsigned int reg) |
198 | { | |
33633583 | 199 | return readl(jzdma->ctrl_base + reg); |
d894fc60 AS |
200 | } |
201 | ||
33633583 | 202 | static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, |
c8c0cda8 | 203 | unsigned int reg, u32 val) |
d894fc60 | 204 | { |
33633583 | 205 | writel(val, jzdma->ctrl_base + reg); |
d894fc60 AS |
206 | } |
207 | ||
29870eb7 PC |
208 | static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma, |
209 | unsigned int chn) | |
210 | { | |
ae9156b6 PC |
211 | if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { |
212 | unsigned int reg; | |
213 | ||
214 | if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) | |
215 | reg = JZ_DMA_REG_DCKE; | |
216 | else | |
217 | reg = JZ_DMA_REG_DCKES; | |
218 | ||
219 | jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn)); | |
220 | } | |
29870eb7 PC |
221 | } |
222 | ||
223 | static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, | |
224 | unsigned int chn) | |
225 | { | |
ae9156b6 PC |
226 | if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && |
227 | !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) | |
29870eb7 | 228 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); |
d894fc60 AS |
229 | } |
230 | ||
76a09663 PC |
231 | static struct jz4780_dma_desc * |
232 | jz4780_dma_desc_alloc(struct jz4780_dma_chan *jzchan, unsigned int count, | |
233 | enum dma_transaction_type type, | |
234 | enum dma_transfer_direction direction) | |
d894fc60 AS |
235 | { |
236 | struct jz4780_dma_desc *desc; | |
237 | ||
238 | if (count > JZ_DMA_MAX_DESC) | |
239 | return NULL; | |
240 | ||
241 | desc = kzalloc(sizeof(*desc), GFP_NOWAIT); | |
242 | if (!desc) | |
243 | return NULL; | |
244 | ||
245 | desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, | |
246 | &desc->desc_phys); | |
247 | if (!desc->desc) { | |
248 | kfree(desc); | |
249 | return NULL; | |
250 | } | |
251 | ||
252 | desc->count = count; | |
253 | desc->type = type; | |
76a09663 PC |
254 | |
255 | if (direction == DMA_DEV_TO_MEM) | |
256 | desc->transfer_type = jzchan->transfer_type_rx; | |
257 | else | |
258 | desc->transfer_type = jzchan->transfer_type_tx; | |
259 | ||
d894fc60 AS |
260 | return desc; |
261 | } | |
262 | ||
263 | static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) | |
264 | { | |
265 | struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); | |
266 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); | |
267 | ||
268 | dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); | |
269 | kfree(desc); | |
270 | } | |
271 | ||
c8c0cda8 PC |
272 | static u32 jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, |
273 | unsigned long val, u32 *shift) | |
d894fc60 | 274 | { |
29870eb7 | 275 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); |
dc578f31 | 276 | int ord = ffs(val) - 1; |
d894fc60 | 277 | |
dc578f31 AS |
278 | /* |
279 | * 8 byte transfer sizes unsupported so fall back on 4. If it's larger | |
280 | * than the maximum, just limit it. It is perfectly safe to fall back | |
281 | * in this way since we won't exceed the maximum burst size supported | |
282 | * by the device, the only effect is reduced efficiency. This is better | |
283 | * than refusing to perform the request at all. | |
284 | */ | |
285 | if (ord == 3) | |
286 | ord = 2; | |
29870eb7 PC |
287 | else if (ord > jzdma->soc_data->transfer_ord_max) |
288 | ord = jzdma->soc_data->transfer_ord_max; | |
dc578f31 AS |
289 | |
290 | *shift = ord; | |
291 | ||
292 | switch (ord) { | |
d894fc60 AS |
293 | case 0: |
294 | return JZ_DMA_SIZE_1_BYTE; | |
295 | case 1: | |
296 | return JZ_DMA_SIZE_2_BYTE; | |
297 | case 2: | |
298 | return JZ_DMA_SIZE_4_BYTE; | |
299 | case 4: | |
300 | return JZ_DMA_SIZE_16_BYTE; | |
301 | case 5: | |
302 | return JZ_DMA_SIZE_32_BYTE; | |
303 | case 6: | |
304 | return JZ_DMA_SIZE_64_BYTE; | |
d894fc60 | 305 | default: |
dc578f31 | 306 | return JZ_DMA_SIZE_128_BYTE; |
d894fc60 AS |
307 | } |
308 | } | |
309 | ||
839896ef | 310 | static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, |
d894fc60 AS |
311 | struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, |
312 | enum dma_transfer_direction direction) | |
313 | { | |
314 | struct dma_slave_config *config = &jzchan->config; | |
c8c0cda8 | 315 | u32 width, maxburst, tsz; |
d894fc60 AS |
316 | |
317 | if (direction == DMA_MEM_TO_DEV) { | |
318 | desc->dcm = JZ_DMA_DCM_SAI; | |
319 | desc->dsa = addr; | |
320 | desc->dta = config->dst_addr; | |
d894fc60 AS |
321 | |
322 | width = config->dst_addr_width; | |
323 | maxburst = config->dst_maxburst; | |
324 | } else { | |
325 | desc->dcm = JZ_DMA_DCM_DAI; | |
326 | desc->dsa = config->src_addr; | |
327 | desc->dta = addr; | |
d894fc60 AS |
328 | |
329 | width = config->src_addr_width; | |
330 | maxburst = config->src_maxburst; | |
331 | } | |
332 | ||
333 | /* | |
334 | * This calculates the maximum transfer size that can be used with the | |
335 | * given address, length, width and maximum burst size. The address | |
336 | * must be aligned to the transfer size, the total length must be | |
337 | * divisible by the transfer size, and we must not use more than the | |
338 | * maximum burst specified by the user. | |
339 | */ | |
29870eb7 | 340 | tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst), |
dc578f31 | 341 | &jzchan->transfer_shift); |
d894fc60 AS |
342 | |
343 | switch (width) { | |
344 | case DMA_SLAVE_BUSWIDTH_1_BYTE: | |
345 | case DMA_SLAVE_BUSWIDTH_2_BYTES: | |
346 | break; | |
347 | case DMA_SLAVE_BUSWIDTH_4_BYTES: | |
348 | width = JZ_DMA_WIDTH_32_BIT; | |
349 | break; | |
350 | default: | |
351 | return -EINVAL; | |
352 | } | |
353 | ||
354 | desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; | |
355 | desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; | |
356 | desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; | |
357 | ||
dc578f31 | 358 | desc->dtc = len >> jzchan->transfer_shift; |
839896ef | 359 | return 0; |
d894fc60 AS |
360 | } |
361 | ||
362 | static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( | |
363 | struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, | |
46fa5168 AS |
364 | enum dma_transfer_direction direction, unsigned long flags, |
365 | void *context) | |
d894fc60 AS |
366 | { |
367 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
f4c255f1 | 368 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); |
d894fc60 AS |
369 | struct jz4780_dma_desc *desc; |
370 | unsigned int i; | |
371 | int err; | |
372 | ||
76a09663 | 373 | desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE, direction); |
d894fc60 AS |
374 | if (!desc) |
375 | return NULL; | |
376 | ||
377 | for (i = 0; i < sg_len; i++) { | |
378 | err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], | |
839896ef AS |
379 | sg_dma_address(&sgl[i]), |
380 | sg_dma_len(&sgl[i]), | |
381 | direction); | |
fc878efe CIK |
382 | if (err < 0) { |
383 | jz4780_dma_desc_free(&jzchan->desc->vdesc); | |
839896ef | 384 | return NULL; |
fc878efe | 385 | } |
d894fc60 AS |
386 | |
387 | desc->desc[i].dcm |= JZ_DMA_DCM_TIE; | |
388 | ||
f4c255f1 PC |
389 | if (i != (sg_len - 1) && |
390 | !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) { | |
0d7c11af | 391 | /* Automatically proceed to the next descriptor. */ |
d894fc60 AS |
392 | desc->desc[i].dcm |= JZ_DMA_DCM_LINK; |
393 | ||
394 | /* | |
395 | * The upper 8 bits of the DTC field in the descriptor | |
396 | * must be set to (offset from descriptor base of next | |
397 | * descriptor >> 4). | |
398 | */ | |
399 | desc->desc[i].dtc |= | |
400 | (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; | |
401 | } | |
402 | } | |
403 | ||
404 | return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); | |
405 | } | |
406 | ||
407 | static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( | |
408 | struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, | |
409 | size_t period_len, enum dma_transfer_direction direction, | |
410 | unsigned long flags) | |
411 | { | |
412 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
413 | struct jz4780_dma_desc *desc; | |
414 | unsigned int periods, i; | |
415 | int err; | |
416 | ||
417 | if (buf_len % period_len) | |
418 | return NULL; | |
419 | ||
420 | periods = buf_len / period_len; | |
421 | ||
76a09663 | 422 | desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC, direction); |
d894fc60 AS |
423 | if (!desc) |
424 | return NULL; | |
425 | ||
426 | for (i = 0; i < periods; i++) { | |
427 | err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, | |
839896ef | 428 | period_len, direction); |
fc878efe CIK |
429 | if (err < 0) { |
430 | jz4780_dma_desc_free(&jzchan->desc->vdesc); | |
839896ef | 431 | return NULL; |
fc878efe | 432 | } |
d894fc60 AS |
433 | |
434 | buf_addr += period_len; | |
435 | ||
436 | /* | |
437 | * Set the link bit to indicate that the controller should | |
438 | * automatically proceed to the next descriptor. In | |
439 | * jz4780_dma_begin(), this will be cleared if we need to issue | |
440 | * an interrupt after each period. | |
441 | */ | |
442 | desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; | |
443 | ||
444 | /* | |
445 | * The upper 8 bits of the DTC field in the descriptor must be | |
446 | * set to (offset from descriptor base of next descriptor >> 4). | |
447 | * If this is the last descriptor, link it back to the first, | |
448 | * i.e. leave offset set to 0, otherwise point to the next one. | |
449 | */ | |
450 | if (i != (periods - 1)) { | |
451 | desc->desc[i].dtc |= | |
452 | (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; | |
453 | } | |
454 | } | |
455 | ||
456 | return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); | |
457 | } | |
458 | ||
4f5db8c8 | 459 | static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( |
d894fc60 AS |
460 | struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, |
461 | size_t len, unsigned long flags) | |
462 | { | |
463 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
464 | struct jz4780_dma_desc *desc; | |
c8c0cda8 | 465 | u32 tsz; |
d894fc60 | 466 | |
76a09663 | 467 | desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY, 0); |
d894fc60 AS |
468 | if (!desc) |
469 | return NULL; | |
470 | ||
29870eb7 | 471 | tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, |
dc578f31 | 472 | &jzchan->transfer_shift); |
d894fc60 | 473 | |
76a09663 | 474 | desc->transfer_type = JZ_DMA_DRT_AUTO; |
5eed7d84 | 475 | |
d894fc60 AS |
476 | desc->desc[0].dsa = src; |
477 | desc->desc[0].dta = dest; | |
d894fc60 AS |
478 | desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | |
479 | tsz << JZ_DMA_DCM_TSZ_SHIFT | | |
480 | JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | | |
481 | JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; | |
839896ef | 482 | desc->desc[0].dtc = len >> jzchan->transfer_shift; |
d894fc60 AS |
483 | |
484 | return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); | |
485 | } | |
486 | ||
487 | static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) | |
488 | { | |
489 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); | |
490 | struct virt_dma_desc *vdesc; | |
491 | unsigned int i; | |
492 | dma_addr_t desc_phys; | |
493 | ||
494 | if (!jzchan->desc) { | |
495 | vdesc = vchan_next_desc(&jzchan->vchan); | |
496 | if (!vdesc) | |
497 | return; | |
498 | ||
499 | list_del(&vdesc->node); | |
500 | ||
501 | jzchan->desc = to_jz4780_dma_desc(vdesc); | |
502 | jzchan->curr_hwdesc = 0; | |
503 | ||
504 | if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { | |
505 | /* | |
506 | * The DMA controller doesn't support triggering an | |
507 | * interrupt after processing each descriptor, only | |
508 | * after processing an entire terminated list of | |
509 | * descriptors. For a cyclic DMA setup the list of | |
510 | * descriptors is not terminated so we can never get an | |
511 | * interrupt. | |
512 | * | |
513 | * If the user requested a callback for a cyclic DMA | |
514 | * setup then we workaround this hardware limitation | |
515 | * here by degrading to a set of unlinked descriptors | |
516 | * which we will submit in sequence in response to the | |
517 | * completion of processing the previous descriptor. | |
518 | */ | |
519 | for (i = 0; i < jzchan->desc->count; i++) | |
520 | jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; | |
521 | } | |
522 | } else { | |
523 | /* | |
524 | * There is an existing transfer, therefore this must be one | |
525 | * for which we unlinked the descriptors above. Advance to the | |
526 | * next one in the list. | |
527 | */ | |
528 | jzchan->curr_hwdesc = | |
529 | (jzchan->curr_hwdesc + 1) % jzchan->desc->count; | |
530 | } | |
531 | ||
29870eb7 PC |
532 | /* Enable the channel's clock. */ |
533 | jz4780_dma_chan_enable(jzdma, jzchan->id); | |
534 | ||
5eed7d84 PC |
535 | /* Use 4-word descriptors. */ |
536 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); | |
537 | ||
538 | /* Set transfer type. */ | |
539 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, | |
76a09663 | 540 | jzchan->desc->transfer_type); |
d894fc60 | 541 | |
9e4e3a4c DS |
542 | /* |
543 | * Set the transfer count. This is redundant for a descriptor-driven | |
544 | * transfer. However, there can be a delay between the transfer start | |
545 | * time and when DTCn reg contains the new transfer count. Setting | |
546 | * it explicitly ensures residue is computed correctly at all times. | |
547 | */ | |
548 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, | |
549 | jzchan->desc->desc[jzchan->curr_hwdesc].dtc); | |
d894fc60 AS |
550 | |
551 | /* Write descriptor address and initiate descriptor fetch. */ | |
552 | desc_phys = jzchan->desc->desc_phys + | |
553 | (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); | |
33633583 PC |
554 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); |
555 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); | |
d894fc60 AS |
556 | |
557 | /* Enable the channel. */ | |
33633583 | 558 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, |
5eed7d84 | 559 | JZ_DMA_DCS_CTE); |
d894fc60 AS |
560 | } |
561 | ||
562 | static void jz4780_dma_issue_pending(struct dma_chan *chan) | |
563 | { | |
564 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
565 | unsigned long flags; | |
566 | ||
567 | spin_lock_irqsave(&jzchan->vchan.lock, flags); | |
568 | ||
569 | if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) | |
570 | jz4780_dma_begin(jzchan); | |
571 | ||
572 | spin_unlock_irqrestore(&jzchan->vchan.lock, flags); | |
573 | } | |
574 | ||
46fa5168 | 575 | static int jz4780_dma_terminate_all(struct dma_chan *chan) |
d894fc60 | 576 | { |
46fa5168 | 577 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); |
d894fc60 AS |
578 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); |
579 | unsigned long flags; | |
580 | LIST_HEAD(head); | |
581 | ||
582 | spin_lock_irqsave(&jzchan->vchan.lock, flags); | |
583 | ||
584 | /* Clear the DMA status and stop the transfer. */ | |
33633583 | 585 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); |
d894fc60 | 586 | if (jzchan->desc) { |
f0dd52c8 | 587 | vchan_terminate_vdesc(&jzchan->desc->vdesc); |
d894fc60 AS |
588 | jzchan->desc = NULL; |
589 | } | |
590 | ||
29870eb7 PC |
591 | jz4780_dma_chan_disable(jzdma, jzchan->id); |
592 | ||
d894fc60 AS |
593 | vchan_get_all_descriptors(&jzchan->vchan, &head); |
594 | ||
595 | spin_unlock_irqrestore(&jzchan->vchan.lock, flags); | |
596 | ||
597 | vchan_dma_desc_free_list(&jzchan->vchan, &head); | |
598 | return 0; | |
599 | } | |
600 | ||
f0dd52c8 PU |
601 | static void jz4780_dma_synchronize(struct dma_chan *chan) |
602 | { | |
603 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
29870eb7 | 604 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); |
f0dd52c8 PU |
605 | |
606 | vchan_synchronize(&jzchan->vchan); | |
29870eb7 | 607 | jz4780_dma_chan_disable(jzdma, jzchan->id); |
f0dd52c8 PU |
608 | } |
609 | ||
46fa5168 AS |
610 | static int jz4780_dma_config(struct dma_chan *chan, |
611 | struct dma_slave_config *config) | |
d894fc60 | 612 | { |
46fa5168 AS |
613 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); |
614 | ||
d894fc60 AS |
615 | if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) |
616 | || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) | |
617 | return -EINVAL; | |
618 | ||
619 | /* Copy the reset of the slave configuration, it is used later. */ | |
620 | memcpy(&jzchan->config, config, sizeof(jzchan->config)); | |
621 | ||
622 | return 0; | |
623 | } | |
624 | ||
625 | static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, | |
626 | struct jz4780_dma_desc *desc, unsigned int next_sg) | |
627 | { | |
628 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); | |
f3c045df | 629 | unsigned int count = 0; |
d894fc60 AS |
630 | unsigned int i; |
631 | ||
d894fc60 | 632 | for (i = next_sg; i < desc->count; i++) |
f3c045df | 633 | count += desc->desc[i].dtc & GENMASK(23, 0); |
d894fc60 | 634 | |
f3c045df DS |
635 | if (next_sg != 0) |
636 | count += jz4780_dma_chn_readl(jzdma, jzchan->id, | |
33633583 | 637 | JZ_DMA_REG_DTC); |
d894fc60 | 638 | |
f3c045df | 639 | return count << jzchan->transfer_shift; |
d894fc60 AS |
640 | } |
641 | ||
642 | static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, | |
643 | dma_cookie_t cookie, struct dma_tx_state *txstate) | |
644 | { | |
645 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
646 | struct virt_dma_desc *vdesc; | |
647 | enum dma_status status; | |
648 | unsigned long flags; | |
1f0b0f23 | 649 | unsigned long residue = 0; |
d894fc60 | 650 | |
baf6fd97 PC |
651 | spin_lock_irqsave(&jzchan->vchan.lock, flags); |
652 | ||
d894fc60 AS |
653 | status = dma_cookie_status(chan, cookie, txstate); |
654 | if ((status == DMA_COMPLETE) || (txstate == NULL)) | |
baf6fd97 | 655 | goto out_unlock_irqrestore; |
d894fc60 AS |
656 | |
657 | vdesc = vchan_find_desc(&jzchan->vchan, cookie); | |
658 | if (vdesc) { | |
659 | /* On the issued list, so hasn't been processed yet */ | |
1f0b0f23 | 660 | residue = jz4780_dma_desc_residue(jzchan, |
d894fc60 AS |
661 | to_jz4780_dma_desc(vdesc), 0); |
662 | } else if (cookie == jzchan->desc->vdesc.tx.cookie) { | |
1f0b0f23 | 663 | residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, |
83ef4fb7 | 664 | jzchan->curr_hwdesc + 1); |
1f0b0f23 DS |
665 | } |
666 | dma_set_residue(txstate, residue); | |
d894fc60 AS |
667 | |
668 | if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc | |
839896ef AS |
669 | && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) |
670 | status = DMA_ERROR; | |
d894fc60 | 671 | |
baf6fd97 | 672 | out_unlock_irqrestore: |
d894fc60 AS |
673 | spin_unlock_irqrestore(&jzchan->vchan.lock, flags); |
674 | return status; | |
675 | } | |
676 | ||
4e4106f5 PC |
677 | static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, |
678 | struct jz4780_dma_chan *jzchan) | |
d894fc60 | 679 | { |
f4c255f1 PC |
680 | const unsigned int soc_flags = jzdma->soc_data->flags; |
681 | struct jz4780_dma_desc *desc = jzchan->desc; | |
c8c0cda8 | 682 | u32 dcs; |
4e4106f5 | 683 | bool ack = true; |
d894fc60 AS |
684 | |
685 | spin_lock(&jzchan->vchan.lock); | |
686 | ||
33633583 PC |
687 | dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); |
688 | jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); | |
d894fc60 AS |
689 | |
690 | if (dcs & JZ_DMA_DCS_AR) { | |
691 | dev_warn(&jzchan->vchan.chan.dev->device, | |
692 | "address error (DCS=0x%x)\n", dcs); | |
693 | } | |
694 | ||
695 | if (dcs & JZ_DMA_DCS_HLT) { | |
696 | dev_warn(&jzchan->vchan.chan.dev->device, | |
697 | "channel halt (DCS=0x%x)\n", dcs); | |
698 | } | |
699 | ||
700 | if (jzchan->desc) { | |
701 | jzchan->desc->status = dcs; | |
702 | ||
703 | if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { | |
704 | if (jzchan->desc->type == DMA_CYCLIC) { | |
705 | vchan_cyclic_callback(&jzchan->desc->vdesc); | |
4e4106f5 PC |
706 | |
707 | jz4780_dma_begin(jzchan); | |
708 | } else if (dcs & JZ_DMA_DCS_TT) { | |
f4c255f1 PC |
709 | if (!(soc_flags & JZ_SOC_DATA_BREAK_LINKS) || |
710 | (jzchan->curr_hwdesc + 1 == desc->count)) { | |
711 | vchan_cookie_complete(&desc->vdesc); | |
712 | jzchan->desc = NULL; | |
713 | } | |
d894fc60 | 714 | |
4e4106f5 PC |
715 | jz4780_dma_begin(jzchan); |
716 | } else { | |
717 | /* False positive - continue the transfer */ | |
718 | ack = false; | |
719 | jz4780_dma_chn_writel(jzdma, jzchan->id, | |
720 | JZ_DMA_REG_DCS, | |
721 | JZ_DMA_DCS_CTE); | |
722 | } | |
d894fc60 AS |
723 | } |
724 | } else { | |
725 | dev_err(&jzchan->vchan.chan.dev->device, | |
726 | "channel IRQ with no active transfer\n"); | |
727 | } | |
728 | ||
729 | spin_unlock(&jzchan->vchan.lock); | |
4e4106f5 PC |
730 | |
731 | return ack; | |
d894fc60 AS |
732 | } |
733 | ||
734 | static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) | |
735 | { | |
736 | struct jz4780_dma_dev *jzdma = data; | |
4e4106f5 | 737 | unsigned int nb_channels = jzdma->soc_data->nb_channels; |
4c89cc73 | 738 | unsigned long pending; |
c8c0cda8 | 739 | u32 dmac; |
d894fc60 AS |
740 | int i; |
741 | ||
33633583 | 742 | pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); |
d894fc60 | 743 | |
4c89cc73 | 744 | for_each_set_bit(i, &pending, nb_channels) { |
4e4106f5 PC |
745 | if (jz4780_dma_chan_irq(jzdma, &jzdma->chan[i])) |
746 | pending &= ~BIT(i); | |
d894fc60 AS |
747 | } |
748 | ||
749 | /* Clear halt and address error status of all channels. */ | |
33633583 | 750 | dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC); |
d894fc60 | 751 | dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); |
33633583 | 752 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac); |
d894fc60 AS |
753 | |
754 | /* Clear interrupt pending status. */ | |
4e4106f5 | 755 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, pending); |
d894fc60 AS |
756 | |
757 | return IRQ_HANDLED; | |
758 | } | |
759 | ||
760 | static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) | |
761 | { | |
762 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
763 | ||
764 | jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), | |
765 | chan->device->dev, | |
766 | JZ_DMA_DESC_BLOCK_SIZE, | |
767 | PAGE_SIZE, 0); | |
768 | if (!jzchan->desc_pool) { | |
769 | dev_err(&chan->dev->device, | |
770 | "failed to allocate descriptor pool\n"); | |
771 | return -ENOMEM; | |
772 | } | |
773 | ||
774 | return 0; | |
775 | } | |
776 | ||
777 | static void jz4780_dma_free_chan_resources(struct dma_chan *chan) | |
778 | { | |
779 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
780 | ||
781 | vchan_free_chan_resources(&jzchan->vchan); | |
782 | dma_pool_destroy(jzchan->desc_pool); | |
783 | jzchan->desc_pool = NULL; | |
784 | } | |
785 | ||
786 | static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) | |
787 | { | |
788 | struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); | |
789 | struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); | |
026fd406 AS |
790 | struct jz4780_dma_filter_data *data = param; |
791 | ||
d894fc60 AS |
792 | |
793 | if (data->channel > -1) { | |
794 | if (data->channel != jzchan->id) | |
795 | return false; | |
796 | } else if (jzdma->chan_reserved & BIT(jzchan->id)) { | |
797 | return false; | |
798 | } | |
799 | ||
76a09663 PC |
800 | jzchan->transfer_type_tx = data->transfer_type_tx; |
801 | jzchan->transfer_type_rx = data->transfer_type_rx; | |
d894fc60 AS |
802 | |
803 | return true; | |
804 | } | |
805 | ||
806 | static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, | |
807 | struct of_dma *ofdma) | |
808 | { | |
809 | struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; | |
810 | dma_cap_mask_t mask = jzdma->dma_device.cap_mask; | |
026fd406 | 811 | struct jz4780_dma_filter_data data; |
d894fc60 | 812 | |
76a09663 PC |
813 | if (dma_spec->args_count == 2) { |
814 | data.transfer_type_tx = dma_spec->args[0]; | |
815 | data.transfer_type_rx = dma_spec->args[0]; | |
816 | data.channel = dma_spec->args[1]; | |
817 | } else if (dma_spec->args_count == 3) { | |
818 | data.transfer_type_tx = dma_spec->args[0]; | |
819 | data.transfer_type_rx = dma_spec->args[1]; | |
820 | data.channel = dma_spec->args[2]; | |
821 | } else { | |
d894fc60 | 822 | return NULL; |
76a09663 | 823 | } |
d894fc60 AS |
824 | |
825 | if (data.channel > -1) { | |
6147b032 | 826 | if (data.channel >= jzdma->soc_data->nb_channels) { |
d894fc60 AS |
827 | dev_err(jzdma->dma_device.dev, |
828 | "device requested non-existent channel %u\n", | |
829 | data.channel); | |
830 | return NULL; | |
831 | } | |
832 | ||
833 | /* Can only select a channel marked as reserved. */ | |
834 | if (!(jzdma->chan_reserved & BIT(data.channel))) { | |
835 | dev_err(jzdma->dma_device.dev, | |
836 | "device requested unreserved channel %u\n", | |
837 | data.channel); | |
838 | return NULL; | |
839 | } | |
d894fc60 | 840 | |
76a09663 PC |
841 | jzdma->chan[data.channel].transfer_type_tx = data.transfer_type_tx; |
842 | jzdma->chan[data.channel].transfer_type_rx = data.transfer_type_rx; | |
d3273e10 AS |
843 | |
844 | return dma_get_slave_channel( | |
845 | &jzdma->chan[data.channel].vchan.chan); | |
846 | } else { | |
c88ba7b9 BW |
847 | return __dma_request_channel(&mask, jz4780_dma_filter_fn, &data, |
848 | ofdma->of_node); | |
d3273e10 | 849 | } |
d894fc60 AS |
850 | } |
851 | ||
852 | static int jz4780_dma_probe(struct platform_device *pdev) | |
853 | { | |
854 | struct device *dev = &pdev->dev; | |
6147b032 | 855 | const struct jz4780_dma_soc_data *soc_data; |
d894fc60 AS |
856 | struct jz4780_dma_dev *jzdma; |
857 | struct jz4780_dma_chan *jzchan; | |
858 | struct dma_device *dd; | |
859 | struct resource *res; | |
860 | int i, ret; | |
861 | ||
54f919a0 PC |
862 | if (!dev->of_node) { |
863 | dev_err(dev, "This driver must be probed from devicetree\n"); | |
864 | return -EINVAL; | |
865 | } | |
866 | ||
6147b032 PC |
867 | soc_data = device_get_match_data(dev); |
868 | if (!soc_data) | |
869 | return -EINVAL; | |
870 | ||
ed414d58 GS |
871 | jzdma = devm_kzalloc(dev, struct_size(jzdma, chan, |
872 | soc_data->nb_channels), GFP_KERNEL); | |
d894fc60 AS |
873 | if (!jzdma) |
874 | return -ENOMEM; | |
875 | ||
6147b032 | 876 | jzdma->soc_data = soc_data; |
d894fc60 AS |
877 | platform_set_drvdata(pdev, jzdma); |
878 | ||
1148ac67 | 879 | jzdma->chn_base = devm_platform_ioremap_resource(pdev, 0); |
33633583 PC |
880 | if (IS_ERR(jzdma->chn_base)) |
881 | return PTR_ERR(jzdma->chn_base); | |
882 | ||
883 | res = platform_get_resource(pdev, IORESOURCE_MEM, 1); | |
884 | if (res) { | |
885 | jzdma->ctrl_base = devm_ioremap_resource(dev, res); | |
886 | if (IS_ERR(jzdma->ctrl_base)) | |
887 | return PTR_ERR(jzdma->ctrl_base); | |
29870eb7 | 888 | } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { |
33633583 PC |
889 | /* |
890 | * On JZ4780, if the second memory resource was not supplied, | |
891 | * assume we're using an old devicetree, and calculate the | |
892 | * offset to the control registers. | |
893 | */ | |
894 | jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; | |
29870eb7 PC |
895 | } else { |
896 | dev_err(dev, "failed to get I/O memory\n"); | |
897 | return -EINVAL; | |
33633583 | 898 | } |
d894fc60 | 899 | |
d894fc60 AS |
900 | jzdma->clk = devm_clk_get(dev, NULL); |
901 | if (IS_ERR(jzdma->clk)) { | |
902 | dev_err(dev, "failed to get clock\n"); | |
d509a83c | 903 | ret = PTR_ERR(jzdma->clk); |
6d6018fc | 904 | return ret; |
d894fc60 AS |
905 | } |
906 | ||
907 | clk_prepare_enable(jzdma->clk); | |
908 | ||
909 | /* Property is optional, if it doesn't exist the value will remain 0. */ | |
910 | of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", | |
911 | 0, &jzdma->chan_reserved); | |
912 | ||
913 | dd = &jzdma->dma_device; | |
914 | ||
2128565a AM |
915 | /* |
916 | * The real segment size limit is dependent on the size unit selected | |
917 | * for the transfer. Because the size unit is selected automatically | |
918 | * and may be as small as 1 byte, use a safe limit of 2^24-1 bytes to | |
919 | * ensure the 24-bit transfer count in the descriptor cannot overflow. | |
920 | */ | |
921 | dma_set_max_seg_size(dev, 0xffffff); | |
922 | ||
d894fc60 AS |
923 | dma_cap_set(DMA_MEMCPY, dd->cap_mask); |
924 | dma_cap_set(DMA_SLAVE, dd->cap_mask); | |
925 | dma_cap_set(DMA_CYCLIC, dd->cap_mask); | |
926 | ||
927 | dd->dev = dev; | |
77a68e56 | 928 | dd->copy_align = DMAENGINE_ALIGN_4_BYTES; |
d894fc60 AS |
929 | dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; |
930 | dd->device_free_chan_resources = jz4780_dma_free_chan_resources; | |
931 | dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; | |
932 | dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; | |
933 | dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; | |
46fa5168 | 934 | dd->device_config = jz4780_dma_config; |
d894fc60 | 935 | dd->device_terminate_all = jz4780_dma_terminate_all; |
f0dd52c8 | 936 | dd->device_synchronize = jz4780_dma_synchronize; |
d894fc60 AS |
937 | dd->device_tx_status = jz4780_dma_tx_status; |
938 | dd->device_issue_pending = jz4780_dma_issue_pending; | |
939 | dd->src_addr_widths = JZ_DMA_BUSWIDTHS; | |
940 | dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; | |
941 | dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); | |
942 | dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; | |
d59f7037 | 943 | dd->max_sg_burst = JZ_DMA_MAX_DESC; |
d894fc60 | 944 | |
d894fc60 AS |
945 | /* |
946 | * Enable DMA controller, mark all channels as not programmable. | |
947 | * Also set the FMSC bit - it increases MSC performance, so it makes | |
948 | * little sense not to enable it. | |
949 | */ | |
17a8e30e PC |
950 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE | |
951 | JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC); | |
29870eb7 PC |
952 | |
953 | if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) | |
954 | jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0); | |
d894fc60 AS |
955 | |
956 | INIT_LIST_HEAD(&dd->channels); | |
957 | ||
6147b032 | 958 | for (i = 0; i < soc_data->nb_channels; i++) { |
d894fc60 AS |
959 | jzchan = &jzdma->chan[i]; |
960 | jzchan->id = i; | |
961 | ||
962 | vchan_init(&jzchan->vchan, dd); | |
963 | jzchan->vchan.desc_free = jz4780_dma_desc_free; | |
964 | } | |
965 | ||
b72cbb1a PC |
966 | /* |
967 | * On JZ4760, chan0 won't enable properly the first time. | |
968 | * Enabling then disabling chan1 will magically make chan0 work | |
969 | * correctly. | |
970 | */ | |
971 | jz4780_dma_chan_enable(jzdma, 1); | |
972 | jz4780_dma_chan_disable(jzdma, 1); | |
973 | ||
6d6018fc MB |
974 | ret = platform_get_irq(pdev, 0); |
975 | if (ret < 0) | |
976 | goto err_disable_clk; | |
977 | ||
978 | jzdma->irq = ret; | |
979 | ||
980 | ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), | |
981 | jzdma); | |
982 | if (ret) { | |
983 | dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); | |
984 | goto err_disable_clk; | |
985 | } | |
986 | ||
0f5a5e57 | 987 | ret = dmaenginem_async_device_register(dd); |
d894fc60 AS |
988 | if (ret) { |
989 | dev_err(dev, "failed to register device\n"); | |
6d6018fc | 990 | goto err_free_irq; |
d894fc60 AS |
991 | } |
992 | ||
993 | /* Register with OF DMA helpers. */ | |
994 | ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, | |
995 | jzdma); | |
996 | if (ret) { | |
997 | dev_err(dev, "failed to register OF DMA controller\n"); | |
6d6018fc | 998 | goto err_free_irq; |
d894fc60 AS |
999 | } |
1000 | ||
1001 | dev_info(dev, "JZ4780 DMA controller initialised\n"); | |
1002 | return 0; | |
1003 | ||
d509a83c AS |
1004 | err_free_irq: |
1005 | free_irq(jzdma->irq, jzdma); | |
6d6018fc MB |
1006 | |
1007 | err_disable_clk: | |
1008 | clk_disable_unprepare(jzdma->clk); | |
d894fc60 AS |
1009 | return ret; |
1010 | } | |
1011 | ||
1012 | static int jz4780_dma_remove(struct platform_device *pdev) | |
1013 | { | |
1014 | struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); | |
ae9c02b4 | 1015 | int i; |
d894fc60 AS |
1016 | |
1017 | of_dma_controller_free(pdev->dev.of_node); | |
ae9c02b4 | 1018 | |
9568feda | 1019 | clk_disable_unprepare(jzdma->clk); |
d509a83c | 1020 | free_irq(jzdma->irq, jzdma); |
ae9c02b4 | 1021 | |
6147b032 | 1022 | for (i = 0; i < jzdma->soc_data->nb_channels; i++) |
ae9c02b4 AS |
1023 | tasklet_kill(&jzdma->chan[i].vchan.task); |
1024 | ||
d894fc60 AS |
1025 | return 0; |
1026 | } | |
1027 | ||
ffaaa8cc PC |
1028 | static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { |
1029 | .nb_channels = 6, | |
1030 | .transfer_ord_max = 5, | |
f4c255f1 | 1031 | .flags = JZ_SOC_DATA_BREAK_LINKS, |
ffaaa8cc PC |
1032 | }; |
1033 | ||
ae9156b6 PC |
1034 | static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { |
1035 | .nb_channels = 6, | |
1036 | .transfer_ord_max = 5, | |
a40c94be PC |
1037 | .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | |
1038 | JZ_SOC_DATA_BREAK_LINKS, | |
ae9156b6 PC |
1039 | }; |
1040 | ||
042427ea SV |
1041 | static const struct jz4780_dma_soc_data jz4755_dma_soc_data = { |
1042 | .nb_channels = 4, | |
1043 | .transfer_ord_max = 5, | |
1044 | .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | | |
1045 | JZ_SOC_DATA_BREAK_LINKS, | |
1046 | }; | |
1047 | ||
d2852a3e PC |
1048 | static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { |
1049 | .nb_channels = 5, | |
1050 | .transfer_ord_max = 6, | |
1051 | .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, | |
1052 | }; | |
1053 | ||
3d70fccf PC |
1054 | static const struct jz4780_dma_soc_data jz4760_mdma_soc_data = { |
1055 | .nb_channels = 2, | |
1056 | .transfer_ord_max = 6, | |
1057 | .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, | |
1058 | }; | |
1059 | ||
1060 | static const struct jz4780_dma_soc_data jz4760_bdma_soc_data = { | |
1061 | .nb_channels = 3, | |
1062 | .transfer_ord_max = 6, | |
1063 | .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, | |
1064 | }; | |
1065 | ||
d2852a3e PC |
1066 | static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { |
1067 | .nb_channels = 5, | |
1068 | .transfer_ord_max = 6, | |
1069 | .flags = JZ_SOC_DATA_PER_CHAN_PM, | |
1070 | }; | |
1071 | ||
3d70fccf PC |
1072 | static const struct jz4780_dma_soc_data jz4760b_mdma_soc_data = { |
1073 | .nb_channels = 2, | |
1074 | .transfer_ord_max = 6, | |
1075 | .flags = JZ_SOC_DATA_PER_CHAN_PM, | |
1076 | }; | |
1077 | ||
1078 | static const struct jz4780_dma_soc_data jz4760b_bdma_soc_data = { | |
1079 | .nb_channels = 3, | |
1080 | .transfer_ord_max = 6, | |
1081 | .flags = JZ_SOC_DATA_PER_CHAN_PM, | |
1082 | }; | |
1083 | ||
29870eb7 PC |
1084 | static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { |
1085 | .nb_channels = 6, | |
1086 | .transfer_ord_max = 6, | |
1087 | .flags = JZ_SOC_DATA_PER_CHAN_PM, | |
1088 | }; | |
1089 | ||
6147b032 PC |
1090 | static const struct jz4780_dma_soc_data jz4780_dma_soc_data = { |
1091 | .nb_channels = 32, | |
29870eb7 PC |
1092 | .transfer_ord_max = 7, |
1093 | .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA, | |
6147b032 PC |
1094 | }; |
1095 | ||
fee175e4 ZY |
1096 | static const struct jz4780_dma_soc_data x1000_dma_soc_data = { |
1097 | .nb_channels = 8, | |
1098 | .transfer_ord_max = 7, | |
1099 | .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, | |
1100 | }; | |
1101 | ||
20f5a659 ZY |
1102 | static const struct jz4780_dma_soc_data x1830_dma_soc_data = { |
1103 | .nb_channels = 32, | |
1104 | .transfer_ord_max = 7, | |
1105 | .flags = JZ_SOC_DATA_PROGRAMMABLE_DMA, | |
1106 | }; | |
1107 | ||
d894fc60 | 1108 | static const struct of_device_id jz4780_dma_dt_match[] = { |
ffaaa8cc | 1109 | { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, |
ae9156b6 | 1110 | { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, |
042427ea | 1111 | { .compatible = "ingenic,jz4755-dma", .data = &jz4755_dma_soc_data }, |
d2852a3e | 1112 | { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, |
3d70fccf PC |
1113 | { .compatible = "ingenic,jz4760-mdma", .data = &jz4760_mdma_soc_data }, |
1114 | { .compatible = "ingenic,jz4760-bdma", .data = &jz4760_bdma_soc_data }, | |
d2852a3e | 1115 | { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, |
3d70fccf PC |
1116 | { .compatible = "ingenic,jz4760b-mdma", .data = &jz4760b_mdma_soc_data }, |
1117 | { .compatible = "ingenic,jz4760b-bdma", .data = &jz4760b_bdma_soc_data }, | |
29870eb7 | 1118 | { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, |
6147b032 | 1119 | { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, |
fee175e4 | 1120 | { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, |
20f5a659 | 1121 | { .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data }, |
d894fc60 AS |
1122 | {}, |
1123 | }; | |
1124 | MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); | |
1125 | ||
1126 | static struct platform_driver jz4780_dma_driver = { | |
1127 | .probe = jz4780_dma_probe, | |
1128 | .remove = jz4780_dma_remove, | |
1129 | .driver = { | |
1130 | .name = "jz4780-dma", | |
255c2cc8 | 1131 | .of_match_table = jz4780_dma_dt_match, |
d894fc60 AS |
1132 | }, |
1133 | }; | |
1134 | ||
1135 | static int __init jz4780_dma_init(void) | |
1136 | { | |
1137 | return platform_driver_register(&jz4780_dma_driver); | |
1138 | } | |
1139 | subsys_initcall(jz4780_dma_init); | |
1140 | ||
1141 | static void __exit jz4780_dma_exit(void) | |
1142 | { | |
1143 | platform_driver_unregister(&jz4780_dma_driver); | |
1144 | } | |
1145 | module_exit(jz4780_dma_exit); | |
1146 | ||
1147 | MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); | |
1148 | MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); | |
1149 | MODULE_LICENSE("GPL"); |