Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
55585930 MR |
2 | /* |
3 | * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd | |
4 | * Author: Sugar <shuge@allwinnertech.com> | |
5 | * | |
6 | * Copyright (C) 2014 Maxime Ripard | |
7 | * Maxime Ripard <maxime.ripard@free-electrons.com> | |
55585930 MR |
8 | */ |
9 | ||
10 | #include <linux/clk.h> | |
11 | #include <linux/delay.h> | |
12 | #include <linux/dmaengine.h> | |
13 | #include <linux/dmapool.h> | |
14 | #include <linux/interrupt.h> | |
15 | #include <linux/module.h> | |
16 | #include <linux/of_dma.h> | |
25a37c2f | 17 | #include <linux/of_device.h> |
55585930 MR |
18 | #include <linux/platform_device.h> |
19 | #include <linux/reset.h> | |
20 | #include <linux/slab.h> | |
21 | #include <linux/types.h> | |
22 | ||
23 | #include "virt-dma.h" | |
24 | ||
55585930 MR |
25 | /* |
26 | * Common registers | |
27 | */ | |
28 | #define DMA_IRQ_EN(x) ((x) * 0x04) | |
29 | #define DMA_IRQ_HALF BIT(0) | |
30 | #define DMA_IRQ_PKG BIT(1) | |
31 | #define DMA_IRQ_QUEUE BIT(2) | |
32 | ||
33 | #define DMA_IRQ_CHAN_NR 8 | |
34 | #define DMA_IRQ_CHAN_WIDTH 4 | |
35 | ||
36 | ||
37 | #define DMA_IRQ_STAT(x) ((x) * 0x04 + 0x10) | |
38 | ||
39 | #define DMA_STAT 0x30 | |
40 | ||
464aa6f5 SB |
41 | /* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */ |
42 | #define DMA_MAX_CHANNELS (DMA_IRQ_CHAN_NR * 0x10 / 4) | |
43 | ||
0b04ddf8 CYT |
44 | /* |
45 | * sun8i specific registers | |
46 | */ | |
47 | #define SUN8I_DMA_GATE 0x20 | |
48 | #define SUN8I_DMA_GATE_ENABLE 0x4 | |
49 | ||
50b12497 SB |
50 | #define SUNXI_H3_SECURE_REG 0x20 |
51 | #define SUNXI_H3_DMA_GATE 0x28 | |
52 | #define SUNXI_H3_DMA_GATE_ENABLE 0x4 | |
55585930 MR |
53 | /* |
54 | * Channels specific registers | |
55 | */ | |
56 | #define DMA_CHAN_ENABLE 0x00 | |
57 | #define DMA_CHAN_ENABLE_START BIT(0) | |
58 | #define DMA_CHAN_ENABLE_STOP 0 | |
59 | ||
60 | #define DMA_CHAN_PAUSE 0x04 | |
61 | #define DMA_CHAN_PAUSE_PAUSE BIT(1) | |
62 | #define DMA_CHAN_PAUSE_RESUME 0 | |
63 | ||
64 | #define DMA_CHAN_LLI_ADDR 0x08 | |
65 | ||
66 | #define DMA_CHAN_CUR_CFG 0x0c | |
67f34055 | 67 | #define DMA_CHAN_MAX_DRQ_A31 0x1f |
2fe5575f | 68 | #define DMA_CHAN_MAX_DRQ_H6 0x3f |
67f34055 | 69 | #define DMA_CHAN_CFG_SRC_DRQ_A31(x) ((x) & DMA_CHAN_MAX_DRQ_A31) |
2fe5575f | 70 | #define DMA_CHAN_CFG_SRC_DRQ_H6(x) ((x) & DMA_CHAN_MAX_DRQ_H6) |
802440bd | 71 | #define DMA_CHAN_CFG_SRC_MODE_A31(x) (((x) & 0x1) << 5) |
2fe5575f | 72 | #define DMA_CHAN_CFG_SRC_MODE_H6(x) (((x) & 0x1) << 8) |
5a6a6202 SB |
73 | #define DMA_CHAN_CFG_SRC_BURST_A31(x) (((x) & 0x3) << 7) |
74 | #define DMA_CHAN_CFG_SRC_BURST_H3(x) (((x) & 0x3) << 6) | |
55585930 MR |
75 | #define DMA_CHAN_CFG_SRC_WIDTH(x) (((x) & 0x3) << 9) |
76 | ||
67f34055 | 77 | #define DMA_CHAN_CFG_DST_DRQ_A31(x) (DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16) |
2fe5575f | 78 | #define DMA_CHAN_CFG_DST_DRQ_H6(x) (DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16) |
802440bd | 79 | #define DMA_CHAN_CFG_DST_MODE_A31(x) (DMA_CHAN_CFG_SRC_MODE_A31(x) << 16) |
2fe5575f | 80 | #define DMA_CHAN_CFG_DST_MODE_H6(x) (DMA_CHAN_CFG_SRC_MODE_H6(x) << 16) |
5a6a6202 SB |
81 | #define DMA_CHAN_CFG_DST_BURST_A31(x) (DMA_CHAN_CFG_SRC_BURST_A31(x) << 16) |
82 | #define DMA_CHAN_CFG_DST_BURST_H3(x) (DMA_CHAN_CFG_SRC_BURST_H3(x) << 16) | |
55585930 MR |
83 | #define DMA_CHAN_CFG_DST_WIDTH(x) (DMA_CHAN_CFG_SRC_WIDTH(x) << 16) |
84 | ||
85 | #define DMA_CHAN_CUR_SRC 0x10 | |
86 | ||
87 | #define DMA_CHAN_CUR_DST 0x14 | |
88 | ||
89 | #define DMA_CHAN_CUR_CNT 0x18 | |
90 | ||
91 | #define DMA_CHAN_CUR_PARA 0x1c | |
92 | ||
93 | ||
94 | /* | |
95 | * Various hardware related defines | |
96 | */ | |
97 | #define LLI_LAST_ITEM 0xfffff800 | |
98 | #define NORMAL_WAIT 8 | |
99 | #define DRQ_SDRAM 1 | |
802440bd JS |
100 | #define LINEAR_MODE 0 |
101 | #define IO_MODE 1 | |
55585930 | 102 | |
50b12497 SB |
103 | /* forward declaration */ |
104 | struct sun6i_dma_dev; | |
105 | ||
25a37c2f CYT |
106 | /* |
107 | * Hardware channels / ports representation | |
108 | * | |
109 | * The hardware is used in several SoCs, with differing numbers | |
110 | * of channels and endpoints. This structure ties those numbers | |
111 | * to a certain compatible string. | |
112 | */ | |
113 | struct sun6i_dma_config { | |
114 | u32 nr_max_channels; | |
115 | u32 nr_max_requests; | |
116 | u32 nr_max_vchans; | |
0430a7c7 IZ |
117 | /* |
118 | * In the datasheets/user manuals of newer Allwinner SoCs, a special | |
119 | * bit (bit 2 at register 0x20) is present. | |
120 | * It's named "DMA MCLK interface circuit auto gating bit" in the | |
121 | * documents, and the footnote of this register says that this bit | |
122 | * should be set up when initializing the DMA controller. | |
123 | * Allwinner A23/A33 user manuals do not have this bit documented, | |
124 | * however these SoCs really have and need this bit, as seen in the | |
125 | * BSP kernel source code. | |
126 | */ | |
50b12497 | 127 | void (*clock_autogate_enable)(struct sun6i_dma_dev *); |
5a6a6202 | 128 | void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst); |
67f34055 | 129 | void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq); |
802440bd | 130 | void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode); |
88d8622c SB |
131 | u32 src_burst_lengths; |
132 | u32 dst_burst_lengths; | |
d5f6d8cf SB |
133 | u32 src_addr_widths; |
134 | u32 dst_addr_widths; | |
43a90fc7 | 135 | bool has_mbus_clk; |
25a37c2f CYT |
136 | }; |
137 | ||
55585930 MR |
138 | /* |
139 | * Hardware representation of the LLI | |
140 | * | |
141 | * The hardware will be fed the physical address of this structure, | |
142 | * and read its content in order to start the transfer. | |
143 | */ | |
144 | struct sun6i_dma_lli { | |
145 | u32 cfg; | |
146 | u32 src; | |
147 | u32 dst; | |
148 | u32 len; | |
149 | u32 para; | |
150 | u32 p_lli_next; | |
151 | ||
152 | /* | |
153 | * This field is not used by the DMA controller, but will be | |
154 | * used by the CPU to go through the list (mostly for dumping | |
155 | * or freeing it). | |
156 | */ | |
157 | struct sun6i_dma_lli *v_lli_next; | |
158 | }; | |
159 | ||
160 | ||
161 | struct sun6i_desc { | |
162 | struct virt_dma_desc vd; | |
163 | dma_addr_t p_lli; | |
164 | struct sun6i_dma_lli *v_lli; | |
165 | }; | |
166 | ||
167 | struct sun6i_pchan { | |
168 | u32 idx; | |
169 | void __iomem *base; | |
170 | struct sun6i_vchan *vchan; | |
171 | struct sun6i_desc *desc; | |
172 | struct sun6i_desc *done; | |
173 | }; | |
174 | ||
175 | struct sun6i_vchan { | |
176 | struct virt_dma_chan vc; | |
177 | struct list_head node; | |
178 | struct dma_slave_config cfg; | |
179 | struct sun6i_pchan *phy; | |
180 | u8 port; | |
a90e173f JFM |
181 | u8 irq_type; |
182 | bool cyclic; | |
55585930 MR |
183 | }; |
184 | ||
185 | struct sun6i_dma_dev { | |
186 | struct dma_device slave; | |
187 | void __iomem *base; | |
188 | struct clk *clk; | |
43a90fc7 | 189 | struct clk *clk_mbus; |
55585930 MR |
190 | int irq; |
191 | spinlock_t lock; | |
192 | struct reset_control *rstc; | |
193 | struct tasklet_struct task; | |
194 | atomic_t tasklet_shutdown; | |
195 | struct list_head pending; | |
196 | struct dma_pool *pool; | |
197 | struct sun6i_pchan *pchans; | |
198 | struct sun6i_vchan *vchans; | |
25a37c2f | 199 | const struct sun6i_dma_config *cfg; |
500fa9e7 SB |
200 | u32 num_pchans; |
201 | u32 num_vchans; | |
202 | u32 max_request; | |
55585930 MR |
203 | }; |
204 | ||
205 | static struct device *chan2dev(struct dma_chan *chan) | |
206 | { | |
207 | return &chan->dev->device; | |
208 | } | |
209 | ||
210 | static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d) | |
211 | { | |
212 | return container_of(d, struct sun6i_dma_dev, slave); | |
213 | } | |
214 | ||
215 | static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan) | |
216 | { | |
217 | return container_of(chan, struct sun6i_vchan, vc.chan); | |
218 | } | |
219 | ||
220 | static inline struct sun6i_desc * | |
221 | to_sun6i_desc(struct dma_async_tx_descriptor *tx) | |
222 | { | |
223 | return container_of(tx, struct sun6i_desc, vd.tx); | |
224 | } | |
225 | ||
226 | static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) | |
227 | { | |
228 | dev_dbg(sdev->slave.dev, "Common register:\n" | |
229 | "\tmask0(%04x): 0x%08x\n" | |
230 | "\tmask1(%04x): 0x%08x\n" | |
231 | "\tpend0(%04x): 0x%08x\n" | |
232 | "\tpend1(%04x): 0x%08x\n" | |
233 | "\tstats(%04x): 0x%08x\n", | |
234 | DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)), | |
235 | DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)), | |
236 | DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)), | |
237 | DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)), | |
238 | DMA_STAT, readl(sdev->base + DMA_STAT)); | |
239 | } | |
240 | ||
241 | static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, | |
242 | struct sun6i_pchan *pchan) | |
243 | { | |
42c0d54e | 244 | phys_addr_t reg = virt_to_phys(pchan->base); |
55585930 MR |
245 | |
246 | dev_dbg(sdev->slave.dev, "Chan %d reg: %pa\n" | |
247 | "\t___en(%04x): \t0x%08x\n" | |
248 | "\tpause(%04x): \t0x%08x\n" | |
249 | "\tstart(%04x): \t0x%08x\n" | |
250 | "\t__cfg(%04x): \t0x%08x\n" | |
251 | "\t__src(%04x): \t0x%08x\n" | |
252 | "\t__dst(%04x): \t0x%08x\n" | |
253 | "\tcount(%04x): \t0x%08x\n" | |
254 | "\t_para(%04x): \t0x%08x\n\n", | |
255 | pchan->idx, ®, | |
256 | DMA_CHAN_ENABLE, | |
257 | readl(pchan->base + DMA_CHAN_ENABLE), | |
258 | DMA_CHAN_PAUSE, | |
259 | readl(pchan->base + DMA_CHAN_PAUSE), | |
260 | DMA_CHAN_LLI_ADDR, | |
261 | readl(pchan->base + DMA_CHAN_LLI_ADDR), | |
262 | DMA_CHAN_CUR_CFG, | |
263 | readl(pchan->base + DMA_CHAN_CUR_CFG), | |
264 | DMA_CHAN_CUR_SRC, | |
265 | readl(pchan->base + DMA_CHAN_CUR_SRC), | |
266 | DMA_CHAN_CUR_DST, | |
267 | readl(pchan->base + DMA_CHAN_CUR_DST), | |
268 | DMA_CHAN_CUR_CNT, | |
269 | readl(pchan->base + DMA_CHAN_CUR_CNT), | |
270 | DMA_CHAN_CUR_PARA, | |
271 | readl(pchan->base + DMA_CHAN_CUR_PARA)); | |
272 | } | |
273 | ||
1f9cd915 | 274 | static inline s8 convert_burst(u32 maxburst) |
55585930 MR |
275 | { |
276 | switch (maxburst) { | |
277 | case 1: | |
1f9cd915 | 278 | return 0; |
d5f6d8cf SB |
279 | case 4: |
280 | return 1; | |
55585930 | 281 | case 8: |
1f9cd915 | 282 | return 2; |
d5f6d8cf SB |
283 | case 16: |
284 | return 3; | |
55585930 MR |
285 | default: |
286 | return -EINVAL; | |
287 | } | |
55585930 MR |
288 | } |
289 | ||
1f9cd915 | 290 | static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width) |
55585930 | 291 | { |
d5f6d8cf | 292 | return ilog2(addr_width); |
55585930 MR |
293 | } |
294 | ||
50b12497 SB |
295 | static void sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev *sdev) |
296 | { | |
297 | writel(SUN8I_DMA_GATE_ENABLE, sdev->base + SUN8I_DMA_GATE); | |
298 | } | |
299 | ||
300 | static void sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev *sdev) | |
301 | { | |
302 | writel(SUNXI_H3_DMA_GATE_ENABLE, sdev->base + SUNXI_H3_DMA_GATE); | |
303 | } | |
304 | ||
5a6a6202 SB |
305 | static void sun6i_set_burst_length_a31(u32 *p_cfg, s8 src_burst, s8 dst_burst) |
306 | { | |
307 | *p_cfg |= DMA_CHAN_CFG_SRC_BURST_A31(src_burst) | | |
308 | DMA_CHAN_CFG_DST_BURST_A31(dst_burst); | |
309 | } | |
310 | ||
311 | static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst) | |
312 | { | |
313 | *p_cfg |= DMA_CHAN_CFG_SRC_BURST_H3(src_burst) | | |
314 | DMA_CHAN_CFG_DST_BURST_H3(dst_burst); | |
315 | } | |
316 | ||
67f34055 JS |
317 | static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq) |
318 | { | |
319 | *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) | | |
320 | DMA_CHAN_CFG_DST_DRQ_A31(dst_drq); | |
321 | } | |
322 | ||
2fe5575f JS |
323 | static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq) |
324 | { | |
325 | *p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) | | |
326 | DMA_CHAN_CFG_DST_DRQ_H6(dst_drq); | |
327 | } | |
328 | ||
802440bd JS |
329 | static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode) |
330 | { | |
331 | *p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) | | |
332 | DMA_CHAN_CFG_DST_MODE_A31(dst_mode); | |
333 | } | |
334 | ||
2fe5575f JS |
335 | static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode) |
336 | { | |
337 | *p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) | | |
338 | DMA_CHAN_CFG_DST_MODE_H6(dst_mode); | |
339 | } | |
340 | ||
a90e173f JFM |
341 | static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) |
342 | { | |
343 | struct sun6i_desc *txd = pchan->desc; | |
344 | struct sun6i_dma_lli *lli; | |
345 | size_t bytes; | |
346 | dma_addr_t pos; | |
347 | ||
348 | pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); | |
349 | bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); | |
350 | ||
351 | if (pos == LLI_LAST_ITEM) | |
352 | return bytes; | |
353 | ||
354 | for (lli = txd->v_lli; lli; lli = lli->v_lli_next) { | |
355 | if (lli->p_lli_next == pos) { | |
356 | for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next) | |
357 | bytes += lli->len; | |
358 | break; | |
359 | } | |
360 | } | |
361 | ||
362 | return bytes; | |
363 | } | |
364 | ||
55585930 MR |
365 | static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, |
366 | struct sun6i_dma_lli *next, | |
367 | dma_addr_t next_phy, | |
368 | struct sun6i_desc *txd) | |
369 | { | |
370 | if ((!prev && !txd) || !next) | |
371 | return NULL; | |
372 | ||
373 | if (!prev) { | |
374 | txd->p_lli = next_phy; | |
375 | txd->v_lli = next; | |
376 | } else { | |
377 | prev->p_lli_next = next_phy; | |
378 | prev->v_lli_next = next; | |
379 | } | |
380 | ||
381 | next->p_lli_next = LLI_LAST_ITEM; | |
382 | next->v_lli_next = NULL; | |
383 | ||
384 | return next; | |
385 | } | |
386 | ||
55585930 MR |
387 | static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, |
388 | struct sun6i_dma_lli *lli) | |
389 | { | |
42c0d54e | 390 | phys_addr_t p_lli = virt_to_phys(lli); |
55585930 MR |
391 | |
392 | dev_dbg(chan2dev(&vchan->vc.chan), | |
393 | "\n\tdesc: p - %pa v - 0x%p\n" | |
394 | "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" | |
395 | "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", | |
396 | &p_lli, lli, | |
397 | lli->cfg, lli->src, lli->dst, | |
398 | lli->len, lli->para, lli->p_lli_next); | |
399 | } | |
400 | ||
401 | static void sun6i_dma_free_desc(struct virt_dma_desc *vd) | |
402 | { | |
403 | struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); | |
404 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device); | |
405 | struct sun6i_dma_lli *v_lli, *v_next; | |
406 | dma_addr_t p_lli, p_next; | |
407 | ||
408 | if (unlikely(!txd)) | |
409 | return; | |
410 | ||
411 | p_lli = txd->p_lli; | |
412 | v_lli = txd->v_lli; | |
413 | ||
414 | while (v_lli) { | |
415 | v_next = v_lli->v_lli_next; | |
416 | p_next = v_lli->p_lli_next; | |
417 | ||
418 | dma_pool_free(sdev->pool, v_lli, p_lli); | |
419 | ||
420 | v_lli = v_next; | |
421 | p_lli = p_next; | |
422 | } | |
423 | ||
424 | kfree(txd); | |
425 | } | |
426 | ||
55585930 MR |
427 | static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) |
428 | { | |
429 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device); | |
430 | struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc); | |
431 | struct sun6i_pchan *pchan = vchan->phy; | |
432 | u32 irq_val, irq_reg, irq_offset; | |
433 | ||
434 | if (!pchan) | |
435 | return -EAGAIN; | |
436 | ||
437 | if (!desc) { | |
438 | pchan->desc = NULL; | |
439 | pchan->done = NULL; | |
440 | return -EAGAIN; | |
441 | } | |
442 | ||
443 | list_del(&desc->node); | |
444 | ||
445 | pchan->desc = to_sun6i_desc(&desc->tx); | |
446 | pchan->done = NULL; | |
447 | ||
448 | sun6i_dma_dump_lli(vchan, pchan->desc->v_lli); | |
449 | ||
450 | irq_reg = pchan->idx / DMA_IRQ_CHAN_NR; | |
451 | irq_offset = pchan->idx % DMA_IRQ_CHAN_NR; | |
452 | ||
a90e173f JFM |
453 | vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE; |
454 | ||
128fe7e9 | 455 | irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg)); |
a90e173f JFM |
456 | irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) << |
457 | (irq_offset * DMA_IRQ_CHAN_WIDTH)); | |
458 | irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH); | |
128fe7e9 | 459 | writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg)); |
55585930 MR |
460 | |
461 | writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR); | |
462 | writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE); | |
463 | ||
464 | sun6i_dma_dump_com_regs(sdev); | |
465 | sun6i_dma_dump_chan_regs(sdev, pchan); | |
466 | ||
467 | return 0; | |
468 | } | |
469 | ||
aaf9d3d6 | 470 | static void sun6i_dma_tasklet(struct tasklet_struct *t) |
55585930 | 471 | { |
aaf9d3d6 | 472 | struct sun6i_dma_dev *sdev = from_tasklet(sdev, t, task); |
55585930 MR |
473 | struct sun6i_vchan *vchan; |
474 | struct sun6i_pchan *pchan; | |
475 | unsigned int pchan_alloc = 0; | |
476 | unsigned int pchan_idx; | |
477 | ||
478 | list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) { | |
479 | spin_lock_irq(&vchan->vc.lock); | |
480 | ||
481 | pchan = vchan->phy; | |
482 | ||
483 | if (pchan && pchan->done) { | |
484 | if (sun6i_dma_start_desc(vchan)) { | |
485 | /* | |
486 | * No current txd associated with this channel | |
487 | */ | |
488 | dev_dbg(sdev->slave.dev, "pchan %u: free\n", | |
489 | pchan->idx); | |
490 | ||
491 | /* Mark this channel free */ | |
492 | vchan->phy = NULL; | |
493 | pchan->vchan = NULL; | |
494 | } | |
495 | } | |
496 | spin_unlock_irq(&vchan->vc.lock); | |
497 | } | |
498 | ||
499 | spin_lock_irq(&sdev->lock); | |
500fa9e7 | 500 | for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { |
55585930 MR |
501 | pchan = &sdev->pchans[pchan_idx]; |
502 | ||
503 | if (pchan->vchan || list_empty(&sdev->pending)) | |
504 | continue; | |
505 | ||
506 | vchan = list_first_entry(&sdev->pending, | |
507 | struct sun6i_vchan, node); | |
508 | ||
509 | /* Remove from pending channels */ | |
510 | list_del_init(&vchan->node); | |
511 | pchan_alloc |= BIT(pchan_idx); | |
512 | ||
513 | /* Mark this channel allocated */ | |
514 | pchan->vchan = vchan; | |
515 | vchan->phy = pchan; | |
516 | dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n", | |
517 | pchan->idx, &vchan->vc); | |
518 | } | |
519 | spin_unlock_irq(&sdev->lock); | |
520 | ||
500fa9e7 | 521 | for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) { |
55585930 MR |
522 | if (!(pchan_alloc & BIT(pchan_idx))) |
523 | continue; | |
524 | ||
525 | pchan = sdev->pchans + pchan_idx; | |
526 | vchan = pchan->vchan; | |
527 | if (vchan) { | |
528 | spin_lock_irq(&vchan->vc.lock); | |
529 | sun6i_dma_start_desc(vchan); | |
530 | spin_unlock_irq(&vchan->vc.lock); | |
531 | } | |
532 | } | |
533 | } | |
534 | ||
535 | static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) | |
536 | { | |
537 | struct sun6i_dma_dev *sdev = dev_id; | |
538 | struct sun6i_vchan *vchan; | |
539 | struct sun6i_pchan *pchan; | |
540 | int i, j, ret = IRQ_NONE; | |
541 | u32 status; | |
542 | ||
500fa9e7 | 543 | for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) { |
55585930 MR |
544 | status = readl(sdev->base + DMA_IRQ_STAT(i)); |
545 | if (!status) | |
546 | continue; | |
547 | ||
548 | dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", | |
549 | i ? "high" : "low", status); | |
550 | ||
551 | writel(status, sdev->base + DMA_IRQ_STAT(i)); | |
552 | ||
25a37c2f | 553 | for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) { |
a90e173f JFM |
554 | pchan = sdev->pchans + j; |
555 | vchan = pchan->vchan; | |
556 | if (vchan && (status & vchan->irq_type)) { | |
557 | if (vchan->cyclic) { | |
558 | vchan_cyclic_callback(&pchan->desc->vd); | |
559 | } else { | |
55585930 MR |
560 | spin_lock(&vchan->vc.lock); |
561 | vchan_cookie_complete(&pchan->desc->vd); | |
562 | pchan->done = pchan->desc; | |
563 | spin_unlock(&vchan->vc.lock); | |
564 | } | |
565 | } | |
566 | ||
25a37c2f | 567 | status = status >> DMA_IRQ_CHAN_WIDTH; |
55585930 MR |
568 | } |
569 | ||
570 | if (!atomic_read(&sdev->tasklet_shutdown)) | |
571 | tasklet_schedule(&sdev->task); | |
572 | ret = IRQ_HANDLED; | |
573 | } | |
574 | ||
575 | return ret; | |
576 | } | |
577 | ||
52c87179 JFM |
578 | static int set_config(struct sun6i_dma_dev *sdev, |
579 | struct dma_slave_config *sconfig, | |
580 | enum dma_transfer_direction direction, | |
581 | u32 *p_cfg) | |
582 | { | |
88d8622c SB |
583 | enum dma_slave_buswidth src_addr_width, dst_addr_width; |
584 | u32 src_maxburst, dst_maxburst; | |
52c87179 JFM |
585 | s8 src_width, dst_width, src_burst, dst_burst; |
586 | ||
88d8622c SB |
587 | src_addr_width = sconfig->src_addr_width; |
588 | dst_addr_width = sconfig->dst_addr_width; | |
589 | src_maxburst = sconfig->src_maxburst; | |
590 | dst_maxburst = sconfig->dst_maxburst; | |
591 | ||
a4eb36b0 JFM |
592 | switch (direction) { |
593 | case DMA_MEM_TO_DEV: | |
88d8622c SB |
594 | if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) |
595 | src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; | |
596 | src_maxburst = src_maxburst ? src_maxburst : 8; | |
a4eb36b0 JFM |
597 | break; |
598 | case DMA_DEV_TO_MEM: | |
88d8622c SB |
599 | if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) |
600 | dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; | |
601 | dst_maxburst = dst_maxburst ? dst_maxburst : 8; | |
a4eb36b0 JFM |
602 | break; |
603 | default: | |
604 | return -EINVAL; | |
605 | } | |
52c87179 | 606 | |
88d8622c SB |
607 | if (!(BIT(src_addr_width) & sdev->slave.src_addr_widths)) |
608 | return -EINVAL; | |
609 | if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths)) | |
610 | return -EINVAL; | |
611 | if (!(BIT(src_maxburst) & sdev->cfg->src_burst_lengths)) | |
612 | return -EINVAL; | |
613 | if (!(BIT(dst_maxburst) & sdev->cfg->dst_burst_lengths)) | |
614 | return -EINVAL; | |
615 | ||
616 | src_width = convert_buswidth(src_addr_width); | |
617 | dst_width = convert_buswidth(dst_addr_width); | |
618 | dst_burst = convert_burst(dst_maxburst); | |
619 | src_burst = convert_burst(src_maxburst); | |
52c87179 | 620 | |
5a6a6202 | 621 | *p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) | |
52c87179 JFM |
622 | DMA_CHAN_CFG_DST_WIDTH(dst_width); |
623 | ||
5a6a6202 SB |
624 | sdev->cfg->set_burst_length(p_cfg, src_burst, dst_burst); |
625 | ||
52c87179 JFM |
626 | return 0; |
627 | } | |
628 | ||
55585930 MR |
629 | static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( |
630 | struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, | |
631 | size_t len, unsigned long flags) | |
632 | { | |
633 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
634 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
55585930 MR |
635 | struct sun6i_dma_lli *v_lli; |
636 | struct sun6i_desc *txd; | |
637 | dma_addr_t p_lli; | |
1f9cd915 | 638 | s8 burst, width; |
55585930 MR |
639 | |
640 | dev_dbg(chan2dev(chan), | |
641 | "%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n", | |
642 | __func__, vchan->vc.chan.chan_id, &dest, &src, len, flags); | |
643 | ||
644 | if (!len) | |
645 | return NULL; | |
646 | ||
647 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); | |
648 | if (!txd) | |
649 | return NULL; | |
650 | ||
651 | v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); | |
652 | if (!v_lli) { | |
653 | dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); | |
4fbd804e | 654 | goto err_txd_free; |
55585930 MR |
655 | } |
656 | ||
1f9cd915 MR |
657 | v_lli->src = src; |
658 | v_lli->dst = dest; | |
659 | v_lli->len = len; | |
660 | v_lli->para = NORMAL_WAIT; | |
55585930 | 661 | |
1f9cd915 MR |
662 | burst = convert_burst(8); |
663 | width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); | |
802440bd | 664 | v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) | |
1f9cd915 | 665 | DMA_CHAN_CFG_DST_WIDTH(width); |
55585930 | 666 | |
5a6a6202 | 667 | sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst); |
67f34055 | 668 | sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM); |
802440bd | 669 | sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE); |
5a6a6202 | 670 | |
55585930 MR |
671 | sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); |
672 | ||
673 | sun6i_dma_dump_lli(vchan, v_lli); | |
674 | ||
675 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
676 | ||
4fbd804e MR |
677 | err_txd_free: |
678 | kfree(txd); | |
55585930 MR |
679 | return NULL; |
680 | } | |
681 | ||
682 | static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( | |
683 | struct dma_chan *chan, struct scatterlist *sgl, | |
684 | unsigned int sg_len, enum dma_transfer_direction dir, | |
685 | unsigned long flags, void *context) | |
686 | { | |
687 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
688 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
689 | struct dma_slave_config *sconfig = &vchan->cfg; | |
690 | struct sun6i_dma_lli *v_lli, *prev = NULL; | |
691 | struct sun6i_desc *txd; | |
692 | struct scatterlist *sg; | |
693 | dma_addr_t p_lli; | |
52c87179 | 694 | u32 lli_cfg; |
55585930 MR |
695 | int i, ret; |
696 | ||
697 | if (!sgl) | |
698 | return NULL; | |
699 | ||
52c87179 JFM |
700 | ret = set_config(sdev, sconfig, dir, &lli_cfg); |
701 | if (ret) { | |
702 | dev_err(chan2dev(chan), "Invalid DMA configuration\n"); | |
703 | return NULL; | |
704 | } | |
705 | ||
55585930 MR |
706 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); |
707 | if (!txd) | |
708 | return NULL; | |
709 | ||
710 | for_each_sg(sgl, sg, sg_len, i) { | |
711 | v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); | |
4fbd804e MR |
712 | if (!v_lli) |
713 | goto err_lli_free; | |
55585930 | 714 | |
52c87179 JFM |
715 | v_lli->len = sg_dma_len(sg); |
716 | v_lli->para = NORMAL_WAIT; | |
55585930 | 717 | |
52c87179 JFM |
718 | if (dir == DMA_MEM_TO_DEV) { |
719 | v_lli->src = sg_dma_address(sg); | |
720 | v_lli->dst = sconfig->dst_addr; | |
802440bd | 721 | v_lli->cfg = lli_cfg; |
67f34055 | 722 | sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); |
802440bd | 723 | sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); |
55585930 MR |
724 | |
725 | dev_dbg(chan2dev(chan), | |
7f5e03e7 | 726 | "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", |
55585930 MR |
727 | __func__, vchan->vc.chan.chan_id, |
728 | &sconfig->dst_addr, &sg_dma_address(sg), | |
729 | sg_dma_len(sg), flags); | |
730 | ||
731 | } else { | |
52c87179 JFM |
732 | v_lli->src = sconfig->src_addr; |
733 | v_lli->dst = sg_dma_address(sg); | |
802440bd | 734 | v_lli->cfg = lli_cfg; |
67f34055 | 735 | sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); |
802440bd | 736 | sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); |
55585930 MR |
737 | |
738 | dev_dbg(chan2dev(chan), | |
7f5e03e7 | 739 | "%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n", |
55585930 MR |
740 | __func__, vchan->vc.chan.chan_id, |
741 | &sg_dma_address(sg), &sconfig->src_addr, | |
742 | sg_dma_len(sg), flags); | |
743 | } | |
744 | ||
745 | prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); | |
746 | } | |
747 | ||
748 | dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli); | |
749 | for (prev = txd->v_lli; prev; prev = prev->v_lli_next) | |
750 | sun6i_dma_dump_lli(vchan, prev); | |
751 | ||
752 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
753 | ||
4fbd804e MR |
754 | err_lli_free: |
755 | for (prev = txd->v_lli; prev; prev = prev->v_lli_next) | |
756 | dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); | |
757 | kfree(txd); | |
55585930 MR |
758 | return NULL; |
759 | } | |
760 | ||
a90e173f JFM |
761 | static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( |
762 | struct dma_chan *chan, | |
763 | dma_addr_t buf_addr, | |
764 | size_t buf_len, | |
765 | size_t period_len, | |
766 | enum dma_transfer_direction dir, | |
767 | unsigned long flags) | |
768 | { | |
769 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
770 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
771 | struct dma_slave_config *sconfig = &vchan->cfg; | |
772 | struct sun6i_dma_lli *v_lli, *prev = NULL; | |
773 | struct sun6i_desc *txd; | |
774 | dma_addr_t p_lli; | |
775 | u32 lli_cfg; | |
776 | unsigned int i, periods = buf_len / period_len; | |
777 | int ret; | |
778 | ||
779 | ret = set_config(sdev, sconfig, dir, &lli_cfg); | |
780 | if (ret) { | |
781 | dev_err(chan2dev(chan), "Invalid DMA configuration\n"); | |
782 | return NULL; | |
783 | } | |
784 | ||
785 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); | |
786 | if (!txd) | |
787 | return NULL; | |
788 | ||
789 | for (i = 0; i < periods; i++) { | |
790 | v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); | |
791 | if (!v_lli) { | |
792 | dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); | |
793 | goto err_lli_free; | |
794 | } | |
795 | ||
796 | v_lli->len = period_len; | |
797 | v_lli->para = NORMAL_WAIT; | |
798 | ||
799 | if (dir == DMA_MEM_TO_DEV) { | |
800 | v_lli->src = buf_addr + period_len * i; | |
801 | v_lli->dst = sconfig->dst_addr; | |
802440bd | 802 | v_lli->cfg = lli_cfg; |
67f34055 | 803 | sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); |
802440bd | 804 | sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); |
a90e173f JFM |
805 | } else { |
806 | v_lli->src = sconfig->src_addr; | |
807 | v_lli->dst = buf_addr + period_len * i; | |
802440bd | 808 | v_lli->cfg = lli_cfg; |
67f34055 | 809 | sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); |
802440bd | 810 | sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); |
a90e173f JFM |
811 | } |
812 | ||
813 | prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd); | |
814 | } | |
815 | ||
816 | prev->p_lli_next = txd->p_lli; /* cyclic list */ | |
817 | ||
818 | vchan->cyclic = true; | |
819 | ||
820 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
821 | ||
822 | err_lli_free: | |
823 | for (prev = txd->v_lli; prev; prev = prev->v_lli_next) | |
824 | dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); | |
825 | kfree(txd); | |
826 | return NULL; | |
827 | } | |
828 | ||
826b15a7 MR |
829 | static int sun6i_dma_config(struct dma_chan *chan, |
830 | struct dma_slave_config *config) | |
831 | { | |
832 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
833 | ||
834 | memcpy(&vchan->cfg, config, sizeof(*config)); | |
835 | ||
836 | return 0; | |
837 | } | |
838 | ||
839 | static int sun6i_dma_pause(struct dma_chan *chan) | |
840 | { | |
841 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
842 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
843 | struct sun6i_pchan *pchan = vchan->phy; | |
844 | ||
845 | dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc); | |
846 | ||
847 | if (pchan) { | |
848 | writel(DMA_CHAN_PAUSE_PAUSE, | |
849 | pchan->base + DMA_CHAN_PAUSE); | |
850 | } else { | |
851 | spin_lock(&sdev->lock); | |
852 | list_del_init(&vchan->node); | |
853 | spin_unlock(&sdev->lock); | |
854 | } | |
855 | ||
856 | return 0; | |
857 | } | |
858 | ||
859 | static int sun6i_dma_resume(struct dma_chan *chan) | |
55585930 MR |
860 | { |
861 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
862 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
863 | struct sun6i_pchan *pchan = vchan->phy; | |
864 | unsigned long flags; | |
55585930 | 865 | |
826b15a7 | 866 | dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); |
55585930 | 867 | |
826b15a7 | 868 | spin_lock_irqsave(&vchan->vc.lock, flags); |
55585930 | 869 | |
826b15a7 MR |
870 | if (pchan) { |
871 | writel(DMA_CHAN_PAUSE_RESUME, | |
872 | pchan->base + DMA_CHAN_PAUSE); | |
873 | } else if (!list_empty(&vchan->vc.desc_issued)) { | |
874 | spin_lock(&sdev->lock); | |
875 | list_add_tail(&vchan->node, &sdev->pending); | |
876 | spin_unlock(&sdev->lock); | |
877 | } | |
55585930 | 878 | |
826b15a7 | 879 | spin_unlock_irqrestore(&vchan->vc.lock, flags); |
55585930 | 880 | |
826b15a7 MR |
881 | return 0; |
882 | } | |
55585930 | 883 | |
826b15a7 MR |
884 | static int sun6i_dma_terminate_all(struct dma_chan *chan) |
885 | { | |
886 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
887 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
888 | struct sun6i_pchan *pchan = vchan->phy; | |
889 | unsigned long flags; | |
890 | LIST_HEAD(head); | |
891 | ||
892 | spin_lock(&sdev->lock); | |
893 | list_del_init(&vchan->node); | |
894 | spin_unlock(&sdev->lock); | |
895 | ||
896 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
897 | ||
a90e173f JFM |
898 | if (vchan->cyclic) { |
899 | vchan->cyclic = false; | |
900 | if (pchan && pchan->desc) { | |
901 | struct virt_dma_desc *vd = &pchan->desc->vd; | |
902 | struct virt_dma_chan *vc = &vchan->vc; | |
903 | ||
904 | list_add_tail(&vd->node, &vc->desc_completed); | |
905 | } | |
906 | } | |
907 | ||
826b15a7 MR |
908 | vchan_get_all_descriptors(&vchan->vc, &head); |
909 | ||
910 | if (pchan) { | |
911 | writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE); | |
912 | writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE); | |
913 | ||
914 | vchan->phy = NULL; | |
915 | pchan->vchan = NULL; | |
916 | pchan->desc = NULL; | |
917 | pchan->done = NULL; | |
55585930 | 918 | } |
826b15a7 MR |
919 | |
920 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
921 | ||
922 | vchan_dma_desc_free_list(&vchan->vc, &head); | |
923 | ||
924 | return 0; | |
55585930 MR |
925 | } |
926 | ||
927 | static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, | |
928 | dma_cookie_t cookie, | |
929 | struct dma_tx_state *state) | |
930 | { | |
931 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
932 | struct sun6i_pchan *pchan = vchan->phy; | |
933 | struct sun6i_dma_lli *lli; | |
934 | struct virt_dma_desc *vd; | |
935 | struct sun6i_desc *txd; | |
936 | enum dma_status ret; | |
937 | unsigned long flags; | |
938 | size_t bytes = 0; | |
939 | ||
940 | ret = dma_cookie_status(chan, cookie, state); | |
b9ab9d10 | 941 | if (ret == DMA_COMPLETE || !state) |
55585930 MR |
942 | return ret; |
943 | ||
944 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
945 | ||
946 | vd = vchan_find_desc(&vchan->vc, cookie); | |
947 | txd = to_sun6i_desc(&vd->tx); | |
948 | ||
949 | if (vd) { | |
950 | for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next) | |
951 | bytes += lli->len; | |
952 | } else if (!pchan || !pchan->desc) { | |
953 | bytes = 0; | |
954 | } else { | |
a90e173f | 955 | bytes = sun6i_get_chan_size(pchan); |
55585930 MR |
956 | } |
957 | ||
958 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
959 | ||
960 | dma_set_residue(state, bytes); | |
961 | ||
962 | return ret; | |
963 | } | |
964 | ||
965 | static void sun6i_dma_issue_pending(struct dma_chan *chan) | |
966 | { | |
967 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
968 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
969 | unsigned long flags; | |
970 | ||
971 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
972 | ||
973 | if (vchan_issue_pending(&vchan->vc)) { | |
974 | spin_lock(&sdev->lock); | |
975 | ||
976 | if (!vchan->phy && list_empty(&vchan->node)) { | |
977 | list_add_tail(&vchan->node, &sdev->pending); | |
978 | tasklet_schedule(&sdev->task); | |
979 | dev_dbg(chan2dev(chan), "vchan %p: issued\n", | |
980 | &vchan->vc); | |
981 | } | |
982 | ||
983 | spin_unlock(&sdev->lock); | |
984 | } else { | |
985 | dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n", | |
986 | &vchan->vc); | |
987 | } | |
988 | ||
989 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
990 | } | |
991 | ||
55585930 MR |
992 | static void sun6i_dma_free_chan_resources(struct dma_chan *chan) |
993 | { | |
994 | struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device); | |
995 | struct sun6i_vchan *vchan = to_sun6i_vchan(chan); | |
996 | unsigned long flags; | |
997 | ||
998 | spin_lock_irqsave(&sdev->lock, flags); | |
999 | list_del_init(&vchan->node); | |
1000 | spin_unlock_irqrestore(&sdev->lock, flags); | |
1001 | ||
1002 | vchan_free_chan_resources(&vchan->vc); | |
1003 | } | |
1004 | ||
1005 | static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec, | |
1006 | struct of_dma *ofdma) | |
1007 | { | |
1008 | struct sun6i_dma_dev *sdev = ofdma->of_dma_data; | |
1009 | struct sun6i_vchan *vchan; | |
1010 | struct dma_chan *chan; | |
1011 | u8 port = dma_spec->args[0]; | |
1012 | ||
500fa9e7 | 1013 | if (port > sdev->max_request) |
55585930 MR |
1014 | return NULL; |
1015 | ||
1016 | chan = dma_get_any_slave_channel(&sdev->slave); | |
1017 | if (!chan) | |
1018 | return NULL; | |
1019 | ||
1020 | vchan = to_sun6i_vchan(chan); | |
1021 | vchan->port = port; | |
1022 | ||
1023 | return chan; | |
1024 | } | |
1025 | ||
1026 | static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev) | |
1027 | { | |
1028 | /* Disable all interrupts from DMA */ | |
1029 | writel(0, sdev->base + DMA_IRQ_EN(0)); | |
1030 | writel(0, sdev->base + DMA_IRQ_EN(1)); | |
1031 | ||
1032 | /* Prevent spurious interrupts from scheduling the tasklet */ | |
1033 | atomic_inc(&sdev->tasklet_shutdown); | |
1034 | ||
174427c1 MR |
1035 | /* Make sure we won't have any further interrupts */ |
1036 | devm_free_irq(sdev->slave.dev, sdev->irq, sdev); | |
55585930 MR |
1037 | |
1038 | /* Actually prevent the tasklet from being scheduled */ | |
1039 | tasklet_kill(&sdev->task); | |
1040 | } | |
1041 | ||
1042 | static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev) | |
1043 | { | |
1044 | int i; | |
1045 | ||
500fa9e7 | 1046 | for (i = 0; i < sdev->num_vchans; i++) { |
55585930 MR |
1047 | struct sun6i_vchan *vchan = &sdev->vchans[i]; |
1048 | ||
1049 | list_del(&vchan->vc.chan.device_node); | |
1050 | tasklet_kill(&vchan->vc.task); | |
1051 | } | |
1052 | } | |
1053 | ||
25a37c2f CYT |
1054 | /* |
1055 | * For A31: | |
1056 | * | |
1057 | * There's 16 physical channels that can work in parallel. | |
1058 | * | |
1059 | * However we have 30 different endpoints for our requests. | |
1060 | * | |
1061 | * Since the channels are able to handle only an unidirectional | |
1062 | * transfer, we need to allocate more virtual channels so that | |
1063 | * everyone can grab one channel. | |
1064 | * | |
1065 | * Some devices can't work in both direction (mostly because it | |
1066 | * wouldn't make sense), so we have a bit fewer virtual channels than | |
1067 | * 2 channels per endpoints. | |
1068 | */ | |
1069 | ||
1070 | static struct sun6i_dma_config sun6i_a31_dma_cfg = { | |
1071 | .nr_max_channels = 16, | |
1072 | .nr_max_requests = 30, | |
1073 | .nr_max_vchans = 53, | |
5a6a6202 | 1074 | .set_burst_length = sun6i_set_burst_length_a31, |
67f34055 | 1075 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1076 | .set_mode = sun6i_set_mode_a31, |
88d8622c SB |
1077 | .src_burst_lengths = BIT(1) | BIT(8), |
1078 | .dst_burst_lengths = BIT(1) | BIT(8), | |
d5f6d8cf SB |
1079 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
1080 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1081 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
1082 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1083 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1084 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
25a37c2f CYT |
1085 | }; |
1086 | ||
0b04ddf8 CYT |
1087 | /* |
1088 | * The A23 only has 8 physical channels, a maximum DRQ port id of 24, | |
1089 | * and a total of 37 usable source and destination endpoints. | |
1090 | */ | |
1091 | ||
1092 | static struct sun6i_dma_config sun8i_a23_dma_cfg = { | |
1093 | .nr_max_channels = 8, | |
1094 | .nr_max_requests = 24, | |
1095 | .nr_max_vchans = 37, | |
50b12497 | 1096 | .clock_autogate_enable = sun6i_enable_clock_autogate_a23, |
5a6a6202 | 1097 | .set_burst_length = sun6i_set_burst_length_a31, |
67f34055 | 1098 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1099 | .set_mode = sun6i_set_mode_a31, |
88d8622c SB |
1100 | .src_burst_lengths = BIT(1) | BIT(8), |
1101 | .dst_burst_lengths = BIT(1) | BIT(8), | |
d5f6d8cf SB |
1102 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
1103 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1104 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
1105 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1106 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1107 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
0b04ddf8 CYT |
1108 | }; |
1109 | ||
3a03ea76 JFM |
1110 | static struct sun6i_dma_config sun8i_a83t_dma_cfg = { |
1111 | .nr_max_channels = 8, | |
1112 | .nr_max_requests = 28, | |
1113 | .nr_max_vchans = 39, | |
50b12497 | 1114 | .clock_autogate_enable = sun6i_enable_clock_autogate_a23, |
5a6a6202 | 1115 | .set_burst_length = sun6i_set_burst_length_a31, |
67f34055 | 1116 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1117 | .set_mode = sun6i_set_mode_a31, |
88d8622c SB |
1118 | .src_burst_lengths = BIT(1) | BIT(8), |
1119 | .dst_burst_lengths = BIT(1) | BIT(8), | |
d5f6d8cf SB |
1120 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
1121 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1122 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
1123 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1124 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1125 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
3a03ea76 JFM |
1126 | }; |
1127 | ||
f008db8c JK |
1128 | /* |
1129 | * The H3 has 12 physical channels, a maximum DRQ port id of 27, | |
1130 | * and a total of 34 usable source and destination endpoints. | |
d5f6d8cf SB |
1131 | * It also supports additional burst lengths and bus widths, |
1132 | * and the burst length fields have different offsets. | |
f008db8c JK |
1133 | */ |
1134 | ||
1135 | static struct sun6i_dma_config sun8i_h3_dma_cfg = { | |
1136 | .nr_max_channels = 12, | |
1137 | .nr_max_requests = 27, | |
1138 | .nr_max_vchans = 34, | |
50b12497 | 1139 | .clock_autogate_enable = sun6i_enable_clock_autogate_h3, |
5a6a6202 | 1140 | .set_burst_length = sun6i_set_burst_length_h3, |
67f34055 | 1141 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1142 | .set_mode = sun6i_set_mode_a31, |
d5f6d8cf SB |
1143 | .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), |
1144 | .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1145 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1146 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1147 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1148 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1149 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1150 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1151 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1152 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
f008db8c JK |
1153 | }; |
1154 | ||
12e01770 SB |
1155 | /* |
1156 | * The A64 binding uses the number of dma channels from the | |
1157 | * device tree node. | |
1158 | */ | |
1159 | static struct sun6i_dma_config sun50i_a64_dma_cfg = { | |
1160 | .clock_autogate_enable = sun6i_enable_clock_autogate_h3, | |
1161 | .set_burst_length = sun6i_set_burst_length_h3, | |
67f34055 | 1162 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1163 | .set_mode = sun6i_set_mode_a31, |
12e01770 SB |
1164 | .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), |
1165 | .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1166 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1167 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1168 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1169 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1170 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1171 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1172 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1173 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1174 | }; | |
1175 | ||
07b55273 YL |
1176 | /* |
1177 | * TODO: Add support for more than 4g physical addressing. | |
1178 | * | |
1179 | * The A100 binding uses the number of dma channels from the | |
1180 | * device tree node. | |
1181 | */ | |
1182 | static struct sun6i_dma_config sun50i_a100_dma_cfg = { | |
1183 | .clock_autogate_enable = sun6i_enable_clock_autogate_h3, | |
1184 | .set_burst_length = sun6i_set_burst_length_h3, | |
1185 | .set_drq = sun6i_set_drq_h6, | |
1186 | .set_mode = sun6i_set_mode_h6, | |
1187 | .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1188 | .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1189 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1190 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1191 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1192 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1193 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1194 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1195 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1196 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1197 | .has_mbus_clk = true, | |
1198 | }; | |
1199 | ||
2fe5575f JS |
1200 | /* |
1201 | * The H6 binding uses the number of dma channels from the | |
1202 | * device tree node. | |
1203 | */ | |
1204 | static struct sun6i_dma_config sun50i_h6_dma_cfg = { | |
1205 | .clock_autogate_enable = sun6i_enable_clock_autogate_h3, | |
1206 | .set_burst_length = sun6i_set_burst_length_h3, | |
1207 | .set_drq = sun6i_set_drq_h6, | |
1208 | .set_mode = sun6i_set_mode_h6, | |
1209 | .src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1210 | .dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16), | |
1211 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1212 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1213 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1214 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1215 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1216 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1217 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | | |
1218 | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), | |
1219 | .has_mbus_clk = true, | |
1220 | }; | |
1221 | ||
a702e47e IZ |
1222 | /* |
1223 | * The V3s have only 8 physical channels, a maximum DRQ port id of 23, | |
1224 | * and a total of 24 usable source and destination endpoints. | |
1225 | */ | |
1226 | ||
1227 | static struct sun6i_dma_config sun8i_v3s_dma_cfg = { | |
1228 | .nr_max_channels = 8, | |
1229 | .nr_max_requests = 23, | |
1230 | .nr_max_vchans = 24, | |
50b12497 | 1231 | .clock_autogate_enable = sun6i_enable_clock_autogate_a23, |
5a6a6202 | 1232 | .set_burst_length = sun6i_set_burst_length_a31, |
67f34055 | 1233 | .set_drq = sun6i_set_drq_a31, |
802440bd | 1234 | .set_mode = sun6i_set_mode_a31, |
88d8622c SB |
1235 | .src_burst_lengths = BIT(1) | BIT(8), |
1236 | .dst_burst_lengths = BIT(1) | BIT(8), | |
d5f6d8cf SB |
1237 | .src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
1238 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1239 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
1240 | .dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | | |
1241 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | | |
1242 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES), | |
a702e47e IZ |
1243 | }; |
1244 | ||
57c03422 | 1245 | static const struct of_device_id sun6i_dma_match[] = { |
25a37c2f | 1246 | { .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg }, |
0b04ddf8 | 1247 | { .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg }, |
3a03ea76 | 1248 | { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg }, |
f008db8c | 1249 | { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, |
a702e47e | 1250 | { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, |
12e01770 | 1251 | { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, |
07b55273 | 1252 | { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, |
2fe5575f | 1253 | { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, |
25a37c2f CYT |
1254 | { /* sentinel */ } |
1255 | }; | |
c719d7fa | 1256 | MODULE_DEVICE_TABLE(of, sun6i_dma_match); |
25a37c2f | 1257 | |
55585930 MR |
1258 | static int sun6i_dma_probe(struct platform_device *pdev) |
1259 | { | |
464aa6f5 | 1260 | struct device_node *np = pdev->dev.of_node; |
55585930 MR |
1261 | struct sun6i_dma_dev *sdc; |
1262 | struct resource *res; | |
55585930 MR |
1263 | int ret, i; |
1264 | ||
1265 | sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL); | |
1266 | if (!sdc) | |
1267 | return -ENOMEM; | |
1268 | ||
8f3b0034 CL |
1269 | sdc->cfg = of_device_get_match_data(&pdev->dev); |
1270 | if (!sdc->cfg) | |
25a37c2f | 1271 | return -ENODEV; |
25a37c2f | 1272 | |
55585930 MR |
1273 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
1274 | sdc->base = devm_ioremap_resource(&pdev->dev, res); | |
1275 | if (IS_ERR(sdc->base)) | |
1276 | return PTR_ERR(sdc->base); | |
1277 | ||
1278 | sdc->irq = platform_get_irq(pdev, 0); | |
e17be6e1 | 1279 | if (sdc->irq < 0) |
55585930 | 1280 | return sdc->irq; |
55585930 MR |
1281 | |
1282 | sdc->clk = devm_clk_get(&pdev->dev, NULL); | |
1283 | if (IS_ERR(sdc->clk)) { | |
1284 | dev_err(&pdev->dev, "No clock specified\n"); | |
1285 | return PTR_ERR(sdc->clk); | |
1286 | } | |
1287 | ||
43a90fc7 JS |
1288 | if (sdc->cfg->has_mbus_clk) { |
1289 | sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus"); | |
1290 | if (IS_ERR(sdc->clk_mbus)) { | |
1291 | dev_err(&pdev->dev, "No mbus clock specified\n"); | |
1292 | return PTR_ERR(sdc->clk_mbus); | |
1293 | } | |
1294 | } | |
1295 | ||
55585930 MR |
1296 | sdc->rstc = devm_reset_control_get(&pdev->dev, NULL); |
1297 | if (IS_ERR(sdc->rstc)) { | |
1298 | dev_err(&pdev->dev, "No reset controller specified\n"); | |
1299 | return PTR_ERR(sdc->rstc); | |
1300 | } | |
1301 | ||
1302 | sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev, | |
1303 | sizeof(struct sun6i_dma_lli), 4, 0); | |
1304 | if (!sdc->pool) { | |
1305 | dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); | |
1306 | return -ENOMEM; | |
1307 | } | |
1308 | ||
1309 | platform_set_drvdata(pdev, sdc); | |
1310 | INIT_LIST_HEAD(&sdc->pending); | |
1311 | spin_lock_init(&sdc->lock); | |
1312 | ||
1313 | dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask); | |
1314 | dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask); | |
1315 | dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask); | |
a90e173f | 1316 | dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask); |
55585930 MR |
1317 | |
1318 | INIT_LIST_HEAD(&sdc->slave.channels); | |
55585930 MR |
1319 | sdc->slave.device_free_chan_resources = sun6i_dma_free_chan_resources; |
1320 | sdc->slave.device_tx_status = sun6i_dma_tx_status; | |
1321 | sdc->slave.device_issue_pending = sun6i_dma_issue_pending; | |
1322 | sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; | |
1323 | sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; | |
a90e173f | 1324 | sdc->slave.device_prep_dma_cyclic = sun6i_dma_prep_dma_cyclic; |
77a68e56 | 1325 | sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES; |
826b15a7 MR |
1326 | sdc->slave.device_config = sun6i_dma_config; |
1327 | sdc->slave.device_pause = sun6i_dma_pause; | |
1328 | sdc->slave.device_resume = sun6i_dma_resume; | |
1329 | sdc->slave.device_terminate_all = sun6i_dma_terminate_all; | |
d5f6d8cf SB |
1330 | sdc->slave.src_addr_widths = sdc->cfg->src_addr_widths; |
1331 | sdc->slave.dst_addr_widths = sdc->cfg->dst_addr_widths; | |
1cac81b4 MR |
1332 | sdc->slave.directions = BIT(DMA_DEV_TO_MEM) | |
1333 | BIT(DMA_MEM_TO_DEV); | |
1334 | sdc->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; | |
55585930 MR |
1335 | sdc->slave.dev = &pdev->dev; |
1336 | ||
500fa9e7 SB |
1337 | sdc->num_pchans = sdc->cfg->nr_max_channels; |
1338 | sdc->num_vchans = sdc->cfg->nr_max_vchans; | |
1339 | sdc->max_request = sdc->cfg->nr_max_requests; | |
1340 | ||
464aa6f5 SB |
1341 | ret = of_property_read_u32(np, "dma-channels", &sdc->num_pchans); |
1342 | if (ret && !sdc->num_pchans) { | |
1343 | dev_err(&pdev->dev, "Can't get dma-channels.\n"); | |
1344 | return ret; | |
1345 | } | |
1346 | ||
1347 | ret = of_property_read_u32(np, "dma-requests", &sdc->max_request); | |
1348 | if (ret && !sdc->max_request) { | |
1349 | dev_info(&pdev->dev, "Missing dma-requests, using %u.\n", | |
67f34055 JS |
1350 | DMA_CHAN_MAX_DRQ_A31); |
1351 | sdc->max_request = DMA_CHAN_MAX_DRQ_A31; | |
464aa6f5 SB |
1352 | } |
1353 | ||
1354 | /* | |
1355 | * If the number of vchans is not specified, derive it from the | |
1356 | * highest port number, at most one channel per port and direction. | |
1357 | */ | |
1358 | if (!sdc->num_vchans) | |
1359 | sdc->num_vchans = 2 * (sdc->max_request + 1); | |
1360 | ||
500fa9e7 | 1361 | sdc->pchans = devm_kcalloc(&pdev->dev, sdc->num_pchans, |
55585930 MR |
1362 | sizeof(struct sun6i_pchan), GFP_KERNEL); |
1363 | if (!sdc->pchans) | |
1364 | return -ENOMEM; | |
1365 | ||
500fa9e7 | 1366 | sdc->vchans = devm_kcalloc(&pdev->dev, sdc->num_vchans, |
55585930 MR |
1367 | sizeof(struct sun6i_vchan), GFP_KERNEL); |
1368 | if (!sdc->vchans) | |
1369 | return -ENOMEM; | |
1370 | ||
aaf9d3d6 | 1371 | tasklet_setup(&sdc->task, sun6i_dma_tasklet); |
55585930 | 1372 | |
500fa9e7 | 1373 | for (i = 0; i < sdc->num_pchans; i++) { |
55585930 MR |
1374 | struct sun6i_pchan *pchan = &sdc->pchans[i]; |
1375 | ||
1376 | pchan->idx = i; | |
1377 | pchan->base = sdc->base + 0x100 + i * 0x40; | |
1378 | } | |
1379 | ||
500fa9e7 | 1380 | for (i = 0; i < sdc->num_vchans; i++) { |
55585930 MR |
1381 | struct sun6i_vchan *vchan = &sdc->vchans[i]; |
1382 | ||
1383 | INIT_LIST_HEAD(&vchan->node); | |
1384 | vchan->vc.desc_free = sun6i_dma_free_desc; | |
1385 | vchan_init(&vchan->vc, &sdc->slave); | |
1386 | } | |
1387 | ||
1388 | ret = reset_control_deassert(sdc->rstc); | |
1389 | if (ret) { | |
1390 | dev_err(&pdev->dev, "Couldn't deassert the device from reset\n"); | |
1391 | goto err_chan_free; | |
1392 | } | |
1393 | ||
1394 | ret = clk_prepare_enable(sdc->clk); | |
1395 | if (ret) { | |
1396 | dev_err(&pdev->dev, "Couldn't enable the clock\n"); | |
1397 | goto err_reset_assert; | |
1398 | } | |
1399 | ||
43a90fc7 JS |
1400 | if (sdc->cfg->has_mbus_clk) { |
1401 | ret = clk_prepare_enable(sdc->clk_mbus); | |
1402 | if (ret) { | |
1403 | dev_err(&pdev->dev, "Couldn't enable mbus clock\n"); | |
1404 | goto err_clk_disable; | |
1405 | } | |
1406 | } | |
1407 | ||
55585930 MR |
1408 | ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0, |
1409 | dev_name(&pdev->dev), sdc); | |
1410 | if (ret) { | |
1411 | dev_err(&pdev->dev, "Cannot request IRQ\n"); | |
43a90fc7 | 1412 | goto err_mbus_clk_disable; |
55585930 MR |
1413 | } |
1414 | ||
1415 | ret = dma_async_device_register(&sdc->slave); | |
1416 | if (ret) { | |
1417 | dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); | |
1418 | goto err_irq_disable; | |
1419 | } | |
1420 | ||
1421 | ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate, | |
1422 | sdc); | |
1423 | if (ret) { | |
1424 | dev_err(&pdev->dev, "of_dma_controller_register failed\n"); | |
1425 | goto err_dma_unregister; | |
1426 | } | |
1427 | ||
50b12497 SB |
1428 | if (sdc->cfg->clock_autogate_enable) |
1429 | sdc->cfg->clock_autogate_enable(sdc); | |
0b04ddf8 | 1430 | |
55585930 MR |
1431 | return 0; |
1432 | ||
1433 | err_dma_unregister: | |
1434 | dma_async_device_unregister(&sdc->slave); | |
1435 | err_irq_disable: | |
1436 | sun6i_kill_tasklet(sdc); | |
43a90fc7 JS |
1437 | err_mbus_clk_disable: |
1438 | clk_disable_unprepare(sdc->clk_mbus); | |
55585930 MR |
1439 | err_clk_disable: |
1440 | clk_disable_unprepare(sdc->clk); | |
1441 | err_reset_assert: | |
1442 | reset_control_assert(sdc->rstc); | |
1443 | err_chan_free: | |
1444 | sun6i_dma_free(sdc); | |
1445 | return ret; | |
1446 | } | |
1447 | ||
1448 | static int sun6i_dma_remove(struct platform_device *pdev) | |
1449 | { | |
1450 | struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev); | |
1451 | ||
1452 | of_dma_controller_free(pdev->dev.of_node); | |
1453 | dma_async_device_unregister(&sdc->slave); | |
1454 | ||
1455 | sun6i_kill_tasklet(sdc); | |
1456 | ||
43a90fc7 | 1457 | clk_disable_unprepare(sdc->clk_mbus); |
55585930 MR |
1458 | clk_disable_unprepare(sdc->clk); |
1459 | reset_control_assert(sdc->rstc); | |
1460 | ||
1461 | sun6i_dma_free(sdc); | |
1462 | ||
1463 | return 0; | |
1464 | } | |
1465 | ||
55585930 MR |
1466 | static struct platform_driver sun6i_dma_driver = { |
1467 | .probe = sun6i_dma_probe, | |
1468 | .remove = sun6i_dma_remove, | |
1469 | .driver = { | |
1470 | .name = "sun6i-dma", | |
1471 | .of_match_table = sun6i_dma_match, | |
1472 | }, | |
1473 | }; | |
1474 | module_platform_driver(sun6i_dma_driver); | |
1475 | ||
1476 | MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver"); | |
1477 | MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>"); | |
1478 | MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>"); | |
1479 | MODULE_LICENSE("GPL"); |