Commit | Line | Data |
---|---|---|
47e20577 MS |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // | |
3 | // Actions Semi Owl SoCs DMA driver | |
4 | // | |
5 | // Copyright (c) 2014 Actions Semi Inc. | |
6 | // Author: David Liu <liuwei@actions-semi.com> | |
7 | // | |
8 | // Copyright (c) 2018 Linaro Ltd. | |
9 | // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> | |
10 | ||
11 | #include <linux/bitops.h> | |
12 | #include <linux/clk.h> | |
13 | #include <linux/delay.h> | |
14 | #include <linux/dmaengine.h> | |
15 | #include <linux/dma-mapping.h> | |
16 | #include <linux/dmapool.h> | |
17 | #include <linux/err.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/io.h> | |
21 | #include <linux/mm.h> | |
22 | #include <linux/module.h> | |
897500c7 | 23 | #include <linux/of.h> |
d64e1b3f | 24 | #include <linux/of_dma.h> |
897500c7 | 25 | #include <linux/platform_device.h> |
47e20577 MS |
26 | #include <linux/slab.h> |
27 | #include "virt-dma.h" | |
28 | ||
29 | #define OWL_DMA_FRAME_MAX_LENGTH 0xfffff | |
30 | ||
31 | /* Global DMA Controller Registers */ | |
32 | #define OWL_DMA_IRQ_PD0 0x00 | |
33 | #define OWL_DMA_IRQ_PD1 0x04 | |
34 | #define OWL_DMA_IRQ_PD2 0x08 | |
35 | #define OWL_DMA_IRQ_PD3 0x0C | |
36 | #define OWL_DMA_IRQ_EN0 0x10 | |
37 | #define OWL_DMA_IRQ_EN1 0x14 | |
38 | #define OWL_DMA_IRQ_EN2 0x18 | |
39 | #define OWL_DMA_IRQ_EN3 0x1C | |
40 | #define OWL_DMA_SECURE_ACCESS_CTL 0x20 | |
41 | #define OWL_DMA_NIC_QOS 0x24 | |
42 | #define OWL_DMA_DBGSEL 0x28 | |
43 | #define OWL_DMA_IDLE_STAT 0x2C | |
44 | ||
45 | /* Channel Registers */ | |
46 | #define OWL_DMA_CHAN_BASE(i) (0x100 + (i) * 0x100) | |
47 | #define OWL_DMAX_MODE 0x00 | |
48 | #define OWL_DMAX_SOURCE 0x04 | |
49 | #define OWL_DMAX_DESTINATION 0x08 | |
50 | #define OWL_DMAX_FRAME_LEN 0x0C | |
51 | #define OWL_DMAX_FRAME_CNT 0x10 | |
52 | #define OWL_DMAX_REMAIN_FRAME_CNT 0x14 | |
53 | #define OWL_DMAX_REMAIN_CNT 0x18 | |
54 | #define OWL_DMAX_SOURCE_STRIDE 0x1C | |
55 | #define OWL_DMAX_DESTINATION_STRIDE 0x20 | |
56 | #define OWL_DMAX_START 0x24 | |
57 | #define OWL_DMAX_PAUSE 0x28 | |
58 | #define OWL_DMAX_CHAINED_CTL 0x2C | |
59 | #define OWL_DMAX_CONSTANT 0x30 | |
60 | #define OWL_DMAX_LINKLIST_CTL 0x34 | |
61 | #define OWL_DMAX_NEXT_DESCRIPTOR 0x38 | |
62 | #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM 0x3C | |
63 | #define OWL_DMAX_INT_CTL 0x40 | |
64 | #define OWL_DMAX_INT_STATUS 0x44 | |
65 | #define OWL_DMAX_CURRENT_SOURCE_POINTER 0x48 | |
66 | #define OWL_DMAX_CURRENT_DESTINATION_POINTER 0x4C | |
67 | ||
68 | /* OWL_DMAX_MODE Bits */ | |
69 | #define OWL_DMA_MODE_TS(x) (((x) & GENMASK(5, 0)) << 0) | |
70 | #define OWL_DMA_MODE_ST(x) (((x) & GENMASK(1, 0)) << 8) | |
71 | #define OWL_DMA_MODE_ST_DEV OWL_DMA_MODE_ST(0) | |
72 | #define OWL_DMA_MODE_ST_DCU OWL_DMA_MODE_ST(2) | |
73 | #define OWL_DMA_MODE_ST_SRAM OWL_DMA_MODE_ST(3) | |
74 | #define OWL_DMA_MODE_DT(x) (((x) & GENMASK(1, 0)) << 10) | |
75 | #define OWL_DMA_MODE_DT_DEV OWL_DMA_MODE_DT(0) | |
76 | #define OWL_DMA_MODE_DT_DCU OWL_DMA_MODE_DT(2) | |
77 | #define OWL_DMA_MODE_DT_SRAM OWL_DMA_MODE_DT(3) | |
78 | #define OWL_DMA_MODE_SAM(x) (((x) & GENMASK(1, 0)) << 16) | |
79 | #define OWL_DMA_MODE_SAM_CONST OWL_DMA_MODE_SAM(0) | |
80 | #define OWL_DMA_MODE_SAM_INC OWL_DMA_MODE_SAM(1) | |
81 | #define OWL_DMA_MODE_SAM_STRIDE OWL_DMA_MODE_SAM(2) | |
82 | #define OWL_DMA_MODE_DAM(x) (((x) & GENMASK(1, 0)) << 18) | |
83 | #define OWL_DMA_MODE_DAM_CONST OWL_DMA_MODE_DAM(0) | |
84 | #define OWL_DMA_MODE_DAM_INC OWL_DMA_MODE_DAM(1) | |
85 | #define OWL_DMA_MODE_DAM_STRIDE OWL_DMA_MODE_DAM(2) | |
86 | #define OWL_DMA_MODE_PW(x) (((x) & GENMASK(2, 0)) << 20) | |
87 | #define OWL_DMA_MODE_CB BIT(23) | |
88 | #define OWL_DMA_MODE_NDDBW(x) (((x) & 0x1) << 28) | |
89 | #define OWL_DMA_MODE_NDDBW_32BIT OWL_DMA_MODE_NDDBW(0) | |
90 | #define OWL_DMA_MODE_NDDBW_8BIT OWL_DMA_MODE_NDDBW(1) | |
91 | #define OWL_DMA_MODE_CFE BIT(29) | |
92 | #define OWL_DMA_MODE_LME BIT(30) | |
93 | #define OWL_DMA_MODE_CME BIT(31) | |
94 | ||
95 | /* OWL_DMAX_LINKLIST_CTL Bits */ | |
96 | #define OWL_DMA_LLC_SAV(x) (((x) & GENMASK(1, 0)) << 8) | |
97 | #define OWL_DMA_LLC_SAV_INC OWL_DMA_LLC_SAV(0) | |
98 | #define OWL_DMA_LLC_SAV_LOAD_NEXT OWL_DMA_LLC_SAV(1) | |
99 | #define OWL_DMA_LLC_SAV_LOAD_PREV OWL_DMA_LLC_SAV(2) | |
100 | #define OWL_DMA_LLC_DAV(x) (((x) & GENMASK(1, 0)) << 10) | |
101 | #define OWL_DMA_LLC_DAV_INC OWL_DMA_LLC_DAV(0) | |
102 | #define OWL_DMA_LLC_DAV_LOAD_NEXT OWL_DMA_LLC_DAV(1) | |
103 | #define OWL_DMA_LLC_DAV_LOAD_PREV OWL_DMA_LLC_DAV(2) | |
104 | #define OWL_DMA_LLC_SUSPEND BIT(16) | |
105 | ||
106 | /* OWL_DMAX_INT_CTL Bits */ | |
107 | #define OWL_DMA_INTCTL_BLOCK BIT(0) | |
108 | #define OWL_DMA_INTCTL_SUPER_BLOCK BIT(1) | |
109 | #define OWL_DMA_INTCTL_FRAME BIT(2) | |
110 | #define OWL_DMA_INTCTL_HALF_FRAME BIT(3) | |
111 | #define OWL_DMA_INTCTL_LAST_FRAME BIT(4) | |
112 | ||
113 | /* OWL_DMAX_INT_STATUS Bits */ | |
114 | #define OWL_DMA_INTSTAT_BLOCK BIT(0) | |
115 | #define OWL_DMA_INTSTAT_SUPER_BLOCK BIT(1) | |
116 | #define OWL_DMA_INTSTAT_FRAME BIT(2) | |
117 | #define OWL_DMA_INTSTAT_HALF_FRAME BIT(3) | |
118 | #define OWL_DMA_INTSTAT_LAST_FRAME BIT(4) | |
119 | ||
120 | /* Pack shift and newshift in a single word */ | |
121 | #define BIT_FIELD(val, width, shift, newshift) \ | |
122 | ((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift)) | |
123 | ||
57937fae AST |
124 | /* Frame count value is fixed as 1 */ |
125 | #define FCNT_VAL 0x1 | |
126 | ||
47e20577 | 127 | /** |
fc143e38 | 128 | * enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link |
57937fae AST |
129 | * list for dma transfer |
130 | * @OWL_DMADESC_NEXT_LLI: physical address of the next link list | |
131 | * @OWL_DMADESC_SADDR: source physical address | |
132 | * @OWL_DMADESC_DADDR: destination physical address | |
133 | * @OWL_DMADESC_FLEN: frame length | |
134 | * @OWL_DMADESC_SRC_STRIDE: source stride | |
135 | * @OWL_DMADESC_DST_STRIDE: destination stride | |
136 | * @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config | |
137 | * @OWL_DMADESC_CTRLB: interrupt config | |
138 | * @OWL_DMADESC_CONST_NUM: data for constant fill | |
fc143e38 | 139 | * @OWL_DMADESC_SIZE: max size of this enum |
47e20577 | 140 | */ |
57937fae AST |
141 | enum owl_dmadesc_offsets { |
142 | OWL_DMADESC_NEXT_LLI = 0, | |
143 | OWL_DMADESC_SADDR, | |
144 | OWL_DMADESC_DADDR, | |
145 | OWL_DMADESC_FLEN, | |
146 | OWL_DMADESC_SRC_STRIDE, | |
147 | OWL_DMADESC_DST_STRIDE, | |
148 | OWL_DMADESC_CTRLA, | |
149 | OWL_DMADESC_CTRLB, | |
150 | OWL_DMADESC_CONST_NUM, | |
151 | OWL_DMADESC_SIZE | |
47e20577 MS |
152 | }; |
153 | ||
6f9e40d4 AST |
154 | enum owl_dma_id { |
155 | S900_DMA, | |
156 | S700_DMA, | |
157 | }; | |
158 | ||
47e20577 MS |
159 | /** |
160 | * struct owl_dma_lli - Link list for dma transfer | |
161 | * @hw: hardware link list | |
162 | * @phys: physical address of hardware link list | |
163 | * @node: node for txd's lli_list | |
164 | */ | |
165 | struct owl_dma_lli { | |
57937fae | 166 | u32 hw[OWL_DMADESC_SIZE]; |
47e20577 MS |
167 | dma_addr_t phys; |
168 | struct list_head node; | |
169 | }; | |
170 | ||
171 | /** | |
172 | * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor | |
173 | * @vd: virtual DMA descriptor | |
174 | * @lli_list: link list of lli nodes | |
a3e40316 | 175 | * @cyclic: flag to indicate cyclic transfers |
47e20577 MS |
176 | */ |
177 | struct owl_dma_txd { | |
178 | struct virt_dma_desc vd; | |
179 | struct list_head lli_list; | |
d64e1b3f | 180 | bool cyclic; |
47e20577 MS |
181 | }; |
182 | ||
183 | /** | |
184 | * struct owl_dma_pchan - Holder for the physical channels | |
185 | * @id: physical index to this channel | |
186 | * @base: virtual memory base for the dma channel | |
187 | * @vchan: the virtual channel currently being served by this physical channel | |
47e20577 MS |
188 | */ |
189 | struct owl_dma_pchan { | |
190 | u32 id; | |
191 | void __iomem *base; | |
192 | struct owl_dma_vchan *vchan; | |
47e20577 MS |
193 | }; |
194 | ||
195 | /** | |
74d7221c | 196 | * struct owl_dma_vchan - Wrapper for DMA ENGINE channel |
2749416f | 197 | * @vc: wrapped virtual channel |
47e20577 MS |
198 | * @pchan: the physical channel utilized by this channel |
199 | * @txd: active transaction on this channel | |
a3e40316 MS |
200 | * @cfg: slave configuration for this channel |
201 | * @drq: physical DMA request ID for this channel | |
47e20577 MS |
202 | */ |
203 | struct owl_dma_vchan { | |
204 | struct virt_dma_chan vc; | |
205 | struct owl_dma_pchan *pchan; | |
206 | struct owl_dma_txd *txd; | |
d64e1b3f MS |
207 | struct dma_slave_config cfg; |
208 | u8 drq; | |
47e20577 MS |
209 | }; |
210 | ||
211 | /** | |
212 | * struct owl_dma - Holder for the Owl DMA controller | |
213 | * @dma: dma engine for this instance | |
214 | * @base: virtual memory base for the DMA controller | |
215 | * @clk: clock for the DMA controller | |
216 | * @lock: a lock to use when change DMA controller global register | |
217 | * @lli_pool: a pool for the LLI descriptors | |
a3e40316 | 218 | * @irq: interrupt ID for the DMA controller |
47e20577 MS |
219 | * @nr_pchans: the number of physical channels |
220 | * @pchans: array of data for the physical channels | |
221 | * @nr_vchans: the number of physical channels | |
222 | * @vchans: array of data for the physical channels | |
6f9e40d4 | 223 | * @devid: device id based on OWL SoC |
47e20577 MS |
224 | */ |
225 | struct owl_dma { | |
226 | struct dma_device dma; | |
227 | void __iomem *base; | |
228 | struct clk *clk; | |
229 | spinlock_t lock; | |
230 | struct dma_pool *lli_pool; | |
231 | int irq; | |
232 | ||
233 | unsigned int nr_pchans; | |
234 | struct owl_dma_pchan *pchans; | |
235 | ||
236 | unsigned int nr_vchans; | |
237 | struct owl_dma_vchan *vchans; | |
6f9e40d4 | 238 | enum owl_dma_id devid; |
47e20577 MS |
239 | }; |
240 | ||
241 | static void pchan_update(struct owl_dma_pchan *pchan, u32 reg, | |
242 | u32 val, bool state) | |
243 | { | |
244 | u32 regval; | |
245 | ||
246 | regval = readl(pchan->base + reg); | |
247 | ||
248 | if (state) | |
249 | regval |= val; | |
250 | else | |
251 | regval &= ~val; | |
252 | ||
253 | writel(val, pchan->base + reg); | |
254 | } | |
255 | ||
256 | static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data) | |
257 | { | |
258 | writel(data, pchan->base + reg); | |
259 | } | |
260 | ||
261 | static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg) | |
262 | { | |
263 | return readl(pchan->base + reg); | |
264 | } | |
265 | ||
266 | static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state) | |
267 | { | |
268 | u32 regval; | |
269 | ||
270 | regval = readl(od->base + reg); | |
271 | ||
272 | if (state) | |
273 | regval |= val; | |
274 | else | |
275 | regval &= ~val; | |
276 | ||
277 | writel(val, od->base + reg); | |
278 | } | |
279 | ||
280 | static void dma_writel(struct owl_dma *od, u32 reg, u32 data) | |
281 | { | |
282 | writel(data, od->base + reg); | |
283 | } | |
284 | ||
285 | static u32 dma_readl(struct owl_dma *od, u32 reg) | |
286 | { | |
287 | return readl(od->base + reg); | |
288 | } | |
289 | ||
290 | static inline struct owl_dma *to_owl_dma(struct dma_device *dd) | |
291 | { | |
292 | return container_of(dd, struct owl_dma, dma); | |
293 | } | |
294 | ||
295 | static struct device *chan2dev(struct dma_chan *chan) | |
296 | { | |
297 | return &chan->dev->device; | |
298 | } | |
299 | ||
300 | static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan) | |
301 | { | |
302 | return container_of(chan, struct owl_dma_vchan, vc.chan); | |
303 | } | |
304 | ||
305 | static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx) | |
306 | { | |
307 | return container_of(tx, struct owl_dma_txd, vd.tx); | |
308 | } | |
309 | ||
310 | static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl) | |
311 | { | |
312 | u32 ctl; | |
313 | ||
314 | ctl = BIT_FIELD(mode, 4, 28, 28) | | |
315 | BIT_FIELD(mode, 8, 16, 20) | | |
316 | BIT_FIELD(mode, 4, 8, 16) | | |
317 | BIT_FIELD(mode, 6, 0, 10) | | |
318 | BIT_FIELD(llc_ctl, 2, 10, 8) | | |
319 | BIT_FIELD(llc_ctl, 2, 8, 6); | |
320 | ||
321 | return ctl; | |
322 | } | |
323 | ||
324 | static inline u32 llc_hw_ctrlb(u32 int_ctl) | |
325 | { | |
326 | u32 ctl; | |
327 | ||
6f9e40d4 AST |
328 | /* |
329 | * Irrespective of the SoC, ctrlb value starts filling from | |
330 | * bit 18. | |
331 | */ | |
47e20577 MS |
332 | ctl = BIT_FIELD(int_ctl, 7, 0, 18); |
333 | ||
334 | return ctl; | |
335 | } | |
336 | ||
57937fae AST |
337 | static u32 llc_hw_flen(struct owl_dma_lli *lli) |
338 | { | |
339 | return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0); | |
340 | } | |
341 | ||
47e20577 MS |
342 | static void owl_dma_free_lli(struct owl_dma *od, |
343 | struct owl_dma_lli *lli) | |
344 | { | |
345 | list_del(&lli->node); | |
346 | dma_pool_free(od->lli_pool, lli, lli->phys); | |
347 | } | |
348 | ||
349 | static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od) | |
350 | { | |
351 | struct owl_dma_lli *lli; | |
352 | dma_addr_t phys; | |
353 | ||
354 | lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys); | |
355 | if (!lli) | |
356 | return NULL; | |
357 | ||
358 | INIT_LIST_HEAD(&lli->node); | |
359 | lli->phys = phys; | |
360 | ||
361 | return lli; | |
362 | } | |
363 | ||
364 | static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd, | |
365 | struct owl_dma_lli *prev, | |
d64e1b3f MS |
366 | struct owl_dma_lli *next, |
367 | bool is_cyclic) | |
47e20577 | 368 | { |
d64e1b3f MS |
369 | if (!is_cyclic) |
370 | list_add_tail(&next->node, &txd->lli_list); | |
47e20577 MS |
371 | |
372 | if (prev) { | |
57937fae AST |
373 | prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys; |
374 | prev->hw[OWL_DMADESC_CTRLA] |= | |
375 | llc_hw_ctrla(OWL_DMA_MODE_LME, 0); | |
47e20577 MS |
376 | } |
377 | ||
378 | return next; | |
379 | } | |
380 | ||
381 | static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan, | |
382 | struct owl_dma_lli *lli, | |
383 | dma_addr_t src, dma_addr_t dst, | |
d64e1b3f MS |
384 | u32 len, enum dma_transfer_direction dir, |
385 | struct dma_slave_config *sconfig, | |
386 | bool is_cyclic) | |
47e20577 | 387 | { |
6f9e40d4 | 388 | struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); |
57937fae | 389 | u32 mode, ctrlb; |
47e20577 MS |
390 | |
391 | mode = OWL_DMA_MODE_PW(0); | |
392 | ||
393 | switch (dir) { | |
394 | case DMA_MEM_TO_MEM: | |
395 | mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU | | |
396 | OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC | | |
397 | OWL_DMA_MODE_DAM_INC; | |
398 | ||
d64e1b3f MS |
399 | break; |
400 | case DMA_MEM_TO_DEV: | |
401 | mode |= OWL_DMA_MODE_TS(vchan->drq) | |
402 | | OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV | |
403 | | OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST; | |
404 | ||
405 | /* | |
406 | * Hardware only supports 32bit and 8bit buswidth. Since the | |
407 | * default is 32bit, select 8bit only when requested. | |
408 | */ | |
409 | if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE) | |
410 | mode |= OWL_DMA_MODE_NDDBW_8BIT; | |
411 | ||
412 | break; | |
413 | case DMA_DEV_TO_MEM: | |
414 | mode |= OWL_DMA_MODE_TS(vchan->drq) | |
415 | | OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU | |
416 | | OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC; | |
417 | ||
418 | /* | |
419 | * Hardware only supports 32bit and 8bit buswidth. Since the | |
420 | * default is 32bit, select 8bit only when requested. | |
421 | */ | |
422 | if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE) | |
423 | mode |= OWL_DMA_MODE_NDDBW_8BIT; | |
424 | ||
47e20577 MS |
425 | break; |
426 | default: | |
427 | return -EINVAL; | |
428 | } | |
429 | ||
57937fae AST |
430 | lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode, |
431 | OWL_DMA_LLC_SAV_LOAD_NEXT | | |
432 | OWL_DMA_LLC_DAV_LOAD_NEXT); | |
47e20577 | 433 | |
d64e1b3f | 434 | if (is_cyclic) |
57937fae | 435 | ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK); |
d64e1b3f | 436 | else |
57937fae AST |
437 | ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK); |
438 | ||
439 | lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */ | |
440 | lli->hw[OWL_DMADESC_SADDR] = src; | |
441 | lli->hw[OWL_DMADESC_DADDR] = dst; | |
442 | lli->hw[OWL_DMADESC_SRC_STRIDE] = 0; | |
443 | lli->hw[OWL_DMADESC_DST_STRIDE] = 0; | |
6f9e40d4 AST |
444 | |
445 | if (od->devid == S700_DMA) { | |
446 | /* Max frame length is 1MB */ | |
447 | lli->hw[OWL_DMADESC_FLEN] = len; | |
448 | /* | |
449 | * On S700, word starts from offset 0x1C is shared between | |
450 | * frame count and ctrlb, where first 12 bits are for frame | |
451 | * count and rest of 20 bits are for ctrlb. | |
452 | */ | |
453 | lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb; | |
454 | } else { | |
455 | /* | |
456 | * On S900, word starts from offset 0xC is shared between | |
457 | * frame length (max frame length is 1MB) and frame count, | |
458 | * where first 20 bits are for frame length and rest of | |
459 | * 12 bits are for frame count. | |
460 | */ | |
461 | lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20; | |
462 | lli->hw[OWL_DMADESC_CTRLB] = ctrlb; | |
463 | } | |
47e20577 MS |
464 | |
465 | return 0; | |
466 | } | |
467 | ||
468 | static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od, | |
469 | struct owl_dma_vchan *vchan) | |
470 | { | |
471 | struct owl_dma_pchan *pchan = NULL; | |
472 | unsigned long flags; | |
473 | int i; | |
474 | ||
475 | for (i = 0; i < od->nr_pchans; i++) { | |
476 | pchan = &od->pchans[i]; | |
477 | ||
f8f482de | 478 | spin_lock_irqsave(&od->lock, flags); |
47e20577 MS |
479 | if (!pchan->vchan) { |
480 | pchan->vchan = vchan; | |
f8f482de | 481 | spin_unlock_irqrestore(&od->lock, flags); |
47e20577 MS |
482 | break; |
483 | } | |
484 | ||
f8f482de | 485 | spin_unlock_irqrestore(&od->lock, flags); |
47e20577 MS |
486 | } |
487 | ||
488 | return pchan; | |
489 | } | |
490 | ||
491 | static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan) | |
492 | { | |
493 | unsigned int val; | |
494 | ||
495 | val = dma_readl(od, OWL_DMA_IDLE_STAT); | |
496 | ||
497 | return !(val & (1 << pchan->id)); | |
498 | } | |
499 | ||
500 | static void owl_dma_terminate_pchan(struct owl_dma *od, | |
501 | struct owl_dma_pchan *pchan) | |
502 | { | |
503 | unsigned long flags; | |
504 | u32 irq_pd; | |
505 | ||
506 | pchan_writel(pchan, OWL_DMAX_START, 0); | |
507 | pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); | |
508 | ||
509 | spin_lock_irqsave(&od->lock, flags); | |
510 | dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false); | |
511 | ||
512 | irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0); | |
513 | if (irq_pd & (1 << pchan->id)) { | |
514 | dev_warn(od->dma.dev, | |
515 | "terminating pchan %d that still has pending irq\n", | |
516 | pchan->id); | |
517 | dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id)); | |
518 | } | |
519 | ||
520 | pchan->vchan = NULL; | |
521 | ||
522 | spin_unlock_irqrestore(&od->lock, flags); | |
523 | } | |
524 | ||
d64e1b3f MS |
525 | static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan) |
526 | { | |
527 | pchan_writel(pchan, 1, OWL_DMAX_PAUSE); | |
528 | } | |
529 | ||
530 | static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan) | |
531 | { | |
532 | pchan_writel(pchan, 0, OWL_DMAX_PAUSE); | |
533 | } | |
534 | ||
47e20577 MS |
535 | static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan) |
536 | { | |
537 | struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); | |
538 | struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc); | |
539 | struct owl_dma_pchan *pchan = vchan->pchan; | |
540 | struct owl_dma_txd *txd = to_owl_txd(&vd->tx); | |
541 | struct owl_dma_lli *lli; | |
542 | unsigned long flags; | |
543 | u32 int_ctl; | |
544 | ||
545 | list_del(&vd->node); | |
546 | ||
547 | vchan->txd = txd; | |
548 | ||
549 | /* Wait for channel inactive */ | |
550 | while (owl_dma_pchan_busy(od, pchan)) | |
551 | cpu_relax(); | |
552 | ||
553 | lli = list_first_entry(&txd->lli_list, | |
554 | struct owl_dma_lli, node); | |
555 | ||
d64e1b3f MS |
556 | if (txd->cyclic) |
557 | int_ctl = OWL_DMA_INTCTL_BLOCK; | |
558 | else | |
559 | int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK; | |
47e20577 MS |
560 | |
561 | pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME); | |
562 | pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL, | |
563 | OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT); | |
564 | pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys); | |
565 | pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl); | |
566 | ||
567 | /* Clear IRQ status for this pchan */ | |
568 | pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); | |
569 | ||
570 | spin_lock_irqsave(&od->lock, flags); | |
571 | ||
572 | dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true); | |
573 | ||
574 | spin_unlock_irqrestore(&od->lock, flags); | |
575 | ||
576 | dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id); | |
577 | ||
578 | /* Start DMA transfer for this pchan */ | |
579 | pchan_writel(pchan, OWL_DMAX_START, 0x1); | |
580 | ||
581 | return 0; | |
582 | } | |
583 | ||
584 | static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan) | |
585 | { | |
586 | /* Ensure that the physical channel is stopped */ | |
587 | owl_dma_terminate_pchan(od, vchan->pchan); | |
588 | ||
589 | vchan->pchan = NULL; | |
590 | } | |
591 | ||
592 | static irqreturn_t owl_dma_interrupt(int irq, void *dev_id) | |
593 | { | |
594 | struct owl_dma *od = dev_id; | |
595 | struct owl_dma_vchan *vchan; | |
596 | struct owl_dma_pchan *pchan; | |
597 | unsigned long pending; | |
598 | int i; | |
599 | unsigned int global_irq_pending, chan_irq_pending; | |
600 | ||
601 | spin_lock(&od->lock); | |
602 | ||
603 | pending = dma_readl(od, OWL_DMA_IRQ_PD0); | |
604 | ||
605 | /* Clear IRQ status for each pchan */ | |
606 | for_each_set_bit(i, &pending, od->nr_pchans) { | |
607 | pchan = &od->pchans[i]; | |
608 | pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false); | |
609 | } | |
610 | ||
611 | /* Clear pending IRQ */ | |
612 | dma_writel(od, OWL_DMA_IRQ_PD0, pending); | |
613 | ||
614 | /* Check missed pending IRQ */ | |
615 | for (i = 0; i < od->nr_pchans; i++) { | |
616 | pchan = &od->pchans[i]; | |
617 | chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) & | |
618 | pchan_readl(pchan, OWL_DMAX_INT_STATUS); | |
619 | ||
620 | /* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */ | |
621 | dma_readl(od, OWL_DMA_IRQ_PD0); | |
622 | ||
623 | global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0); | |
624 | ||
6f9e40d4 | 625 | if (chan_irq_pending && !(global_irq_pending & BIT(i))) { |
47e20577 MS |
626 | dev_dbg(od->dma.dev, |
627 | "global and channel IRQ pending match err\n"); | |
628 | ||
629 | /* Clear IRQ status for this pchan */ | |
630 | pchan_update(pchan, OWL_DMAX_INT_STATUS, | |
631 | 0xff, false); | |
632 | ||
633 | /* Update global IRQ pending */ | |
634 | pending |= BIT(i); | |
635 | } | |
636 | } | |
637 | ||
638 | spin_unlock(&od->lock); | |
639 | ||
640 | for_each_set_bit(i, &pending, od->nr_pchans) { | |
641 | struct owl_dma_txd *txd; | |
642 | ||
643 | pchan = &od->pchans[i]; | |
644 | ||
645 | vchan = pchan->vchan; | |
646 | if (!vchan) { | |
647 | dev_warn(od->dma.dev, "no vchan attached on pchan %d\n", | |
648 | pchan->id); | |
649 | continue; | |
650 | } | |
651 | ||
652 | spin_lock(&vchan->vc.lock); | |
653 | ||
654 | txd = vchan->txd; | |
655 | if (txd) { | |
656 | vchan->txd = NULL; | |
657 | ||
658 | vchan_cookie_complete(&txd->vd); | |
659 | ||
660 | /* | |
661 | * Start the next descriptor (if any), | |
662 | * otherwise free this channel. | |
663 | */ | |
664 | if (vchan_next_desc(&vchan->vc)) | |
665 | owl_dma_start_next_txd(vchan); | |
666 | else | |
667 | owl_dma_phy_free(od, vchan); | |
668 | } | |
669 | ||
670 | spin_unlock(&vchan->vc.lock); | |
671 | } | |
672 | ||
673 | return IRQ_HANDLED; | |
674 | } | |
675 | ||
676 | static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd) | |
677 | { | |
678 | struct owl_dma_lli *lli, *_lli; | |
679 | ||
680 | if (unlikely(!txd)) | |
681 | return; | |
682 | ||
683 | list_for_each_entry_safe(lli, _lli, &txd->lli_list, node) | |
684 | owl_dma_free_lli(od, lli); | |
685 | ||
686 | kfree(txd); | |
687 | } | |
688 | ||
689 | static void owl_dma_desc_free(struct virt_dma_desc *vd) | |
690 | { | |
691 | struct owl_dma *od = to_owl_dma(vd->tx.chan->device); | |
692 | struct owl_dma_txd *txd = to_owl_txd(&vd->tx); | |
693 | ||
694 | owl_dma_free_txd(od, txd); | |
695 | } | |
696 | ||
697 | static int owl_dma_terminate_all(struct dma_chan *chan) | |
698 | { | |
699 | struct owl_dma *od = to_owl_dma(chan->device); | |
700 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
701 | unsigned long flags; | |
702 | LIST_HEAD(head); | |
703 | ||
704 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
705 | ||
706 | if (vchan->pchan) | |
707 | owl_dma_phy_free(od, vchan); | |
708 | ||
709 | if (vchan->txd) { | |
710 | owl_dma_desc_free(&vchan->txd->vd); | |
711 | vchan->txd = NULL; | |
712 | } | |
713 | ||
714 | vchan_get_all_descriptors(&vchan->vc, &head); | |
47e20577 MS |
715 | |
716 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
717 | ||
51fe9cd2 SH |
718 | vchan_dma_desc_free_list(&vchan->vc, &head); |
719 | ||
47e20577 MS |
720 | return 0; |
721 | } | |
722 | ||
d64e1b3f MS |
723 | static int owl_dma_config(struct dma_chan *chan, |
724 | struct dma_slave_config *config) | |
725 | { | |
726 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
727 | ||
728 | /* Reject definitely invalid configurations */ | |
729 | if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || | |
730 | config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) | |
731 | return -EINVAL; | |
732 | ||
733 | memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config)); | |
734 | ||
735 | return 0; | |
736 | } | |
737 | ||
738 | static int owl_dma_pause(struct dma_chan *chan) | |
739 | { | |
740 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
741 | unsigned long flags; | |
742 | ||
743 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
744 | ||
745 | owl_dma_pause_pchan(vchan->pchan); | |
746 | ||
747 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
748 | ||
749 | return 0; | |
750 | } | |
751 | ||
752 | static int owl_dma_resume(struct dma_chan *chan) | |
753 | { | |
754 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
755 | unsigned long flags; | |
756 | ||
757 | if (!vchan->pchan && !vchan->txd) | |
758 | return 0; | |
759 | ||
760 | dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc); | |
761 | ||
762 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
763 | ||
764 | owl_dma_resume_pchan(vchan->pchan); | |
765 | ||
766 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
767 | ||
768 | return 0; | |
769 | } | |
770 | ||
47e20577 MS |
771 | static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan) |
772 | { | |
773 | struct owl_dma_pchan *pchan; | |
774 | struct owl_dma_txd *txd; | |
775 | struct owl_dma_lli *lli; | |
776 | unsigned int next_lli_phy; | |
777 | size_t bytes; | |
778 | ||
779 | pchan = vchan->pchan; | |
780 | txd = vchan->txd; | |
781 | ||
782 | if (!pchan || !txd) | |
783 | return 0; | |
784 | ||
785 | /* Get remain count of current node in link list */ | |
786 | bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT); | |
787 | ||
788 | /* Loop through the preceding nodes to get total remaining bytes */ | |
789 | if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) { | |
790 | next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR); | |
791 | list_for_each_entry(lli, &txd->lli_list, node) { | |
792 | /* Start from the next active node */ | |
793 | if (lli->phys == next_lli_phy) { | |
794 | list_for_each_entry(lli, &txd->lli_list, node) | |
57937fae | 795 | bytes += llc_hw_flen(lli); |
47e20577 MS |
796 | break; |
797 | } | |
798 | } | |
799 | } | |
800 | ||
801 | return bytes; | |
802 | } | |
803 | ||
804 | static enum dma_status owl_dma_tx_status(struct dma_chan *chan, | |
805 | dma_cookie_t cookie, | |
806 | struct dma_tx_state *state) | |
807 | { | |
808 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
809 | struct owl_dma_lli *lli; | |
810 | struct virt_dma_desc *vd; | |
811 | struct owl_dma_txd *txd; | |
812 | enum dma_status ret; | |
813 | unsigned long flags; | |
814 | size_t bytes = 0; | |
815 | ||
816 | ret = dma_cookie_status(chan, cookie, state); | |
817 | if (ret == DMA_COMPLETE || !state) | |
818 | return ret; | |
819 | ||
820 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
821 | ||
822 | vd = vchan_find_desc(&vchan->vc, cookie); | |
823 | if (vd) { | |
824 | txd = to_owl_txd(&vd->tx); | |
825 | list_for_each_entry(lli, &txd->lli_list, node) | |
57937fae | 826 | bytes += llc_hw_flen(lli); |
47e20577 MS |
827 | } else { |
828 | bytes = owl_dma_getbytes_chan(vchan); | |
829 | } | |
830 | ||
831 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
832 | ||
833 | dma_set_residue(state, bytes); | |
834 | ||
835 | return ret; | |
836 | } | |
837 | ||
838 | static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan) | |
839 | { | |
840 | struct owl_dma *od = to_owl_dma(vchan->vc.chan.device); | |
841 | struct owl_dma_pchan *pchan; | |
842 | ||
843 | pchan = owl_dma_get_pchan(od, vchan); | |
844 | if (!pchan) | |
845 | return; | |
846 | ||
847 | dev_dbg(od->dma.dev, "allocated pchan %d\n", pchan->id); | |
848 | ||
849 | vchan->pchan = pchan; | |
850 | owl_dma_start_next_txd(vchan); | |
851 | } | |
852 | ||
853 | static void owl_dma_issue_pending(struct dma_chan *chan) | |
854 | { | |
855 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
856 | unsigned long flags; | |
857 | ||
858 | spin_lock_irqsave(&vchan->vc.lock, flags); | |
859 | if (vchan_issue_pending(&vchan->vc)) { | |
860 | if (!vchan->pchan) | |
861 | owl_dma_phy_alloc_and_start(vchan); | |
862 | } | |
863 | spin_unlock_irqrestore(&vchan->vc.lock, flags); | |
864 | } | |
865 | ||
866 | static struct dma_async_tx_descriptor | |
867 | *owl_dma_prep_memcpy(struct dma_chan *chan, | |
868 | dma_addr_t dst, dma_addr_t src, | |
869 | size_t len, unsigned long flags) | |
870 | { | |
871 | struct owl_dma *od = to_owl_dma(chan->device); | |
872 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
873 | struct owl_dma_txd *txd; | |
874 | struct owl_dma_lli *lli, *prev = NULL; | |
875 | size_t offset, bytes; | |
876 | int ret; | |
877 | ||
878 | if (!len) | |
879 | return NULL; | |
880 | ||
881 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); | |
882 | if (!txd) | |
883 | return NULL; | |
884 | ||
885 | INIT_LIST_HEAD(&txd->lli_list); | |
886 | ||
887 | /* Process the transfer as frame by frame */ | |
888 | for (offset = 0; offset < len; offset += bytes) { | |
889 | lli = owl_dma_alloc_lli(od); | |
890 | if (!lli) { | |
891 | dev_warn(chan2dev(chan), "failed to allocate lli\n"); | |
892 | goto err_txd_free; | |
893 | } | |
894 | ||
895 | bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH); | |
896 | ||
897 | ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset, | |
d64e1b3f MS |
898 | bytes, DMA_MEM_TO_MEM, |
899 | &vchan->cfg, txd->cyclic); | |
47e20577 MS |
900 | if (ret) { |
901 | dev_warn(chan2dev(chan), "failed to config lli\n"); | |
902 | goto err_txd_free; | |
903 | } | |
904 | ||
d64e1b3f | 905 | prev = owl_dma_add_lli(txd, prev, lli, false); |
47e20577 MS |
906 | } |
907 | ||
908 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
909 | ||
910 | err_txd_free: | |
911 | owl_dma_free_txd(od, txd); | |
912 | return NULL; | |
913 | } | |
914 | ||
d64e1b3f MS |
915 | static struct dma_async_tx_descriptor |
916 | *owl_dma_prep_slave_sg(struct dma_chan *chan, | |
917 | struct scatterlist *sgl, | |
918 | unsigned int sg_len, | |
919 | enum dma_transfer_direction dir, | |
920 | unsigned long flags, void *context) | |
921 | { | |
922 | struct owl_dma *od = to_owl_dma(chan->device); | |
923 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
924 | struct dma_slave_config *sconfig = &vchan->cfg; | |
925 | struct owl_dma_txd *txd; | |
926 | struct owl_dma_lli *lli, *prev = NULL; | |
927 | struct scatterlist *sg; | |
928 | dma_addr_t addr, src = 0, dst = 0; | |
929 | size_t len; | |
930 | int ret, i; | |
931 | ||
932 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); | |
933 | if (!txd) | |
934 | return NULL; | |
935 | ||
936 | INIT_LIST_HEAD(&txd->lli_list); | |
937 | ||
938 | for_each_sg(sgl, sg, sg_len, i) { | |
939 | addr = sg_dma_address(sg); | |
940 | len = sg_dma_len(sg); | |
941 | ||
942 | if (len > OWL_DMA_FRAME_MAX_LENGTH) { | |
943 | dev_err(od->dma.dev, | |
944 | "frame length exceeds max supported length"); | |
945 | goto err_txd_free; | |
946 | } | |
947 | ||
948 | lli = owl_dma_alloc_lli(od); | |
949 | if (!lli) { | |
950 | dev_err(chan2dev(chan), "failed to allocate lli"); | |
951 | goto err_txd_free; | |
952 | } | |
953 | ||
954 | if (dir == DMA_MEM_TO_DEV) { | |
955 | src = addr; | |
956 | dst = sconfig->dst_addr; | |
957 | } else { | |
958 | src = sconfig->src_addr; | |
959 | dst = addr; | |
960 | } | |
961 | ||
962 | ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig, | |
963 | txd->cyclic); | |
964 | if (ret) { | |
965 | dev_warn(chan2dev(chan), "failed to config lli"); | |
966 | goto err_txd_free; | |
967 | } | |
968 | ||
969 | prev = owl_dma_add_lli(txd, prev, lli, false); | |
970 | } | |
971 | ||
972 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
973 | ||
974 | err_txd_free: | |
975 | owl_dma_free_txd(od, txd); | |
976 | ||
977 | return NULL; | |
978 | } | |
979 | ||
980 | static struct dma_async_tx_descriptor | |
981 | *owl_prep_dma_cyclic(struct dma_chan *chan, | |
982 | dma_addr_t buf_addr, size_t buf_len, | |
983 | size_t period_len, | |
984 | enum dma_transfer_direction dir, | |
985 | unsigned long flags) | |
986 | { | |
987 | struct owl_dma *od = to_owl_dma(chan->device); | |
988 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
989 | struct dma_slave_config *sconfig = &vchan->cfg; | |
990 | struct owl_dma_txd *txd; | |
991 | struct owl_dma_lli *lli, *prev = NULL, *first = NULL; | |
992 | dma_addr_t src = 0, dst = 0; | |
993 | unsigned int periods = buf_len / period_len; | |
994 | int ret, i; | |
995 | ||
996 | txd = kzalloc(sizeof(*txd), GFP_NOWAIT); | |
997 | if (!txd) | |
998 | return NULL; | |
999 | ||
1000 | INIT_LIST_HEAD(&txd->lli_list); | |
1001 | txd->cyclic = true; | |
1002 | ||
1003 | for (i = 0; i < periods; i++) { | |
1004 | lli = owl_dma_alloc_lli(od); | |
1005 | if (!lli) { | |
1006 | dev_warn(chan2dev(chan), "failed to allocate lli"); | |
1007 | goto err_txd_free; | |
1008 | } | |
1009 | ||
1010 | if (dir == DMA_MEM_TO_DEV) { | |
1011 | src = buf_addr + (period_len * i); | |
1012 | dst = sconfig->dst_addr; | |
1013 | } else if (dir == DMA_DEV_TO_MEM) { | |
1014 | src = sconfig->src_addr; | |
1015 | dst = buf_addr + (period_len * i); | |
1016 | } | |
1017 | ||
1018 | ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len, | |
1019 | dir, sconfig, txd->cyclic); | |
1020 | if (ret) { | |
1021 | dev_warn(chan2dev(chan), "failed to config lli"); | |
1022 | goto err_txd_free; | |
1023 | } | |
1024 | ||
1025 | if (!first) | |
1026 | first = lli; | |
1027 | ||
1028 | prev = owl_dma_add_lli(txd, prev, lli, false); | |
1029 | } | |
1030 | ||
1031 | /* close the cyclic list */ | |
1032 | owl_dma_add_lli(txd, prev, first, true); | |
1033 | ||
1034 | return vchan_tx_prep(&vchan->vc, &txd->vd, flags); | |
1035 | ||
1036 | err_txd_free: | |
1037 | owl_dma_free_txd(od, txd); | |
1038 | ||
1039 | return NULL; | |
1040 | } | |
1041 | ||
47e20577 MS |
1042 | static void owl_dma_free_chan_resources(struct dma_chan *chan) |
1043 | { | |
1044 | struct owl_dma_vchan *vchan = to_owl_vchan(chan); | |
1045 | ||
1046 | /* Ensure all queued descriptors are freed */ | |
1047 | vchan_free_chan_resources(&vchan->vc); | |
1048 | } | |
1049 | ||
1050 | static inline void owl_dma_free(struct owl_dma *od) | |
1051 | { | |
1052 | struct owl_dma_vchan *vchan = NULL; | |
1053 | struct owl_dma_vchan *next; | |
1054 | ||
1055 | list_for_each_entry_safe(vchan, | |
1056 | next, &od->dma.channels, vc.chan.device_node) { | |
1057 | list_del(&vchan->vc.chan.device_node); | |
1058 | tasklet_kill(&vchan->vc.task); | |
1059 | } | |
1060 | } | |
1061 | ||
d64e1b3f MS |
1062 | static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec, |
1063 | struct of_dma *ofdma) | |
1064 | { | |
1065 | struct owl_dma *od = ofdma->of_dma_data; | |
1066 | struct owl_dma_vchan *vchan; | |
1067 | struct dma_chan *chan; | |
1068 | u8 drq = dma_spec->args[0]; | |
1069 | ||
1070 | if (drq > od->nr_vchans) | |
1071 | return NULL; | |
1072 | ||
1073 | chan = dma_get_any_slave_channel(&od->dma); | |
1074 | if (!chan) | |
1075 | return NULL; | |
1076 | ||
1077 | vchan = to_owl_vchan(chan); | |
1078 | vchan->drq = drq; | |
1079 | ||
1080 | return chan; | |
1081 | } | |
1082 | ||
6f9e40d4 | 1083 | static const struct of_device_id owl_dma_match[] = { |
c518a2fd | 1084 | { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,}, |
6f9e40d4 | 1085 | { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,}, |
c518a2fd | 1086 | { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, |
6f9e40d4 AST |
1087 | { /* sentinel */ }, |
1088 | }; | |
1089 | MODULE_DEVICE_TABLE(of, owl_dma_match); | |
1090 | ||
47e20577 MS |
1091 | static int owl_dma_probe(struct platform_device *pdev) |
1092 | { | |
1093 | struct device_node *np = pdev->dev.of_node; | |
1094 | struct owl_dma *od; | |
47e20577 MS |
1095 | int ret, i, nr_channels, nr_requests; |
1096 | ||
1097 | od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); | |
1098 | if (!od) | |
1099 | return -ENOMEM; | |
1100 | ||
ecb4d34f | 1101 | od->base = devm_platform_ioremap_resource(pdev, 0); |
47e20577 MS |
1102 | if (IS_ERR(od->base)) |
1103 | return PTR_ERR(od->base); | |
1104 | ||
1105 | ret = of_property_read_u32(np, "dma-channels", &nr_channels); | |
1106 | if (ret) { | |
1107 | dev_err(&pdev->dev, "can't get dma-channels\n"); | |
1108 | return ret; | |
1109 | } | |
1110 | ||
1111 | ret = of_property_read_u32(np, "dma-requests", &nr_requests); | |
1112 | if (ret) { | |
1113 | dev_err(&pdev->dev, "can't get dma-requests\n"); | |
1114 | return ret; | |
1115 | } | |
1116 | ||
1117 | dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n", | |
1118 | nr_channels, nr_requests); | |
1119 | ||
1fbda5f4 | 1120 | od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev); |
6f9e40d4 | 1121 | |
47e20577 MS |
1122 | od->nr_pchans = nr_channels; |
1123 | od->nr_vchans = nr_requests; | |
1124 | ||
1125 | pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); | |
1126 | ||
1127 | platform_set_drvdata(pdev, od); | |
1128 | spin_lock_init(&od->lock); | |
1129 | ||
1130 | dma_cap_set(DMA_MEMCPY, od->dma.cap_mask); | |
d64e1b3f MS |
1131 | dma_cap_set(DMA_SLAVE, od->dma.cap_mask); |
1132 | dma_cap_set(DMA_CYCLIC, od->dma.cap_mask); | |
47e20577 MS |
1133 | |
1134 | od->dma.dev = &pdev->dev; | |
1135 | od->dma.device_free_chan_resources = owl_dma_free_chan_resources; | |
1136 | od->dma.device_tx_status = owl_dma_tx_status; | |
1137 | od->dma.device_issue_pending = owl_dma_issue_pending; | |
1138 | od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy; | |
d64e1b3f MS |
1139 | od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg; |
1140 | od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic; | |
1141 | od->dma.device_config = owl_dma_config; | |
1142 | od->dma.device_pause = owl_dma_pause; | |
1143 | od->dma.device_resume = owl_dma_resume; | |
47e20577 MS |
1144 | od->dma.device_terminate_all = owl_dma_terminate_all; |
1145 | od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); | |
1146 | od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); | |
1147 | od->dma.directions = BIT(DMA_MEM_TO_MEM); | |
1148 | od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; | |
1149 | ||
1150 | INIT_LIST_HEAD(&od->dma.channels); | |
1151 | ||
1152 | od->clk = devm_clk_get(&pdev->dev, NULL); | |
1153 | if (IS_ERR(od->clk)) { | |
1154 | dev_err(&pdev->dev, "unable to get clock\n"); | |
1155 | return PTR_ERR(od->clk); | |
1156 | } | |
1157 | ||
1158 | /* | |
1159 | * Eventhough the DMA controller is capable of generating 4 | |
1160 | * IRQ's for DMA priority feature, we only use 1 IRQ for | |
1161 | * simplification. | |
1162 | */ | |
1163 | od->irq = platform_get_irq(pdev, 0); | |
1164 | ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0, | |
1165 | dev_name(&pdev->dev), od); | |
1166 | if (ret) { | |
1167 | dev_err(&pdev->dev, "unable to request IRQ\n"); | |
1168 | return ret; | |
1169 | } | |
1170 | ||
1171 | /* Init physical channel */ | |
1172 | od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans, | |
1173 | sizeof(struct owl_dma_pchan), GFP_KERNEL); | |
1174 | if (!od->pchans) | |
1175 | return -ENOMEM; | |
1176 | ||
1177 | for (i = 0; i < od->nr_pchans; i++) { | |
1178 | struct owl_dma_pchan *pchan = &od->pchans[i]; | |
1179 | ||
1180 | pchan->id = i; | |
1181 | pchan->base = od->base + OWL_DMA_CHAN_BASE(i); | |
1182 | } | |
1183 | ||
1184 | /* Init virtual channel */ | |
1185 | od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans, | |
1186 | sizeof(struct owl_dma_vchan), GFP_KERNEL); | |
1187 | if (!od->vchans) | |
1188 | return -ENOMEM; | |
1189 | ||
1190 | for (i = 0; i < od->nr_vchans; i++) { | |
1191 | struct owl_dma_vchan *vchan = &od->vchans[i]; | |
1192 | ||
1193 | vchan->vc.desc_free = owl_dma_desc_free; | |
1194 | vchan_init(&vchan->vc, &od->dma); | |
1195 | } | |
1196 | ||
1197 | /* Create a pool of consistent memory blocks for hardware descriptors */ | |
1198 | od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev, | |
1199 | sizeof(struct owl_dma_lli), | |
1200 | __alignof__(struct owl_dma_lli), | |
1201 | 0); | |
1202 | if (!od->lli_pool) { | |
1203 | dev_err(&pdev->dev, "unable to allocate DMA descriptor pool\n"); | |
1204 | return -ENOMEM; | |
1205 | } | |
1206 | ||
1207 | clk_prepare_enable(od->clk); | |
1208 | ||
1209 | ret = dma_async_device_register(&od->dma); | |
1210 | if (ret) { | |
1211 | dev_err(&pdev->dev, "failed to register DMA engine device\n"); | |
1212 | goto err_pool_free; | |
1213 | } | |
1214 | ||
d64e1b3f MS |
1215 | /* Device-tree DMA controller registration */ |
1216 | ret = of_dma_controller_register(pdev->dev.of_node, | |
1217 | owl_dma_of_xlate, od); | |
1218 | if (ret) { | |
1219 | dev_err(&pdev->dev, "of_dma_controller_register failed\n"); | |
1220 | goto err_dma_unregister; | |
1221 | } | |
1222 | ||
47e20577 MS |
1223 | return 0; |
1224 | ||
d64e1b3f MS |
1225 | err_dma_unregister: |
1226 | dma_async_device_unregister(&od->dma); | |
47e20577 MS |
1227 | err_pool_free: |
1228 | clk_disable_unprepare(od->clk); | |
1229 | dma_pool_destroy(od->lli_pool); | |
1230 | ||
1231 | return ret; | |
1232 | } | |
1233 | ||
1260486a | 1234 | static void owl_dma_remove(struct platform_device *pdev) |
47e20577 MS |
1235 | { |
1236 | struct owl_dma *od = platform_get_drvdata(pdev); | |
1237 | ||
d64e1b3f | 1238 | of_dma_controller_free(pdev->dev.of_node); |
47e20577 MS |
1239 | dma_async_device_unregister(&od->dma); |
1240 | ||
1241 | /* Mask all interrupts for this execution environment */ | |
1242 | dma_writel(od, OWL_DMA_IRQ_EN0, 0x0); | |
1243 | ||
1244 | /* Make sure we won't have any further interrupts */ | |
1245 | devm_free_irq(od->dma.dev, od->irq, od); | |
1246 | ||
1247 | owl_dma_free(od); | |
1248 | ||
1249 | clk_disable_unprepare(od->clk); | |
1f0a16f0 | 1250 | dma_pool_destroy(od->lli_pool); |
47e20577 MS |
1251 | } |
1252 | ||
47e20577 MS |
1253 | static struct platform_driver owl_dma_driver = { |
1254 | .probe = owl_dma_probe, | |
1260486a | 1255 | .remove_new = owl_dma_remove, |
47e20577 MS |
1256 | .driver = { |
1257 | .name = "dma-owl", | |
1258 | .of_match_table = of_match_ptr(owl_dma_match), | |
1259 | }, | |
1260 | }; | |
1261 | ||
1262 | static int owl_dma_init(void) | |
1263 | { | |
1264 | return platform_driver_register(&owl_dma_driver); | |
1265 | } | |
1266 | subsys_initcall(owl_dma_init); | |
1267 | ||
1268 | static void __exit owl_dma_exit(void) | |
1269 | { | |
1270 | platform_driver_unregister(&owl_dma_driver); | |
1271 | } | |
1272 | module_exit(owl_dma_exit); | |
1273 | ||
1274 | MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>"); | |
1275 | MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>"); | |
1276 | MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver"); | |
1277 | MODULE_LICENSE("GPL"); |