Commit | Line | Data |
---|---|---|
0853c7a5 JC |
1 | /* |
2 | * Copyright (C) 2015, Michael Lee <igvtee@gmail.com> | |
3 | * MTK HSDMA support | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License as published by the | |
7 | * Free Software Foundation; either version 2 of the License, or (at your | |
8 | * option) any later version. | |
9 | * | |
10 | */ | |
11 | ||
12 | #include <linux/dmaengine.h> | |
13 | #include <linux/dma-mapping.h> | |
14 | #include <linux/err.h> | |
15 | #include <linux/init.h> | |
16 | #include <linux/list.h> | |
17 | #include <linux/module.h> | |
18 | #include <linux/platform_device.h> | |
19 | #include <linux/slab.h> | |
20 | #include <linux/spinlock.h> | |
21 | #include <linux/irq.h> | |
22 | #include <linux/of_dma.h> | |
23 | #include <linux/reset.h> | |
24 | #include <linux/of_device.h> | |
25 | ||
26 | #include "virt-dma.h" | |
27 | ||
28 | #define HSDMA_BASE_OFFSET 0x800 | |
29 | ||
30 | #define HSDMA_REG_TX_BASE 0x00 | |
31 | #define HSDMA_REG_TX_CNT 0x04 | |
32 | #define HSDMA_REG_TX_CTX 0x08 | |
33 | #define HSDMA_REG_TX_DTX 0x0c | |
34 | #define HSDMA_REG_RX_BASE 0x100 | |
35 | #define HSDMA_REG_RX_CNT 0x104 | |
36 | #define HSDMA_REG_RX_CRX 0x108 | |
37 | #define HSDMA_REG_RX_DRX 0x10c | |
38 | #define HSDMA_REG_INFO 0x200 | |
39 | #define HSDMA_REG_GLO_CFG 0x204 | |
40 | #define HSDMA_REG_RST_CFG 0x208 | |
41 | #define HSDMA_REG_DELAY_INT 0x20c | |
42 | #define HSDMA_REG_FREEQ_THRES 0x210 | |
43 | #define HSDMA_REG_INT_STATUS 0x220 | |
44 | #define HSDMA_REG_INT_MASK 0x228 | |
45 | #define HSDMA_REG_SCH_Q01 0x280 | |
46 | #define HSDMA_REG_SCH_Q23 0x284 | |
47 | ||
48 | #define HSDMA_DESCS_MAX 0xfff | |
49 | #define HSDMA_DESCS_NUM 8 | |
50 | #define HSDMA_DESCS_MASK (HSDMA_DESCS_NUM - 1) | |
51 | #define HSDMA_NEXT_DESC(x) (((x) + 1) & HSDMA_DESCS_MASK) | |
52 | ||
53 | /* HSDMA_REG_INFO */ | |
54 | #define HSDMA_INFO_INDEX_MASK 0xf | |
55 | #define HSDMA_INFO_INDEX_SHIFT 24 | |
56 | #define HSDMA_INFO_BASE_MASK 0xff | |
57 | #define HSDMA_INFO_BASE_SHIFT 16 | |
58 | #define HSDMA_INFO_RX_MASK 0xff | |
59 | #define HSDMA_INFO_RX_SHIFT 8 | |
60 | #define HSDMA_INFO_TX_MASK 0xff | |
61 | #define HSDMA_INFO_TX_SHIFT 0 | |
62 | ||
63 | /* HSDMA_REG_GLO_CFG */ | |
64 | #define HSDMA_GLO_TX_2B_OFFSET BIT(31) | |
65 | #define HSDMA_GLO_CLK_GATE BIT(30) | |
66 | #define HSDMA_GLO_BYTE_SWAP BIT(29) | |
67 | #define HSDMA_GLO_MULTI_DMA BIT(10) | |
68 | #define HSDMA_GLO_TWO_BUF BIT(9) | |
69 | #define HSDMA_GLO_32B_DESC BIT(8) | |
70 | #define HSDMA_GLO_BIG_ENDIAN BIT(7) | |
71 | #define HSDMA_GLO_TX_DONE BIT(6) | |
72 | #define HSDMA_GLO_BT_MASK 0x3 | |
73 | #define HSDMA_GLO_BT_SHIFT 4 | |
74 | #define HSDMA_GLO_RX_BUSY BIT(3) | |
75 | #define HSDMA_GLO_RX_DMA BIT(2) | |
76 | #define HSDMA_GLO_TX_BUSY BIT(1) | |
77 | #define HSDMA_GLO_TX_DMA BIT(0) | |
78 | ||
79 | #define HSDMA_BT_SIZE_16BYTES (0 << HSDMA_GLO_BT_SHIFT) | |
80 | #define HSDMA_BT_SIZE_32BYTES (1 << HSDMA_GLO_BT_SHIFT) | |
81 | #define HSDMA_BT_SIZE_64BYTES (2 << HSDMA_GLO_BT_SHIFT) | |
82 | #define HSDMA_BT_SIZE_128BYTES (3 << HSDMA_GLO_BT_SHIFT) | |
83 | ||
84 | #define HSDMA_GLO_DEFAULT (HSDMA_GLO_MULTI_DMA | \ | |
85 | HSDMA_GLO_RX_DMA | HSDMA_GLO_TX_DMA | HSDMA_BT_SIZE_32BYTES) | |
86 | ||
87 | /* HSDMA_REG_RST_CFG */ | |
88 | #define HSDMA_RST_RX_SHIFT 16 | |
89 | #define HSDMA_RST_TX_SHIFT 0 | |
90 | ||
91 | /* HSDMA_REG_DELAY_INT */ | |
92 | #define HSDMA_DELAY_INT_EN BIT(15) | |
93 | #define HSDMA_DELAY_PEND_OFFSET 8 | |
94 | #define HSDMA_DELAY_TIME_OFFSET 0 | |
95 | #define HSDMA_DELAY_TX_OFFSET 16 | |
96 | #define HSDMA_DELAY_RX_OFFSET 0 | |
97 | ||
98 | #define HSDMA_DELAY_INIT(x) (HSDMA_DELAY_INT_EN | \ | |
99 | ((x) << HSDMA_DELAY_PEND_OFFSET)) | |
100 | #define HSDMA_DELAY(x) ((HSDMA_DELAY_INIT(x) << \ | |
101 | HSDMA_DELAY_TX_OFFSET) | HSDMA_DELAY_INIT(x)) | |
102 | ||
103 | /* HSDMA_REG_INT_STATUS */ | |
104 | #define HSDMA_INT_DELAY_RX_COH BIT(31) | |
105 | #define HSDMA_INT_DELAY_RX_INT BIT(30) | |
106 | #define HSDMA_INT_DELAY_TX_COH BIT(29) | |
107 | #define HSDMA_INT_DELAY_TX_INT BIT(28) | |
108 | #define HSDMA_INT_RX_MASK 0x3 | |
109 | #define HSDMA_INT_RX_SHIFT 16 | |
110 | #define HSDMA_INT_RX_Q0 BIT(16) | |
111 | #define HSDMA_INT_TX_MASK 0xf | |
112 | #define HSDMA_INT_TX_SHIFT 0 | |
113 | #define HSDMA_INT_TX_Q0 BIT(0) | |
114 | ||
115 | /* tx/rx dma desc flags */ | |
116 | #define HSDMA_PLEN_MASK 0x3fff | |
117 | #define HSDMA_DESC_DONE BIT(31) | |
118 | #define HSDMA_DESC_LS0 BIT(30) | |
119 | #define HSDMA_DESC_PLEN0(_x) (((_x) & HSDMA_PLEN_MASK) << 16) | |
120 | #define HSDMA_DESC_TAG BIT(15) | |
121 | #define HSDMA_DESC_LS1 BIT(14) | |
122 | #define HSDMA_DESC_PLEN1(_x) ((_x) & HSDMA_PLEN_MASK) | |
123 | ||
124 | /* align 4 bytes */ | |
125 | #define HSDMA_ALIGN_SIZE 3 | |
126 | /* align size 128bytes */ | |
127 | #define HSDMA_MAX_PLEN 0x3f80 | |
128 | ||
129 | struct hsdma_desc { | |
130 | u32 addr0; | |
131 | u32 flags; | |
132 | u32 addr1; | |
133 | u32 unused; | |
134 | }; | |
135 | ||
136 | struct mtk_hsdma_sg { | |
137 | dma_addr_t src_addr; | |
138 | dma_addr_t dst_addr; | |
139 | u32 len; | |
140 | }; | |
141 | ||
142 | struct mtk_hsdma_desc { | |
143 | struct virt_dma_desc vdesc; | |
144 | unsigned int num_sgs; | |
145 | struct mtk_hsdma_sg sg[1]; | |
146 | }; | |
147 | ||
148 | struct mtk_hsdma_chan { | |
149 | struct virt_dma_chan vchan; | |
150 | unsigned int id; | |
151 | dma_addr_t desc_addr; | |
152 | int tx_idx; | |
153 | int rx_idx; | |
154 | struct hsdma_desc *tx_ring; | |
155 | struct hsdma_desc *rx_ring; | |
156 | struct mtk_hsdma_desc *desc; | |
157 | unsigned int next_sg; | |
158 | }; | |
159 | ||
160 | struct mtk_hsdam_engine { | |
161 | struct dma_device ddev; | |
162 | struct device_dma_parameters dma_parms; | |
163 | void __iomem *base; | |
164 | struct tasklet_struct task; | |
165 | volatile unsigned long chan_issued; | |
166 | ||
167 | struct mtk_hsdma_chan chan[1]; | |
168 | }; | |
169 | ||
170 | static inline struct mtk_hsdam_engine *mtk_hsdma_chan_get_dev( | |
171 | struct mtk_hsdma_chan *chan) | |
172 | { | |
173 | return container_of(chan->vchan.chan.device, struct mtk_hsdam_engine, | |
174 | ddev); | |
175 | } | |
176 | ||
177 | static inline struct mtk_hsdma_chan *to_mtk_hsdma_chan(struct dma_chan *c) | |
178 | { | |
179 | return container_of(c, struct mtk_hsdma_chan, vchan.chan); | |
180 | } | |
181 | ||
182 | static inline struct mtk_hsdma_desc *to_mtk_hsdma_desc( | |
183 | struct virt_dma_desc *vdesc) | |
184 | { | |
185 | return container_of(vdesc, struct mtk_hsdma_desc, vdesc); | |
186 | } | |
187 | ||
188 | static inline u32 mtk_hsdma_read(struct mtk_hsdam_engine *hsdma, u32 reg) | |
189 | { | |
190 | return readl(hsdma->base + reg); | |
191 | } | |
192 | ||
193 | static inline void mtk_hsdma_write(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 194 | unsigned reg, u32 val) |
0853c7a5 JC |
195 | { |
196 | writel(val, hsdma->base + reg); | |
197 | } | |
198 | ||
199 | static void mtk_hsdma_reset_chan(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 200 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
201 | { |
202 | chan->tx_idx = 0; | |
203 | chan->rx_idx = HSDMA_DESCS_NUM - 1; | |
204 | ||
205 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx); | |
206 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx); | |
207 | ||
208 | mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG, | |
209 | 0x1 << (chan->id + HSDMA_RST_TX_SHIFT)); | |
210 | mtk_hsdma_write(hsdma, HSDMA_REG_RST_CFG, | |
211 | 0x1 << (chan->id + HSDMA_RST_RX_SHIFT)); | |
212 | } | |
213 | ||
214 | static void hsdma_dump_reg(struct mtk_hsdam_engine *hsdma) | |
215 | { | |
216 | dev_dbg(hsdma->ddev.dev, "tbase %08x, tcnt %08x, " \ | |
217 | "tctx %08x, tdtx: %08x, rbase %08x, " \ | |
218 | "rcnt %08x, rctx %08x, rdtx %08x\n", | |
219 | mtk_hsdma_read(hsdma, HSDMA_REG_TX_BASE), | |
220 | mtk_hsdma_read(hsdma, HSDMA_REG_TX_CNT), | |
221 | mtk_hsdma_read(hsdma, HSDMA_REG_TX_CTX), | |
222 | mtk_hsdma_read(hsdma, HSDMA_REG_TX_DTX), | |
223 | mtk_hsdma_read(hsdma, HSDMA_REG_RX_BASE), | |
224 | mtk_hsdma_read(hsdma, HSDMA_REG_RX_CNT), | |
225 | mtk_hsdma_read(hsdma, HSDMA_REG_RX_CRX), | |
226 | mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX)); | |
227 | ||
228 | dev_dbg(hsdma->ddev.dev, "info %08x, glo %08x, delay %08x, " \ | |
229 | "intr_stat %08x, intr_mask %08x\n", | |
230 | mtk_hsdma_read(hsdma, HSDMA_REG_INFO), | |
231 | mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG), | |
232 | mtk_hsdma_read(hsdma, HSDMA_REG_DELAY_INT), | |
233 | mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS), | |
234 | mtk_hsdma_read(hsdma, HSDMA_REG_INT_MASK)); | |
235 | } | |
236 | ||
237 | static void hsdma_dump_desc(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 238 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
239 | { |
240 | struct hsdma_desc *tx_desc; | |
241 | struct hsdma_desc *rx_desc; | |
242 | int i; | |
243 | ||
244 | dev_dbg(hsdma->ddev.dev, "tx idx: %d, rx idx: %d\n", | |
245 | chan->tx_idx, chan->rx_idx); | |
246 | ||
247 | for (i = 0; i < HSDMA_DESCS_NUM; i++) { | |
248 | tx_desc = &chan->tx_ring[i]; | |
249 | rx_desc = &chan->rx_ring[i]; | |
250 | ||
251 | dev_dbg(hsdma->ddev.dev, "%d tx addr0: %08x, flags %08x, " \ | |
252 | "tx addr1: %08x, rx addr0 %08x, flags %08x\n", | |
253 | i, tx_desc->addr0, tx_desc->flags, \ | |
254 | tx_desc->addr1, rx_desc->addr0, rx_desc->flags); | |
255 | } | |
256 | } | |
257 | ||
258 | static void mtk_hsdma_reset(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 259 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
260 | { |
261 | int i; | |
262 | ||
263 | /* disable dma */ | |
264 | mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0); | |
265 | ||
266 | /* disable intr */ | |
267 | mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0); | |
268 | ||
269 | /* init desc value */ | |
270 | for (i = 0; i < HSDMA_DESCS_NUM; i++) { | |
271 | chan->tx_ring[i].addr0 = 0; | |
272 | chan->tx_ring[i].flags = HSDMA_DESC_LS0 | | |
273 | HSDMA_DESC_DONE; | |
274 | } | |
275 | for (i = 0; i < HSDMA_DESCS_NUM; i++) { | |
276 | chan->rx_ring[i].addr0 = 0; | |
277 | chan->rx_ring[i].flags = 0; | |
278 | } | |
279 | ||
280 | /* reset */ | |
281 | mtk_hsdma_reset_chan(hsdma, chan); | |
282 | ||
283 | /* enable intr */ | |
284 | mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0); | |
285 | ||
286 | /* enable dma */ | |
287 | mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT); | |
288 | } | |
289 | ||
290 | static int mtk_hsdma_terminate_all(struct dma_chan *c) | |
291 | { | |
292 | struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c); | |
293 | struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan); | |
294 | unsigned long timeout; | |
295 | LIST_HEAD(head); | |
296 | ||
297 | spin_lock_bh(&chan->vchan.lock); | |
298 | chan->desc = NULL; | |
299 | clear_bit(chan->id, &hsdma->chan_issued); | |
300 | vchan_get_all_descriptors(&chan->vchan, &head); | |
301 | spin_unlock_bh(&chan->vchan.lock); | |
302 | ||
303 | vchan_dma_desc_free_list(&chan->vchan, &head); | |
304 | ||
305 | /* wait dma transfer complete */ | |
306 | timeout = jiffies + msecs_to_jiffies(2000); | |
307 | while (mtk_hsdma_read(hsdma, HSDMA_REG_GLO_CFG) & | |
308 | (HSDMA_GLO_RX_BUSY | HSDMA_GLO_TX_BUSY)) { | |
309 | if (time_after_eq(jiffies, timeout)) { | |
310 | hsdma_dump_desc(hsdma, chan); | |
311 | mtk_hsdma_reset(hsdma, chan); | |
312 | dev_err(hsdma->ddev.dev, "timeout, reset it\n"); | |
313 | break; | |
314 | } | |
315 | cpu_relax(); | |
316 | } | |
317 | ||
318 | return 0; | |
319 | } | |
320 | ||
321 | static int mtk_hsdma_start_transfer(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 322 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
323 | { |
324 | dma_addr_t src, dst; | |
325 | size_t len, tlen; | |
326 | struct hsdma_desc *tx_desc, *rx_desc; | |
327 | struct mtk_hsdma_sg *sg; | |
328 | unsigned int i; | |
329 | int rx_idx; | |
330 | ||
331 | sg = &chan->desc->sg[0]; | |
332 | len = sg->len; | |
333 | chan->desc->num_sgs = DIV_ROUND_UP(len, HSDMA_MAX_PLEN); | |
334 | ||
335 | /* tx desc */ | |
336 | src = sg->src_addr; | |
337 | for (i = 0; i < chan->desc->num_sgs; i++) { | |
338 | if (len > HSDMA_MAX_PLEN) | |
339 | tlen = HSDMA_MAX_PLEN; | |
340 | else | |
341 | tlen = len; | |
342 | ||
343 | if (i & 0x1) { | |
344 | tx_desc->addr1 = src; | |
345 | tx_desc->flags |= HSDMA_DESC_PLEN1(tlen); | |
346 | } else { | |
347 | tx_desc = &chan->tx_ring[chan->tx_idx]; | |
348 | tx_desc->addr0 = src; | |
349 | tx_desc->flags = HSDMA_DESC_PLEN0(tlen); | |
350 | ||
351 | /* update index */ | |
352 | chan->tx_idx = HSDMA_NEXT_DESC(chan->tx_idx); | |
353 | } | |
354 | ||
355 | src += tlen; | |
356 | len -= tlen; | |
357 | } | |
358 | if (i & 0x1) | |
359 | tx_desc->flags |= HSDMA_DESC_LS0; | |
360 | else | |
361 | tx_desc->flags |= HSDMA_DESC_LS1; | |
362 | ||
363 | /* rx desc */ | |
364 | rx_idx = HSDMA_NEXT_DESC(chan->rx_idx); | |
365 | len = sg->len; | |
366 | dst = sg->dst_addr; | |
367 | for (i = 0; i < chan->desc->num_sgs; i++) { | |
368 | rx_desc = &chan->rx_ring[rx_idx]; | |
369 | if (len > HSDMA_MAX_PLEN) | |
370 | tlen = HSDMA_MAX_PLEN; | |
371 | else | |
372 | tlen = len; | |
373 | ||
374 | rx_desc->addr0 = dst; | |
375 | rx_desc->flags = HSDMA_DESC_PLEN0(tlen); | |
376 | ||
377 | dst += tlen; | |
378 | len -= tlen; | |
379 | ||
380 | /* update index */ | |
381 | rx_idx = HSDMA_NEXT_DESC(rx_idx); | |
382 | } | |
383 | ||
384 | /* make sure desc and index all up to date */ | |
385 | wmb(); | |
386 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_CTX, chan->tx_idx); | |
387 | ||
388 | return 0; | |
389 | } | |
390 | ||
391 | static int gdma_next_desc(struct mtk_hsdma_chan *chan) | |
392 | { | |
393 | struct virt_dma_desc *vdesc; | |
394 | ||
395 | vdesc = vchan_next_desc(&chan->vchan); | |
396 | if (!vdesc) { | |
397 | chan->desc = NULL; | |
398 | return 0; | |
399 | } | |
400 | chan->desc = to_mtk_hsdma_desc(vdesc); | |
401 | chan->next_sg = 0; | |
402 | ||
403 | return 1; | |
404 | } | |
405 | ||
406 | static void mtk_hsdma_chan_done(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 407 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
408 | { |
409 | struct mtk_hsdma_desc *desc; | |
410 | int chan_issued; | |
411 | ||
412 | chan_issued = 0; | |
413 | spin_lock_bh(&chan->vchan.lock); | |
414 | desc = chan->desc; | |
415 | if (likely(desc)) { | |
416 | if (chan->next_sg == desc->num_sgs) { | |
417 | list_del(&desc->vdesc.node); | |
418 | vchan_cookie_complete(&desc->vdesc); | |
419 | chan_issued = gdma_next_desc(chan); | |
420 | } | |
421 | } else | |
422 | dev_dbg(hsdma->ddev.dev, "no desc to complete\n"); | |
423 | ||
424 | if (chan_issued) | |
425 | set_bit(chan->id, &hsdma->chan_issued); | |
426 | spin_unlock_bh(&chan->vchan.lock); | |
427 | } | |
428 | ||
429 | static irqreturn_t mtk_hsdma_irq(int irq, void *devid) | |
430 | { | |
431 | struct mtk_hsdam_engine *hsdma = devid; | |
432 | u32 status; | |
433 | ||
434 | status = mtk_hsdma_read(hsdma, HSDMA_REG_INT_STATUS); | |
435 | if (unlikely(!status)) | |
436 | return IRQ_NONE; | |
437 | ||
438 | if (likely(status & HSDMA_INT_RX_Q0)) | |
439 | tasklet_schedule(&hsdma->task); | |
440 | else | |
441 | dev_dbg(hsdma->ddev.dev, "unhandle irq status %08x\n", | |
4554af34 | 442 | status); |
0853c7a5 JC |
443 | /* clean intr bits */ |
444 | mtk_hsdma_write(hsdma, HSDMA_REG_INT_STATUS, status); | |
445 | ||
446 | return IRQ_HANDLED; | |
447 | } | |
448 | ||
449 | static void mtk_hsdma_issue_pending(struct dma_chan *c) | |
450 | { | |
451 | struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c); | |
452 | struct mtk_hsdam_engine *hsdma = mtk_hsdma_chan_get_dev(chan); | |
453 | ||
454 | spin_lock_bh(&chan->vchan.lock); | |
455 | if (vchan_issue_pending(&chan->vchan) && !chan->desc) { | |
456 | if (gdma_next_desc(chan)) { | |
457 | set_bit(chan->id, &hsdma->chan_issued); | |
458 | tasklet_schedule(&hsdma->task); | |
459 | } else | |
460 | dev_dbg(hsdma->ddev.dev, "no desc to issue\n"); | |
461 | } | |
462 | spin_unlock_bh(&chan->vchan.lock); | |
463 | } | |
464 | ||
c91034b2 | 465 | static struct dma_async_tx_descriptor *mtk_hsdma_prep_dma_memcpy( |
0853c7a5 JC |
466 | struct dma_chan *c, dma_addr_t dest, dma_addr_t src, |
467 | size_t len, unsigned long flags) | |
468 | { | |
469 | struct mtk_hsdma_chan *chan = to_mtk_hsdma_chan(c); | |
470 | struct mtk_hsdma_desc *desc; | |
471 | ||
472 | if (len <= 0) | |
473 | return NULL; | |
474 | ||
475 | desc = kzalloc(sizeof(struct mtk_hsdma_desc), GFP_ATOMIC); | |
476 | if (!desc) { | |
477 | dev_err(c->device->dev, "alloc memcpy decs error\n"); | |
478 | return NULL; | |
479 | } | |
480 | ||
481 | desc->sg[0].src_addr = src; | |
482 | desc->sg[0].dst_addr = dest; | |
483 | desc->sg[0].len = len; | |
484 | ||
485 | return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); | |
486 | } | |
487 | ||
488 | static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c, | |
4554af34 CLS |
489 | dma_cookie_t cookie, |
490 | struct dma_tx_state *state) | |
0853c7a5 JC |
491 | { |
492 | return dma_cookie_status(c, cookie, state); | |
493 | } | |
494 | ||
495 | static void mtk_hsdma_free_chan_resources(struct dma_chan *c) | |
496 | { | |
497 | vchan_free_chan_resources(to_virt_chan(c)); | |
498 | } | |
499 | ||
500 | static void mtk_hsdma_desc_free(struct virt_dma_desc *vdesc) | |
501 | { | |
502 | kfree(container_of(vdesc, struct mtk_hsdma_desc, vdesc)); | |
503 | } | |
504 | ||
505 | static void mtk_hsdma_tx(struct mtk_hsdam_engine *hsdma) | |
506 | { | |
507 | struct mtk_hsdma_chan *chan; | |
508 | ||
509 | if (test_and_clear_bit(0, &hsdma->chan_issued)) { | |
510 | chan = &hsdma->chan[0]; | |
5e62653b | 511 | if (chan->desc) |
0853c7a5 | 512 | mtk_hsdma_start_transfer(hsdma, chan); |
5e62653b | 513 | else |
f22d8242 | 514 | dev_dbg(hsdma->ddev.dev, "chan 0 no desc to issue\n"); |
0853c7a5 JC |
515 | } |
516 | } | |
517 | ||
518 | static void mtk_hsdma_rx(struct mtk_hsdam_engine *hsdma) | |
519 | { | |
520 | struct mtk_hsdma_chan *chan; | |
521 | int next_idx, drx_idx, cnt; | |
522 | ||
523 | chan = &hsdma->chan[0]; | |
524 | next_idx = HSDMA_NEXT_DESC(chan->rx_idx); | |
525 | drx_idx = mtk_hsdma_read(hsdma, HSDMA_REG_RX_DRX); | |
526 | ||
527 | cnt = (drx_idx - next_idx) & HSDMA_DESCS_MASK; | |
528 | if (!cnt) | |
529 | return; | |
530 | ||
531 | chan->next_sg += cnt; | |
532 | chan->rx_idx = (chan->rx_idx + cnt) & HSDMA_DESCS_MASK; | |
533 | ||
534 | /* update rx crx */ | |
535 | wmb(); | |
536 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_CRX, chan->rx_idx); | |
537 | ||
538 | mtk_hsdma_chan_done(hsdma, chan); | |
539 | } | |
540 | ||
541 | static void mtk_hsdma_tasklet(unsigned long arg) | |
542 | { | |
543 | struct mtk_hsdam_engine *hsdma = (struct mtk_hsdam_engine *)arg; | |
544 | ||
545 | mtk_hsdma_rx(hsdma); | |
546 | mtk_hsdma_tx(hsdma); | |
547 | } | |
548 | ||
549 | static int mtk_hsdam_alloc_desc(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 550 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
551 | { |
552 | int i; | |
553 | ||
554 | chan->tx_ring = dma_alloc_coherent(hsdma->ddev.dev, | |
555 | 2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring), | |
556 | &chan->desc_addr, GFP_ATOMIC | __GFP_ZERO); | |
557 | if (!chan->tx_ring) | |
558 | goto no_mem; | |
559 | ||
560 | chan->rx_ring = &chan->tx_ring[HSDMA_DESCS_NUM]; | |
561 | ||
562 | /* init tx ring value */ | |
563 | for (i = 0; i < HSDMA_DESCS_NUM; i++) | |
564 | chan->tx_ring[i].flags = HSDMA_DESC_LS0 | HSDMA_DESC_DONE; | |
565 | ||
566 | return 0; | |
567 | no_mem: | |
568 | return -ENOMEM; | |
569 | } | |
570 | ||
571 | static void mtk_hsdam_free_desc(struct mtk_hsdam_engine *hsdma, | |
4554af34 | 572 | struct mtk_hsdma_chan *chan) |
0853c7a5 JC |
573 | { |
574 | if (chan->tx_ring) { | |
575 | dma_free_coherent(hsdma->ddev.dev, | |
576 | 2 * HSDMA_DESCS_NUM * sizeof(*chan->tx_ring), | |
577 | chan->tx_ring, chan->desc_addr); | |
578 | chan->tx_ring = NULL; | |
579 | chan->rx_ring = NULL; | |
580 | } | |
581 | } | |
582 | ||
583 | static int mtk_hsdma_init(struct mtk_hsdam_engine *hsdma) | |
584 | { | |
585 | struct mtk_hsdma_chan *chan; | |
586 | int ret; | |
587 | u32 reg; | |
588 | ||
589 | /* init desc */ | |
590 | chan = &hsdma->chan[0]; | |
591 | ret = mtk_hsdam_alloc_desc(hsdma, chan); | |
592 | if (ret) | |
593 | return ret; | |
594 | ||
595 | /* tx */ | |
596 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, chan->desc_addr); | |
597 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, HSDMA_DESCS_NUM); | |
598 | /* rx */ | |
599 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, chan->desc_addr + | |
600 | (sizeof(struct hsdma_desc) * HSDMA_DESCS_NUM)); | |
601 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, HSDMA_DESCS_NUM); | |
602 | /* reset */ | |
603 | mtk_hsdma_reset_chan(hsdma, chan); | |
604 | ||
605 | /* enable rx intr */ | |
606 | mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, HSDMA_INT_RX_Q0); | |
607 | ||
608 | /* enable dma */ | |
609 | mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, HSDMA_GLO_DEFAULT); | |
610 | ||
611 | /* hardware info */ | |
612 | reg = mtk_hsdma_read(hsdma, HSDMA_REG_INFO); | |
613 | dev_info(hsdma->ddev.dev, "rx: %d, tx: %d\n", | |
4554af34 CLS |
614 | (reg >> HSDMA_INFO_RX_SHIFT) & HSDMA_INFO_RX_MASK, |
615 | (reg >> HSDMA_INFO_TX_SHIFT) & HSDMA_INFO_TX_MASK); | |
0853c7a5 JC |
616 | |
617 | hsdma_dump_reg(hsdma); | |
618 | ||
619 | return ret; | |
620 | } | |
621 | ||
622 | static void mtk_hsdma_uninit(struct mtk_hsdam_engine *hsdma) | |
623 | { | |
624 | struct mtk_hsdma_chan *chan; | |
625 | ||
626 | /* disable dma */ | |
627 | mtk_hsdma_write(hsdma, HSDMA_REG_GLO_CFG, 0); | |
628 | ||
629 | /* disable intr */ | |
630 | mtk_hsdma_write(hsdma, HSDMA_REG_INT_MASK, 0); | |
631 | ||
632 | /* free desc */ | |
633 | chan = &hsdma->chan[0]; | |
634 | mtk_hsdam_free_desc(hsdma, chan); | |
635 | ||
636 | /* tx */ | |
637 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_BASE, 0); | |
638 | mtk_hsdma_write(hsdma, HSDMA_REG_TX_CNT, 0); | |
639 | /* rx */ | |
640 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_BASE, 0); | |
641 | mtk_hsdma_write(hsdma, HSDMA_REG_RX_CNT, 0); | |
642 | /* reset */ | |
643 | mtk_hsdma_reset_chan(hsdma, chan); | |
644 | } | |
645 | ||
646 | static const struct of_device_id mtk_hsdma_of_match[] = { | |
647 | { .compatible = "mediatek,mt7621-hsdma" }, | |
648 | { }, | |
649 | }; | |
650 | ||
651 | static int mtk_hsdma_probe(struct platform_device *pdev) | |
652 | { | |
653 | const struct of_device_id *match; | |
654 | struct mtk_hsdma_chan *chan; | |
655 | struct mtk_hsdam_engine *hsdma; | |
656 | struct dma_device *dd; | |
657 | struct resource *res; | |
658 | int ret; | |
659 | int irq; | |
660 | void __iomem *base; | |
661 | ||
662 | ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); | |
663 | if (ret) | |
664 | return ret; | |
665 | ||
666 | match = of_match_device(mtk_hsdma_of_match, &pdev->dev); | |
667 | if (!match) | |
668 | return -EINVAL; | |
669 | ||
670 | hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL); | |
671 | if (!hsdma) { | |
672 | dev_err(&pdev->dev, "alloc dma device failed\n"); | |
673 | return -EINVAL; | |
674 | } | |
675 | ||
676 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
677 | base = devm_ioremap_resource(&pdev->dev, res); | |
678 | if (IS_ERR(base)) | |
679 | return PTR_ERR(base); | |
680 | hsdma->base = base + HSDMA_BASE_OFFSET; | |
681 | tasklet_init(&hsdma->task, mtk_hsdma_tasklet, (unsigned long)hsdma); | |
682 | ||
683 | irq = platform_get_irq(pdev, 0); | |
684 | if (irq < 0) { | |
685 | dev_err(&pdev->dev, "failed to get irq\n"); | |
686 | return -EINVAL; | |
687 | } | |
688 | ret = devm_request_irq(&pdev->dev, irq, mtk_hsdma_irq, | |
4554af34 | 689 | 0, dev_name(&pdev->dev), hsdma); |
0853c7a5 JC |
690 | if (ret) { |
691 | dev_err(&pdev->dev, "failed to request irq\n"); | |
692 | return ret; | |
693 | } | |
694 | ||
695 | device_reset(&pdev->dev); | |
696 | ||
697 | dd = &hsdma->ddev; | |
698 | dma_cap_set(DMA_MEMCPY, dd->cap_mask); | |
699 | dd->copy_align = HSDMA_ALIGN_SIZE; | |
700 | dd->device_free_chan_resources = mtk_hsdma_free_chan_resources; | |
701 | dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy; | |
702 | dd->device_terminate_all = mtk_hsdma_terminate_all; | |
703 | dd->device_tx_status = mtk_hsdma_tx_status; | |
704 | dd->device_issue_pending = mtk_hsdma_issue_pending; | |
705 | dd->dev = &pdev->dev; | |
706 | dd->dev->dma_parms = &hsdma->dma_parms; | |
707 | dma_set_max_seg_size(dd->dev, HSDMA_MAX_PLEN); | |
708 | INIT_LIST_HEAD(&dd->channels); | |
709 | ||
710 | chan = &hsdma->chan[0]; | |
711 | chan->id = 0; | |
712 | chan->vchan.desc_free = mtk_hsdma_desc_free; | |
713 | vchan_init(&chan->vchan, dd); | |
714 | ||
715 | /* init hardware */ | |
716 | ret = mtk_hsdma_init(hsdma); | |
717 | if (ret) { | |
718 | dev_err(&pdev->dev, "failed to alloc ring descs\n"); | |
719 | return ret; | |
720 | } | |
721 | ||
722 | ret = dma_async_device_register(dd); | |
723 | if (ret) { | |
724 | dev_err(&pdev->dev, "failed to register dma device\n"); | |
725 | return ret; | |
726 | } | |
727 | ||
728 | ret = of_dma_controller_register(pdev->dev.of_node, | |
4554af34 | 729 | of_dma_xlate_by_chan_id, hsdma); |
0853c7a5 JC |
730 | if (ret) { |
731 | dev_err(&pdev->dev, "failed to register of dma controller\n"); | |
732 | goto err_unregister; | |
733 | } | |
734 | ||
735 | platform_set_drvdata(pdev, hsdma); | |
736 | ||
737 | return 0; | |
738 | ||
739 | err_unregister: | |
740 | dma_async_device_unregister(dd); | |
741 | return ret; | |
742 | } | |
743 | ||
744 | static int mtk_hsdma_remove(struct platform_device *pdev) | |
745 | { | |
746 | struct mtk_hsdam_engine *hsdma = platform_get_drvdata(pdev); | |
747 | ||
748 | mtk_hsdma_uninit(hsdma); | |
749 | ||
750 | of_dma_controller_free(pdev->dev.of_node); | |
751 | dma_async_device_unregister(&hsdma->ddev); | |
752 | ||
753 | return 0; | |
754 | } | |
755 | ||
756 | static struct platform_driver mtk_hsdma_driver = { | |
757 | .probe = mtk_hsdma_probe, | |
758 | .remove = mtk_hsdma_remove, | |
759 | .driver = { | |
760 | .name = "hsdma-mt7621", | |
761 | .of_match_table = mtk_hsdma_of_match, | |
762 | }, | |
763 | }; | |
764 | module_platform_driver(mtk_hsdma_driver); | |
765 | ||
766 | MODULE_AUTHOR("Michael Lee <igvtee@gmail.com>"); | |
767 | MODULE_DESCRIPTION("MTK HSDMA driver"); | |
768 | MODULE_LICENSE("GPL v2"); |