Commit | Line | Data |
---|---|---|
ea76f0b3 AN |
1 | /* |
2 | * Driver for the TXx9 SoC DMA Controller | |
3 | * | |
4 | * Copyright (C) 2009 Atsushi Nemoto | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | */ | |
10 | #include <linux/dma-mapping.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/interrupt.h> | |
13 | #include <linux/io.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/platform_device.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/scatterlist.h> | |
d2ebfb33 RKAL |
18 | |
19 | #include "dmaengine.h" | |
ea76f0b3 AN |
20 | #include "txx9dmac.h" |
21 | ||
22 | static struct txx9dmac_chan *to_txx9dmac_chan(struct dma_chan *chan) | |
23 | { | |
24 | return container_of(chan, struct txx9dmac_chan, chan); | |
25 | } | |
26 | ||
27 | static struct txx9dmac_cregs __iomem *__dma_regs(const struct txx9dmac_chan *dc) | |
28 | { | |
29 | return dc->ch_regs; | |
30 | } | |
31 | ||
32 | static struct txx9dmac_cregs32 __iomem *__dma_regs32( | |
33 | const struct txx9dmac_chan *dc) | |
34 | { | |
35 | return dc->ch_regs; | |
36 | } | |
37 | ||
38 | #define channel64_readq(dc, name) \ | |
39 | __raw_readq(&(__dma_regs(dc)->name)) | |
40 | #define channel64_writeq(dc, name, val) \ | |
41 | __raw_writeq((val), &(__dma_regs(dc)->name)) | |
42 | #define channel64_readl(dc, name) \ | |
43 | __raw_readl(&(__dma_regs(dc)->name)) | |
44 | #define channel64_writel(dc, name, val) \ | |
45 | __raw_writel((val), &(__dma_regs(dc)->name)) | |
46 | ||
47 | #define channel32_readl(dc, name) \ | |
48 | __raw_readl(&(__dma_regs32(dc)->name)) | |
49 | #define channel32_writel(dc, name, val) \ | |
50 | __raw_writel((val), &(__dma_regs32(dc)->name)) | |
51 | ||
52 | #define channel_readq(dc, name) channel64_readq(dc, name) | |
53 | #define channel_writeq(dc, name, val) channel64_writeq(dc, name, val) | |
54 | #define channel_readl(dc, name) \ | |
55 | (is_dmac64(dc) ? \ | |
56 | channel64_readl(dc, name) : channel32_readl(dc, name)) | |
57 | #define channel_writel(dc, name, val) \ | |
58 | (is_dmac64(dc) ? \ | |
59 | channel64_writel(dc, name, val) : channel32_writel(dc, name, val)) | |
60 | ||
61 | static dma_addr_t channel64_read_CHAR(const struct txx9dmac_chan *dc) | |
62 | { | |
63 | if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64)) | |
64 | return channel64_readq(dc, CHAR); | |
65 | else | |
66 | return channel64_readl(dc, CHAR); | |
67 | } | |
68 | ||
69 | static void channel64_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val) | |
70 | { | |
71 | if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64)) | |
72 | channel64_writeq(dc, CHAR, val); | |
73 | else | |
74 | channel64_writel(dc, CHAR, val); | |
75 | } | |
76 | ||
77 | static void channel64_clear_CHAR(const struct txx9dmac_chan *dc) | |
78 | { | |
79 | #if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR) | |
80 | channel64_writel(dc, CHAR, 0); | |
81 | channel64_writel(dc, __pad_CHAR, 0); | |
82 | #else | |
83 | channel64_writeq(dc, CHAR, 0); | |
84 | #endif | |
85 | } | |
86 | ||
87 | static dma_addr_t channel_read_CHAR(const struct txx9dmac_chan *dc) | |
88 | { | |
89 | if (is_dmac64(dc)) | |
90 | return channel64_read_CHAR(dc); | |
91 | else | |
92 | return channel32_readl(dc, CHAR); | |
93 | } | |
94 | ||
95 | static void channel_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val) | |
96 | { | |
97 | if (is_dmac64(dc)) | |
98 | channel64_write_CHAR(dc, val); | |
99 | else | |
100 | channel32_writel(dc, CHAR, val); | |
101 | } | |
102 | ||
103 | static struct txx9dmac_regs __iomem *__txx9dmac_regs( | |
104 | const struct txx9dmac_dev *ddev) | |
105 | { | |
106 | return ddev->regs; | |
107 | } | |
108 | ||
109 | static struct txx9dmac_regs32 __iomem *__txx9dmac_regs32( | |
110 | const struct txx9dmac_dev *ddev) | |
111 | { | |
112 | return ddev->regs; | |
113 | } | |
114 | ||
115 | #define dma64_readl(ddev, name) \ | |
116 | __raw_readl(&(__txx9dmac_regs(ddev)->name)) | |
117 | #define dma64_writel(ddev, name, val) \ | |
118 | __raw_writel((val), &(__txx9dmac_regs(ddev)->name)) | |
119 | ||
120 | #define dma32_readl(ddev, name) \ | |
121 | __raw_readl(&(__txx9dmac_regs32(ddev)->name)) | |
122 | #define dma32_writel(ddev, name, val) \ | |
123 | __raw_writel((val), &(__txx9dmac_regs32(ddev)->name)) | |
124 | ||
125 | #define dma_readl(ddev, name) \ | |
126 | (__is_dmac64(ddev) ? \ | |
127 | dma64_readl(ddev, name) : dma32_readl(ddev, name)) | |
128 | #define dma_writel(ddev, name, val) \ | |
129 | (__is_dmac64(ddev) ? \ | |
130 | dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val)) | |
131 | ||
132 | static struct device *chan2dev(struct dma_chan *chan) | |
133 | { | |
134 | return &chan->dev->device; | |
135 | } | |
136 | static struct device *chan2parent(struct dma_chan *chan) | |
137 | { | |
138 | return chan->dev->device.parent; | |
139 | } | |
140 | ||
141 | static struct txx9dmac_desc * | |
142 | txd_to_txx9dmac_desc(struct dma_async_tx_descriptor *txd) | |
143 | { | |
144 | return container_of(txd, struct txx9dmac_desc, txd); | |
145 | } | |
146 | ||
147 | static dma_addr_t desc_read_CHAR(const struct txx9dmac_chan *dc, | |
148 | const struct txx9dmac_desc *desc) | |
149 | { | |
150 | return is_dmac64(dc) ? desc->hwdesc.CHAR : desc->hwdesc32.CHAR; | |
151 | } | |
152 | ||
153 | static void desc_write_CHAR(const struct txx9dmac_chan *dc, | |
154 | struct txx9dmac_desc *desc, dma_addr_t val) | |
155 | { | |
156 | if (is_dmac64(dc)) | |
157 | desc->hwdesc.CHAR = val; | |
158 | else | |
159 | desc->hwdesc32.CHAR = val; | |
160 | } | |
161 | ||
162 | #define TXX9_DMA_MAX_COUNT 0x04000000 | |
163 | ||
164 | #define TXX9_DMA_INITIAL_DESC_COUNT 64 | |
165 | ||
166 | static struct txx9dmac_desc *txx9dmac_first_active(struct txx9dmac_chan *dc) | |
167 | { | |
168 | return list_entry(dc->active_list.next, | |
169 | struct txx9dmac_desc, desc_node); | |
170 | } | |
171 | ||
172 | static struct txx9dmac_desc *txx9dmac_last_active(struct txx9dmac_chan *dc) | |
173 | { | |
174 | return list_entry(dc->active_list.prev, | |
175 | struct txx9dmac_desc, desc_node); | |
176 | } | |
177 | ||
178 | static struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc) | |
179 | { | |
180 | return list_entry(dc->queue.next, struct txx9dmac_desc, desc_node); | |
181 | } | |
182 | ||
183 | static struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc) | |
184 | { | |
1979b186 DW |
185 | if (!list_empty(&desc->tx_list)) |
186 | desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node); | |
ea76f0b3 AN |
187 | return desc; |
188 | } | |
189 | ||
190 | static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx); | |
191 | ||
192 | static struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc, | |
193 | gfp_t flags) | |
194 | { | |
195 | struct txx9dmac_dev *ddev = dc->ddev; | |
196 | struct txx9dmac_desc *desc; | |
197 | ||
198 | desc = kzalloc(sizeof(*desc), flags); | |
199 | if (!desc) | |
200 | return NULL; | |
1979b186 | 201 | INIT_LIST_HEAD(&desc->tx_list); |
ea76f0b3 AN |
202 | dma_async_tx_descriptor_init(&desc->txd, &dc->chan); |
203 | desc->txd.tx_submit = txx9dmac_tx_submit; | |
204 | /* txd.flags will be overwritten in prep funcs */ | |
205 | desc->txd.flags = DMA_CTRL_ACK; | |
206 | desc->txd.phys = dma_map_single(chan2parent(&dc->chan), &desc->hwdesc, | |
207 | ddev->descsize, DMA_TO_DEVICE); | |
208 | return desc; | |
209 | } | |
210 | ||
211 | static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc) | |
212 | { | |
213 | struct txx9dmac_desc *desc, *_desc; | |
214 | struct txx9dmac_desc *ret = NULL; | |
215 | unsigned int i = 0; | |
216 | ||
217 | spin_lock_bh(&dc->lock); | |
218 | list_for_each_entry_safe(desc, _desc, &dc->free_list, desc_node) { | |
219 | if (async_tx_test_ack(&desc->txd)) { | |
220 | list_del(&desc->desc_node); | |
221 | ret = desc; | |
222 | break; | |
223 | } | |
224 | dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc); | |
225 | i++; | |
226 | } | |
227 | spin_unlock_bh(&dc->lock); | |
228 | ||
229 | dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n", | |
230 | i); | |
231 | if (!ret) { | |
232 | ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC); | |
233 | if (ret) { | |
234 | spin_lock_bh(&dc->lock); | |
235 | dc->descs_allocated++; | |
236 | spin_unlock_bh(&dc->lock); | |
237 | } else | |
238 | dev_err(chan2dev(&dc->chan), | |
239 | "not enough descriptors available\n"); | |
240 | } | |
241 | return ret; | |
242 | } | |
243 | ||
244 | static void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc, | |
245 | struct txx9dmac_desc *desc) | |
246 | { | |
247 | struct txx9dmac_dev *ddev = dc->ddev; | |
248 | struct txx9dmac_desc *child; | |
249 | ||
1979b186 | 250 | list_for_each_entry(child, &desc->tx_list, desc_node) |
ea76f0b3 AN |
251 | dma_sync_single_for_cpu(chan2parent(&dc->chan), |
252 | child->txd.phys, ddev->descsize, | |
253 | DMA_TO_DEVICE); | |
254 | dma_sync_single_for_cpu(chan2parent(&dc->chan), | |
255 | desc->txd.phys, ddev->descsize, | |
256 | DMA_TO_DEVICE); | |
257 | } | |
258 | ||
259 | /* | |
260 | * Move a descriptor, including any children, to the free list. | |
261 | * `desc' must not be on any lists. | |
262 | */ | |
263 | static void txx9dmac_desc_put(struct txx9dmac_chan *dc, | |
264 | struct txx9dmac_desc *desc) | |
265 | { | |
266 | if (desc) { | |
267 | struct txx9dmac_desc *child; | |
268 | ||
269 | txx9dmac_sync_desc_for_cpu(dc, desc); | |
270 | ||
271 | spin_lock_bh(&dc->lock); | |
1979b186 | 272 | list_for_each_entry(child, &desc->tx_list, desc_node) |
ea76f0b3 AN |
273 | dev_vdbg(chan2dev(&dc->chan), |
274 | "moving child desc %p to freelist\n", | |
275 | child); | |
1979b186 | 276 | list_splice_init(&desc->tx_list, &dc->free_list); |
ea76f0b3 AN |
277 | dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n", |
278 | desc); | |
279 | list_add(&desc->desc_node, &dc->free_list); | |
280 | spin_unlock_bh(&dc->lock); | |
281 | } | |
282 | } | |
283 | ||
284 | /* Called with dc->lock held and bh disabled */ | |
285 | static dma_cookie_t | |
286 | txx9dmac_assign_cookie(struct txx9dmac_chan *dc, struct txx9dmac_desc *desc) | |
287 | { | |
288 | dma_cookie_t cookie = dc->chan.cookie; | |
289 | ||
290 | if (++cookie < 0) | |
291 | cookie = 1; | |
292 | ||
293 | dc->chan.cookie = cookie; | |
294 | desc->txd.cookie = cookie; | |
295 | ||
296 | return cookie; | |
297 | } | |
298 | ||
299 | /*----------------------------------------------------------------------*/ | |
300 | ||
301 | static void txx9dmac_dump_regs(struct txx9dmac_chan *dc) | |
302 | { | |
303 | if (is_dmac64(dc)) | |
304 | dev_err(chan2dev(&dc->chan), | |
305 | " CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x" | |
306 | " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", | |
307 | (u64)channel64_read_CHAR(dc), | |
308 | channel64_readq(dc, SAR), | |
309 | channel64_readq(dc, DAR), | |
310 | channel64_readl(dc, CNTR), | |
311 | channel64_readl(dc, SAIR), | |
312 | channel64_readl(dc, DAIR), | |
313 | channel64_readl(dc, CCR), | |
314 | channel64_readl(dc, CSR)); | |
315 | else | |
316 | dev_err(chan2dev(&dc->chan), | |
317 | " CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x" | |
318 | " SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n", | |
319 | channel32_readl(dc, CHAR), | |
320 | channel32_readl(dc, SAR), | |
321 | channel32_readl(dc, DAR), | |
322 | channel32_readl(dc, CNTR), | |
323 | channel32_readl(dc, SAIR), | |
324 | channel32_readl(dc, DAIR), | |
325 | channel32_readl(dc, CCR), | |
326 | channel32_readl(dc, CSR)); | |
327 | } | |
328 | ||
329 | static void txx9dmac_reset_chan(struct txx9dmac_chan *dc) | |
330 | { | |
331 | channel_writel(dc, CCR, TXX9_DMA_CCR_CHRST); | |
332 | if (is_dmac64(dc)) { | |
333 | channel64_clear_CHAR(dc); | |
334 | channel_writeq(dc, SAR, 0); | |
335 | channel_writeq(dc, DAR, 0); | |
336 | } else { | |
337 | channel_writel(dc, CHAR, 0); | |
338 | channel_writel(dc, SAR, 0); | |
339 | channel_writel(dc, DAR, 0); | |
340 | } | |
341 | channel_writel(dc, CNTR, 0); | |
342 | channel_writel(dc, SAIR, 0); | |
343 | channel_writel(dc, DAIR, 0); | |
344 | channel_writel(dc, CCR, 0); | |
345 | mmiowb(); | |
346 | } | |
347 | ||
348 | /* Called with dc->lock held and bh disabled */ | |
349 | static void txx9dmac_dostart(struct txx9dmac_chan *dc, | |
350 | struct txx9dmac_desc *first) | |
351 | { | |
352 | struct txx9dmac_slave *ds = dc->chan.private; | |
353 | u32 sai, dai; | |
354 | ||
355 | dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n", | |
356 | first->txd.cookie, first); | |
357 | /* ASSERT: channel is idle */ | |
358 | if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { | |
359 | dev_err(chan2dev(&dc->chan), | |
360 | "BUG: Attempted to start non-idle channel\n"); | |
361 | txx9dmac_dump_regs(dc); | |
362 | /* The tasklet will hopefully advance the queue... */ | |
363 | return; | |
364 | } | |
365 | ||
366 | if (is_dmac64(dc)) { | |
367 | channel64_writel(dc, CNTR, 0); | |
368 | channel64_writel(dc, CSR, 0xffffffff); | |
369 | if (ds) { | |
370 | if (ds->tx_reg) { | |
371 | sai = ds->reg_width; | |
372 | dai = 0; | |
373 | } else { | |
374 | sai = 0; | |
375 | dai = ds->reg_width; | |
376 | } | |
377 | } else { | |
378 | sai = 8; | |
379 | dai = 8; | |
380 | } | |
381 | channel64_writel(dc, SAIR, sai); | |
382 | channel64_writel(dc, DAIR, dai); | |
383 | /* All 64-bit DMAC supports SMPCHN */ | |
384 | channel64_writel(dc, CCR, dc->ccr); | |
385 | /* Writing a non zero value to CHAR will assert XFACT */ | |
386 | channel64_write_CHAR(dc, first->txd.phys); | |
387 | } else { | |
388 | channel32_writel(dc, CNTR, 0); | |
389 | channel32_writel(dc, CSR, 0xffffffff); | |
390 | if (ds) { | |
391 | if (ds->tx_reg) { | |
392 | sai = ds->reg_width; | |
393 | dai = 0; | |
394 | } else { | |
395 | sai = 0; | |
396 | dai = ds->reg_width; | |
397 | } | |
398 | } else { | |
399 | sai = 4; | |
400 | dai = 4; | |
401 | } | |
402 | channel32_writel(dc, SAIR, sai); | |
403 | channel32_writel(dc, DAIR, dai); | |
404 | if (txx9_dma_have_SMPCHN()) { | |
405 | channel32_writel(dc, CCR, dc->ccr); | |
406 | /* Writing a non zero value to CHAR will assert XFACT */ | |
407 | channel32_writel(dc, CHAR, first->txd.phys); | |
408 | } else { | |
409 | channel32_writel(dc, CHAR, first->txd.phys); | |
410 | channel32_writel(dc, CCR, dc->ccr); | |
411 | } | |
412 | } | |
413 | } | |
414 | ||
415 | /*----------------------------------------------------------------------*/ | |
416 | ||
417 | static void | |
418 | txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, | |
419 | struct txx9dmac_desc *desc) | |
420 | { | |
421 | dma_async_tx_callback callback; | |
422 | void *param; | |
423 | struct dma_async_tx_descriptor *txd = &desc->txd; | |
424 | struct txx9dmac_slave *ds = dc->chan.private; | |
425 | ||
426 | dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n", | |
427 | txd->cookie, desc); | |
428 | ||
4d4e58de | 429 | dc->chan.completed_cookie = txd->cookie; |
ea76f0b3 AN |
430 | callback = txd->callback; |
431 | param = txd->callback_param; | |
432 | ||
433 | txx9dmac_sync_desc_for_cpu(dc, desc); | |
1979b186 | 434 | list_splice_init(&desc->tx_list, &dc->free_list); |
ea76f0b3 AN |
435 | list_move(&desc->desc_node, &dc->free_list); |
436 | ||
ea76f0b3 AN |
437 | if (!ds) { |
438 | dma_addr_t dmaaddr; | |
439 | if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { | |
440 | dmaaddr = is_dmac64(dc) ? | |
441 | desc->hwdesc.DAR : desc->hwdesc32.DAR; | |
4ac4aa5c AN |
442 | if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) |
443 | dma_unmap_single(chan2parent(&dc->chan), | |
444 | dmaaddr, desc->len, DMA_FROM_DEVICE); | |
445 | else | |
446 | dma_unmap_page(chan2parent(&dc->chan), | |
447 | dmaaddr, desc->len, DMA_FROM_DEVICE); | |
ea76f0b3 AN |
448 | } |
449 | if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { | |
450 | dmaaddr = is_dmac64(dc) ? | |
451 | desc->hwdesc.SAR : desc->hwdesc32.SAR; | |
4ac4aa5c AN |
452 | if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) |
453 | dma_unmap_single(chan2parent(&dc->chan), | |
454 | dmaaddr, desc->len, DMA_TO_DEVICE); | |
455 | else | |
456 | dma_unmap_page(chan2parent(&dc->chan), | |
457 | dmaaddr, desc->len, DMA_TO_DEVICE); | |
ea76f0b3 AN |
458 | } |
459 | } | |
460 | ||
461 | /* | |
462 | * The API requires that no submissions are done from a | |
463 | * callback, so we don't need to drop the lock here | |
464 | */ | |
465 | if (callback) | |
466 | callback(param); | |
467 | dma_run_dependencies(txd); | |
468 | } | |
469 | ||
470 | static void txx9dmac_dequeue(struct txx9dmac_chan *dc, struct list_head *list) | |
471 | { | |
472 | struct txx9dmac_dev *ddev = dc->ddev; | |
473 | struct txx9dmac_desc *desc; | |
474 | struct txx9dmac_desc *prev = NULL; | |
475 | ||
476 | BUG_ON(!list_empty(list)); | |
477 | do { | |
478 | desc = txx9dmac_first_queued(dc); | |
479 | if (prev) { | |
480 | desc_write_CHAR(dc, prev, desc->txd.phys); | |
481 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
482 | prev->txd.phys, ddev->descsize, | |
483 | DMA_TO_DEVICE); | |
484 | } | |
485 | prev = txx9dmac_last_child(desc); | |
486 | list_move_tail(&desc->desc_node, list); | |
487 | /* Make chain-completion interrupt happen */ | |
488 | if ((desc->txd.flags & DMA_PREP_INTERRUPT) && | |
489 | !txx9dmac_chan_INTENT(dc)) | |
490 | break; | |
491 | } while (!list_empty(&dc->queue)); | |
492 | } | |
493 | ||
494 | static void txx9dmac_complete_all(struct txx9dmac_chan *dc) | |
495 | { | |
496 | struct txx9dmac_desc *desc, *_desc; | |
497 | LIST_HEAD(list); | |
498 | ||
499 | /* | |
500 | * Submit queued descriptors ASAP, i.e. before we go through | |
501 | * the completed ones. | |
502 | */ | |
503 | list_splice_init(&dc->active_list, &list); | |
504 | if (!list_empty(&dc->queue)) { | |
505 | txx9dmac_dequeue(dc, &dc->active_list); | |
506 | txx9dmac_dostart(dc, txx9dmac_first_active(dc)); | |
507 | } | |
508 | ||
509 | list_for_each_entry_safe(desc, _desc, &list, desc_node) | |
510 | txx9dmac_descriptor_complete(dc, desc); | |
511 | } | |
512 | ||
513 | static void txx9dmac_dump_desc(struct txx9dmac_chan *dc, | |
514 | struct txx9dmac_hwdesc *desc) | |
515 | { | |
516 | if (is_dmac64(dc)) { | |
517 | #ifdef TXX9_DMA_USE_SIMPLE_CHAIN | |
518 | dev_crit(chan2dev(&dc->chan), | |
519 | " desc: ch%#llx s%#llx d%#llx c%#x\n", | |
520 | (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR); | |
521 | #else | |
522 | dev_crit(chan2dev(&dc->chan), | |
523 | " desc: ch%#llx s%#llx d%#llx c%#x" | |
524 | " si%#x di%#x cc%#x cs%#x\n", | |
525 | (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR, | |
526 | desc->SAIR, desc->DAIR, desc->CCR, desc->CSR); | |
527 | #endif | |
528 | } else { | |
529 | struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc; | |
530 | #ifdef TXX9_DMA_USE_SIMPLE_CHAIN | |
531 | dev_crit(chan2dev(&dc->chan), | |
532 | " desc: ch%#x s%#x d%#x c%#x\n", | |
533 | d->CHAR, d->SAR, d->DAR, d->CNTR); | |
534 | #else | |
535 | dev_crit(chan2dev(&dc->chan), | |
536 | " desc: ch%#x s%#x d%#x c%#x" | |
537 | " si%#x di%#x cc%#x cs%#x\n", | |
538 | d->CHAR, d->SAR, d->DAR, d->CNTR, | |
539 | d->SAIR, d->DAIR, d->CCR, d->CSR); | |
540 | #endif | |
541 | } | |
542 | } | |
543 | ||
544 | static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr) | |
545 | { | |
546 | struct txx9dmac_desc *bad_desc; | |
547 | struct txx9dmac_desc *child; | |
548 | u32 errors; | |
549 | ||
550 | /* | |
551 | * The descriptor currently at the head of the active list is | |
552 | * borked. Since we don't have any way to report errors, we'll | |
553 | * just have to scream loudly and try to carry on. | |
554 | */ | |
555 | dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n"); | |
556 | txx9dmac_dump_regs(dc); | |
557 | ||
558 | bad_desc = txx9dmac_first_active(dc); | |
559 | list_del_init(&bad_desc->desc_node); | |
560 | ||
561 | /* Clear all error flags and try to restart the controller */ | |
562 | errors = csr & (TXX9_DMA_CSR_ABCHC | | |
563 | TXX9_DMA_CSR_CFERR | TXX9_DMA_CSR_CHERR | | |
564 | TXX9_DMA_CSR_DESERR | TXX9_DMA_CSR_SORERR); | |
565 | channel_writel(dc, CSR, errors); | |
566 | ||
567 | if (list_empty(&dc->active_list) && !list_empty(&dc->queue)) | |
568 | txx9dmac_dequeue(dc, &dc->active_list); | |
569 | if (!list_empty(&dc->active_list)) | |
570 | txx9dmac_dostart(dc, txx9dmac_first_active(dc)); | |
571 | ||
572 | dev_crit(chan2dev(&dc->chan), | |
573 | "Bad descriptor submitted for DMA! (cookie: %d)\n", | |
574 | bad_desc->txd.cookie); | |
575 | txx9dmac_dump_desc(dc, &bad_desc->hwdesc); | |
1979b186 | 576 | list_for_each_entry(child, &bad_desc->tx_list, desc_node) |
ea76f0b3 AN |
577 | txx9dmac_dump_desc(dc, &child->hwdesc); |
578 | /* Pretend the descriptor completed successfully */ | |
579 | txx9dmac_descriptor_complete(dc, bad_desc); | |
580 | } | |
581 | ||
582 | static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc) | |
583 | { | |
584 | dma_addr_t chain; | |
585 | struct txx9dmac_desc *desc, *_desc; | |
586 | struct txx9dmac_desc *child; | |
587 | u32 csr; | |
588 | ||
589 | if (is_dmac64(dc)) { | |
590 | chain = channel64_read_CHAR(dc); | |
591 | csr = channel64_readl(dc, CSR); | |
592 | channel64_writel(dc, CSR, csr); | |
593 | } else { | |
594 | chain = channel32_readl(dc, CHAR); | |
595 | csr = channel32_readl(dc, CSR); | |
596 | channel32_writel(dc, CSR, csr); | |
597 | } | |
598 | /* For dynamic chain, we should look at XFACT instead of NCHNC */ | |
599 | if (!(csr & (TXX9_DMA_CSR_XFACT | TXX9_DMA_CSR_ABCHC))) { | |
600 | /* Everything we've submitted is done */ | |
601 | txx9dmac_complete_all(dc); | |
602 | return; | |
603 | } | |
604 | if (!(csr & TXX9_DMA_CSR_CHNEN)) | |
605 | chain = 0; /* last descriptor of this chain */ | |
606 | ||
607 | dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n", | |
608 | (u64)chain); | |
609 | ||
610 | list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) { | |
611 | if (desc_read_CHAR(dc, desc) == chain) { | |
612 | /* This one is currently in progress */ | |
613 | if (csr & TXX9_DMA_CSR_ABCHC) | |
614 | goto scan_done; | |
615 | return; | |
616 | } | |
617 | ||
1979b186 | 618 | list_for_each_entry(child, &desc->tx_list, desc_node) |
ea76f0b3 AN |
619 | if (desc_read_CHAR(dc, child) == chain) { |
620 | /* Currently in progress */ | |
621 | if (csr & TXX9_DMA_CSR_ABCHC) | |
622 | goto scan_done; | |
623 | return; | |
624 | } | |
625 | ||
626 | /* | |
627 | * No descriptors so far seem to be in progress, i.e. | |
628 | * this one must be done. | |
629 | */ | |
630 | txx9dmac_descriptor_complete(dc, desc); | |
631 | } | |
632 | scan_done: | |
633 | if (csr & TXX9_DMA_CSR_ABCHC) { | |
634 | txx9dmac_handle_error(dc, csr); | |
635 | return; | |
636 | } | |
637 | ||
638 | dev_err(chan2dev(&dc->chan), | |
639 | "BUG: All descriptors done, but channel not idle!\n"); | |
640 | ||
641 | /* Try to continue after resetting the channel... */ | |
642 | txx9dmac_reset_chan(dc); | |
643 | ||
644 | if (!list_empty(&dc->queue)) { | |
645 | txx9dmac_dequeue(dc, &dc->active_list); | |
646 | txx9dmac_dostart(dc, txx9dmac_first_active(dc)); | |
647 | } | |
648 | } | |
649 | ||
650 | static void txx9dmac_chan_tasklet(unsigned long data) | |
651 | { | |
652 | int irq; | |
653 | u32 csr; | |
654 | struct txx9dmac_chan *dc; | |
655 | ||
656 | dc = (struct txx9dmac_chan *)data; | |
657 | csr = channel_readl(dc, CSR); | |
658 | dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr); | |
659 | ||
660 | spin_lock(&dc->lock); | |
661 | if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | | |
662 | TXX9_DMA_CSR_NTRNFC)) | |
663 | txx9dmac_scan_descriptors(dc); | |
664 | spin_unlock(&dc->lock); | |
665 | irq = dc->irq; | |
666 | ||
667 | enable_irq(irq); | |
668 | } | |
669 | ||
670 | static irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id) | |
671 | { | |
672 | struct txx9dmac_chan *dc = dev_id; | |
673 | ||
674 | dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n", | |
675 | channel_readl(dc, CSR)); | |
676 | ||
677 | tasklet_schedule(&dc->tasklet); | |
678 | /* | |
679 | * Just disable the interrupts. We'll turn them back on in the | |
680 | * softirq handler. | |
681 | */ | |
682 | disable_irq_nosync(irq); | |
683 | ||
684 | return IRQ_HANDLED; | |
685 | } | |
686 | ||
687 | static void txx9dmac_tasklet(unsigned long data) | |
688 | { | |
689 | int irq; | |
690 | u32 csr; | |
691 | struct txx9dmac_chan *dc; | |
692 | ||
693 | struct txx9dmac_dev *ddev = (struct txx9dmac_dev *)data; | |
694 | u32 mcr; | |
695 | int i; | |
696 | ||
697 | mcr = dma_readl(ddev, MCR); | |
698 | dev_vdbg(ddev->chan[0]->dma.dev, "tasklet: mcr=%x\n", mcr); | |
699 | for (i = 0; i < TXX9_DMA_MAX_NR_CHANNELS; i++) { | |
700 | if ((mcr >> (24 + i)) & 0x11) { | |
701 | dc = ddev->chan[i]; | |
702 | csr = channel_readl(dc, CSR); | |
703 | dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", | |
704 | csr); | |
705 | spin_lock(&dc->lock); | |
706 | if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC | | |
707 | TXX9_DMA_CSR_NTRNFC)) | |
708 | txx9dmac_scan_descriptors(dc); | |
709 | spin_unlock(&dc->lock); | |
710 | } | |
711 | } | |
712 | irq = ddev->irq; | |
713 | ||
714 | enable_irq(irq); | |
715 | } | |
716 | ||
717 | static irqreturn_t txx9dmac_interrupt(int irq, void *dev_id) | |
718 | { | |
719 | struct txx9dmac_dev *ddev = dev_id; | |
720 | ||
721 | dev_vdbg(ddev->chan[0]->dma.dev, "interrupt: status=%#x\n", | |
722 | dma_readl(ddev, MCR)); | |
723 | ||
724 | tasklet_schedule(&ddev->tasklet); | |
725 | /* | |
726 | * Just disable the interrupts. We'll turn them back on in the | |
727 | * softirq handler. | |
728 | */ | |
729 | disable_irq_nosync(irq); | |
730 | ||
731 | return IRQ_HANDLED; | |
732 | } | |
733 | ||
734 | /*----------------------------------------------------------------------*/ | |
735 | ||
736 | static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx) | |
737 | { | |
738 | struct txx9dmac_desc *desc = txd_to_txx9dmac_desc(tx); | |
739 | struct txx9dmac_chan *dc = to_txx9dmac_chan(tx->chan); | |
740 | dma_cookie_t cookie; | |
741 | ||
742 | spin_lock_bh(&dc->lock); | |
743 | cookie = txx9dmac_assign_cookie(dc, desc); | |
744 | ||
745 | dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n", | |
746 | desc->txd.cookie, desc); | |
747 | ||
748 | list_add_tail(&desc->desc_node, &dc->queue); | |
749 | spin_unlock_bh(&dc->lock); | |
750 | ||
751 | return cookie; | |
752 | } | |
753 | ||
754 | static struct dma_async_tx_descriptor * | |
755 | txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, | |
756 | size_t len, unsigned long flags) | |
757 | { | |
758 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
759 | struct txx9dmac_dev *ddev = dc->ddev; | |
760 | struct txx9dmac_desc *desc; | |
761 | struct txx9dmac_desc *first; | |
762 | struct txx9dmac_desc *prev; | |
763 | size_t xfer_count; | |
764 | size_t offset; | |
765 | ||
766 | dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n", | |
767 | (u64)dest, (u64)src, len, flags); | |
768 | ||
769 | if (unlikely(!len)) { | |
770 | dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); | |
771 | return NULL; | |
772 | } | |
773 | ||
774 | prev = first = NULL; | |
775 | ||
776 | for (offset = 0; offset < len; offset += xfer_count) { | |
777 | xfer_count = min_t(size_t, len - offset, TXX9_DMA_MAX_COUNT); | |
778 | /* | |
779 | * Workaround for ERT-TX49H2-033, ERT-TX49H3-020, | |
780 | * ERT-TX49H4-016 (slightly conservative) | |
781 | */ | |
782 | if (__is_dmac64(ddev)) { | |
783 | if (xfer_count > 0x100 && | |
784 | (xfer_count & 0xff) >= 0xfa && | |
785 | (xfer_count & 0xff) <= 0xff) | |
786 | xfer_count -= 0x20; | |
787 | } else { | |
788 | if (xfer_count > 0x80 && | |
789 | (xfer_count & 0x7f) >= 0x7e && | |
790 | (xfer_count & 0x7f) <= 0x7f) | |
791 | xfer_count -= 0x20; | |
792 | } | |
793 | ||
794 | desc = txx9dmac_desc_get(dc); | |
795 | if (!desc) { | |
796 | txx9dmac_desc_put(dc, first); | |
797 | return NULL; | |
798 | } | |
799 | ||
800 | if (__is_dmac64(ddev)) { | |
801 | desc->hwdesc.SAR = src + offset; | |
802 | desc->hwdesc.DAR = dest + offset; | |
803 | desc->hwdesc.CNTR = xfer_count; | |
804 | txx9dmac_desc_set_nosimple(ddev, desc, 8, 8, | |
805 | dc->ccr | TXX9_DMA_CCR_XFACT); | |
806 | } else { | |
807 | desc->hwdesc32.SAR = src + offset; | |
808 | desc->hwdesc32.DAR = dest + offset; | |
809 | desc->hwdesc32.CNTR = xfer_count; | |
810 | txx9dmac_desc_set_nosimple(ddev, desc, 4, 4, | |
811 | dc->ccr | TXX9_DMA_CCR_XFACT); | |
812 | } | |
813 | ||
814 | /* | |
815 | * The descriptors on tx_list are not reachable from | |
816 | * the dc->queue list or dc->active_list after a | |
817 | * submit. If we put all descriptors on active_list, | |
818 | * calling of callback on the completion will be more | |
819 | * complex. | |
820 | */ | |
821 | if (!first) { | |
822 | first = desc; | |
823 | } else { | |
824 | desc_write_CHAR(dc, prev, desc->txd.phys); | |
825 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
826 | prev->txd.phys, ddev->descsize, | |
827 | DMA_TO_DEVICE); | |
1979b186 | 828 | list_add_tail(&desc->desc_node, &first->tx_list); |
ea76f0b3 AN |
829 | } |
830 | prev = desc; | |
831 | } | |
832 | ||
833 | /* Trigger interrupt after last block */ | |
834 | if (flags & DMA_PREP_INTERRUPT) | |
835 | txx9dmac_desc_set_INTENT(ddev, prev); | |
836 | ||
837 | desc_write_CHAR(dc, prev, 0); | |
838 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
839 | prev->txd.phys, ddev->descsize, | |
840 | DMA_TO_DEVICE); | |
841 | ||
842 | first->txd.flags = flags; | |
843 | first->len = len; | |
844 | ||
845 | return &first->txd; | |
846 | } | |
847 | ||
848 | static struct dma_async_tx_descriptor * | |
849 | txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, | |
db8196df | 850 | unsigned int sg_len, enum dma_transfer_direction direction, |
ea76f0b3 AN |
851 | unsigned long flags) |
852 | { | |
853 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
854 | struct txx9dmac_dev *ddev = dc->ddev; | |
855 | struct txx9dmac_slave *ds = chan->private; | |
856 | struct txx9dmac_desc *prev; | |
857 | struct txx9dmac_desc *first; | |
858 | unsigned int i; | |
859 | struct scatterlist *sg; | |
860 | ||
861 | dev_vdbg(chan2dev(chan), "prep_dma_slave\n"); | |
862 | ||
863 | BUG_ON(!ds || !ds->reg_width); | |
864 | if (ds->tx_reg) | |
db8196df | 865 | BUG_ON(direction != DMA_MEM_TO_DEV); |
ea76f0b3 | 866 | else |
db8196df | 867 | BUG_ON(direction != DMA_DEV_TO_MEM); |
ea76f0b3 AN |
868 | if (unlikely(!sg_len)) |
869 | return NULL; | |
870 | ||
871 | prev = first = NULL; | |
872 | ||
873 | for_each_sg(sgl, sg, sg_len, i) { | |
874 | struct txx9dmac_desc *desc; | |
875 | dma_addr_t mem; | |
876 | u32 sai, dai; | |
877 | ||
878 | desc = txx9dmac_desc_get(dc); | |
879 | if (!desc) { | |
880 | txx9dmac_desc_put(dc, first); | |
881 | return NULL; | |
882 | } | |
883 | ||
884 | mem = sg_dma_address(sg); | |
885 | ||
886 | if (__is_dmac64(ddev)) { | |
db8196df | 887 | if (direction == DMA_MEM_TO_DEV) { |
ea76f0b3 AN |
888 | desc->hwdesc.SAR = mem; |
889 | desc->hwdesc.DAR = ds->tx_reg; | |
890 | } else { | |
891 | desc->hwdesc.SAR = ds->rx_reg; | |
892 | desc->hwdesc.DAR = mem; | |
893 | } | |
894 | desc->hwdesc.CNTR = sg_dma_len(sg); | |
895 | } else { | |
db8196df | 896 | if (direction == DMA_MEM_TO_DEV) { |
ea76f0b3 AN |
897 | desc->hwdesc32.SAR = mem; |
898 | desc->hwdesc32.DAR = ds->tx_reg; | |
899 | } else { | |
900 | desc->hwdesc32.SAR = ds->rx_reg; | |
901 | desc->hwdesc32.DAR = mem; | |
902 | } | |
903 | desc->hwdesc32.CNTR = sg_dma_len(sg); | |
904 | } | |
db8196df | 905 | if (direction == DMA_MEM_TO_DEV) { |
ea76f0b3 AN |
906 | sai = ds->reg_width; |
907 | dai = 0; | |
908 | } else { | |
909 | sai = 0; | |
910 | dai = ds->reg_width; | |
911 | } | |
912 | txx9dmac_desc_set_nosimple(ddev, desc, sai, dai, | |
913 | dc->ccr | TXX9_DMA_CCR_XFACT); | |
914 | ||
915 | if (!first) { | |
916 | first = desc; | |
917 | } else { | |
918 | desc_write_CHAR(dc, prev, desc->txd.phys); | |
919 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
920 | prev->txd.phys, | |
921 | ddev->descsize, | |
922 | DMA_TO_DEVICE); | |
1979b186 | 923 | list_add_tail(&desc->desc_node, &first->tx_list); |
ea76f0b3 AN |
924 | } |
925 | prev = desc; | |
926 | } | |
927 | ||
928 | /* Trigger interrupt after last block */ | |
929 | if (flags & DMA_PREP_INTERRUPT) | |
930 | txx9dmac_desc_set_INTENT(ddev, prev); | |
931 | ||
932 | desc_write_CHAR(dc, prev, 0); | |
933 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
934 | prev->txd.phys, ddev->descsize, | |
935 | DMA_TO_DEVICE); | |
936 | ||
937 | first->txd.flags = flags; | |
938 | first->len = 0; | |
939 | ||
940 | return &first->txd; | |
941 | } | |
942 | ||
05827630 LW |
943 | static int txx9dmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, |
944 | unsigned long arg) | |
ea76f0b3 AN |
945 | { |
946 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
947 | struct txx9dmac_desc *desc, *_desc; | |
948 | LIST_HEAD(list); | |
949 | ||
c3635c78 LW |
950 | /* Only supports DMA_TERMINATE_ALL */ |
951 | if (cmd != DMA_TERMINATE_ALL) | |
952 | return -EINVAL; | |
953 | ||
ea76f0b3 AN |
954 | dev_vdbg(chan2dev(chan), "terminate_all\n"); |
955 | spin_lock_bh(&dc->lock); | |
956 | ||
957 | txx9dmac_reset_chan(dc); | |
958 | ||
959 | /* active_list entries will end up before queued entries */ | |
960 | list_splice_init(&dc->queue, &list); | |
961 | list_splice_init(&dc->active_list, &list); | |
962 | ||
963 | spin_unlock_bh(&dc->lock); | |
964 | ||
965 | /* Flush all pending and queued descriptors */ | |
966 | list_for_each_entry_safe(desc, _desc, &list, desc_node) | |
967 | txx9dmac_descriptor_complete(dc, desc); | |
c3635c78 LW |
968 | |
969 | return 0; | |
ea76f0b3 AN |
970 | } |
971 | ||
972 | static enum dma_status | |
07934481 LW |
973 | txx9dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, |
974 | struct dma_tx_state *txstate) | |
ea76f0b3 AN |
975 | { |
976 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
977 | dma_cookie_t last_used; | |
978 | dma_cookie_t last_complete; | |
979 | int ret; | |
980 | ||
4d4e58de | 981 | last_complete = chan->completed_cookie; |
ea76f0b3 AN |
982 | last_used = chan->cookie; |
983 | ||
984 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
985 | if (ret != DMA_SUCCESS) { | |
986 | spin_lock_bh(&dc->lock); | |
987 | txx9dmac_scan_descriptors(dc); | |
988 | spin_unlock_bh(&dc->lock); | |
989 | ||
4d4e58de | 990 | last_complete = chan->completed_cookie; |
ea76f0b3 AN |
991 | last_used = chan->cookie; |
992 | ||
993 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
994 | } | |
995 | ||
bca34692 | 996 | dma_set_tx_state(txstate, last_complete, last_used, 0); |
ea76f0b3 AN |
997 | |
998 | return ret; | |
999 | } | |
1000 | ||
1001 | static void txx9dmac_chain_dynamic(struct txx9dmac_chan *dc, | |
1002 | struct txx9dmac_desc *prev) | |
1003 | { | |
1004 | struct txx9dmac_dev *ddev = dc->ddev; | |
1005 | struct txx9dmac_desc *desc; | |
1006 | LIST_HEAD(list); | |
1007 | ||
1008 | prev = txx9dmac_last_child(prev); | |
1009 | txx9dmac_dequeue(dc, &list); | |
1010 | desc = list_entry(list.next, struct txx9dmac_desc, desc_node); | |
1011 | desc_write_CHAR(dc, prev, desc->txd.phys); | |
1012 | dma_sync_single_for_device(chan2parent(&dc->chan), | |
1013 | prev->txd.phys, ddev->descsize, | |
1014 | DMA_TO_DEVICE); | |
1015 | mmiowb(); | |
1016 | if (!(channel_readl(dc, CSR) & TXX9_DMA_CSR_CHNEN) && | |
1017 | channel_read_CHAR(dc) == prev->txd.phys) | |
1018 | /* Restart chain DMA */ | |
1019 | channel_write_CHAR(dc, desc->txd.phys); | |
1020 | list_splice_tail(&list, &dc->active_list); | |
1021 | } | |
1022 | ||
1023 | static void txx9dmac_issue_pending(struct dma_chan *chan) | |
1024 | { | |
1025 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
1026 | ||
1027 | spin_lock_bh(&dc->lock); | |
1028 | ||
1029 | if (!list_empty(&dc->active_list)) | |
1030 | txx9dmac_scan_descriptors(dc); | |
1031 | if (!list_empty(&dc->queue)) { | |
1032 | if (list_empty(&dc->active_list)) { | |
1033 | txx9dmac_dequeue(dc, &dc->active_list); | |
1034 | txx9dmac_dostart(dc, txx9dmac_first_active(dc)); | |
1035 | } else if (txx9_dma_have_SMPCHN()) { | |
1036 | struct txx9dmac_desc *prev = txx9dmac_last_active(dc); | |
1037 | ||
1038 | if (!(prev->txd.flags & DMA_PREP_INTERRUPT) || | |
1039 | txx9dmac_chan_INTENT(dc)) | |
1040 | txx9dmac_chain_dynamic(dc, prev); | |
1041 | } | |
1042 | } | |
1043 | ||
1044 | spin_unlock_bh(&dc->lock); | |
1045 | } | |
1046 | ||
1047 | static int txx9dmac_alloc_chan_resources(struct dma_chan *chan) | |
1048 | { | |
1049 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
1050 | struct txx9dmac_slave *ds = chan->private; | |
1051 | struct txx9dmac_desc *desc; | |
1052 | int i; | |
1053 | ||
1054 | dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); | |
1055 | ||
1056 | /* ASSERT: channel is idle */ | |
1057 | if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) { | |
1058 | dev_dbg(chan2dev(chan), "DMA channel not idle?\n"); | |
1059 | return -EIO; | |
1060 | } | |
1061 | ||
4d4e58de | 1062 | chan->completed_cookie = chan->cookie = 1; |
ea76f0b3 AN |
1063 | |
1064 | dc->ccr = TXX9_DMA_CCR_IMMCHN | TXX9_DMA_CCR_INTENE | CCR_LE; | |
1065 | txx9dmac_chan_set_SMPCHN(dc); | |
1066 | if (!txx9_dma_have_SMPCHN() || (dc->ccr & TXX9_DMA_CCR_SMPCHN)) | |
1067 | dc->ccr |= TXX9_DMA_CCR_INTENC; | |
1068 | if (chan->device->device_prep_dma_memcpy) { | |
1069 | if (ds) | |
1070 | return -EINVAL; | |
1071 | dc->ccr |= TXX9_DMA_CCR_XFSZ_X8; | |
1072 | } else { | |
1073 | if (!ds || | |
1074 | (ds->tx_reg && ds->rx_reg) || (!ds->tx_reg && !ds->rx_reg)) | |
1075 | return -EINVAL; | |
1076 | dc->ccr |= TXX9_DMA_CCR_EXTRQ | | |
1077 | TXX9_DMA_CCR_XFSZ(__ffs(ds->reg_width)); | |
1078 | txx9dmac_chan_set_INTENT(dc); | |
1079 | } | |
1080 | ||
1081 | spin_lock_bh(&dc->lock); | |
1082 | i = dc->descs_allocated; | |
1083 | while (dc->descs_allocated < TXX9_DMA_INITIAL_DESC_COUNT) { | |
1084 | spin_unlock_bh(&dc->lock); | |
1085 | ||
1086 | desc = txx9dmac_desc_alloc(dc, GFP_KERNEL); | |
1087 | if (!desc) { | |
1088 | dev_info(chan2dev(chan), | |
1089 | "only allocated %d descriptors\n", i); | |
1090 | spin_lock_bh(&dc->lock); | |
1091 | break; | |
1092 | } | |
1093 | txx9dmac_desc_put(dc, desc); | |
1094 | ||
1095 | spin_lock_bh(&dc->lock); | |
1096 | i = ++dc->descs_allocated; | |
1097 | } | |
1098 | spin_unlock_bh(&dc->lock); | |
1099 | ||
1100 | dev_dbg(chan2dev(chan), | |
1101 | "alloc_chan_resources allocated %d descriptors\n", i); | |
1102 | ||
1103 | return i; | |
1104 | } | |
1105 | ||
1106 | static void txx9dmac_free_chan_resources(struct dma_chan *chan) | |
1107 | { | |
1108 | struct txx9dmac_chan *dc = to_txx9dmac_chan(chan); | |
1109 | struct txx9dmac_dev *ddev = dc->ddev; | |
1110 | struct txx9dmac_desc *desc, *_desc; | |
1111 | LIST_HEAD(list); | |
1112 | ||
1113 | dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n", | |
1114 | dc->descs_allocated); | |
1115 | ||
1116 | /* ASSERT: channel is idle */ | |
1117 | BUG_ON(!list_empty(&dc->active_list)); | |
1118 | BUG_ON(!list_empty(&dc->queue)); | |
1119 | BUG_ON(channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT); | |
1120 | ||
1121 | spin_lock_bh(&dc->lock); | |
1122 | list_splice_init(&dc->free_list, &list); | |
1123 | dc->descs_allocated = 0; | |
1124 | spin_unlock_bh(&dc->lock); | |
1125 | ||
1126 | list_for_each_entry_safe(desc, _desc, &list, desc_node) { | |
1127 | dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc); | |
1128 | dma_unmap_single(chan2parent(chan), desc->txd.phys, | |
1129 | ddev->descsize, DMA_TO_DEVICE); | |
1130 | kfree(desc); | |
1131 | } | |
1132 | ||
1133 | dev_vdbg(chan2dev(chan), "free_chan_resources done\n"); | |
1134 | } | |
1135 | ||
1136 | /*----------------------------------------------------------------------*/ | |
1137 | ||
1138 | static void txx9dmac_off(struct txx9dmac_dev *ddev) | |
1139 | { | |
1140 | dma_writel(ddev, MCR, 0); | |
1141 | mmiowb(); | |
1142 | } | |
1143 | ||
1144 | static int __init txx9dmac_chan_probe(struct platform_device *pdev) | |
1145 | { | |
1146 | struct txx9dmac_chan_platform_data *cpdata = pdev->dev.platform_data; | |
1147 | struct platform_device *dmac_dev = cpdata->dmac_dev; | |
1148 | struct txx9dmac_platform_data *pdata = dmac_dev->dev.platform_data; | |
1149 | struct txx9dmac_chan *dc; | |
1150 | int err; | |
1151 | int ch = pdev->id % TXX9_DMA_MAX_NR_CHANNELS; | |
1152 | int irq; | |
1153 | ||
1154 | dc = devm_kzalloc(&pdev->dev, sizeof(*dc), GFP_KERNEL); | |
1155 | if (!dc) | |
1156 | return -ENOMEM; | |
1157 | ||
1158 | dc->dma.dev = &pdev->dev; | |
1159 | dc->dma.device_alloc_chan_resources = txx9dmac_alloc_chan_resources; | |
1160 | dc->dma.device_free_chan_resources = txx9dmac_free_chan_resources; | |
c3635c78 | 1161 | dc->dma.device_control = txx9dmac_control; |
07934481 | 1162 | dc->dma.device_tx_status = txx9dmac_tx_status; |
ea76f0b3 AN |
1163 | dc->dma.device_issue_pending = txx9dmac_issue_pending; |
1164 | if (pdata && pdata->memcpy_chan == ch) { | |
1165 | dc->dma.device_prep_dma_memcpy = txx9dmac_prep_dma_memcpy; | |
1166 | dma_cap_set(DMA_MEMCPY, dc->dma.cap_mask); | |
1167 | } else { | |
1168 | dc->dma.device_prep_slave_sg = txx9dmac_prep_slave_sg; | |
1169 | dma_cap_set(DMA_SLAVE, dc->dma.cap_mask); | |
1170 | dma_cap_set(DMA_PRIVATE, dc->dma.cap_mask); | |
1171 | } | |
1172 | ||
1173 | INIT_LIST_HEAD(&dc->dma.channels); | |
1174 | dc->ddev = platform_get_drvdata(dmac_dev); | |
1175 | if (dc->ddev->irq < 0) { | |
1176 | irq = platform_get_irq(pdev, 0); | |
1177 | if (irq < 0) | |
1178 | return irq; | |
1179 | tasklet_init(&dc->tasklet, txx9dmac_chan_tasklet, | |
1180 | (unsigned long)dc); | |
1181 | dc->irq = irq; | |
1182 | err = devm_request_irq(&pdev->dev, dc->irq, | |
1183 | txx9dmac_chan_interrupt, 0, dev_name(&pdev->dev), dc); | |
1184 | if (err) | |
1185 | return err; | |
1186 | } else | |
1187 | dc->irq = -1; | |
1188 | dc->ddev->chan[ch] = dc; | |
1189 | dc->chan.device = &dc->dma; | |
1190 | list_add_tail(&dc->chan.device_node, &dc->chan.device->channels); | |
4d4e58de | 1191 | dc->chan.cookie = dc->chan.completed_cookie = 1; |
ea76f0b3 AN |
1192 | |
1193 | if (is_dmac64(dc)) | |
1194 | dc->ch_regs = &__txx9dmac_regs(dc->ddev)->CHAN[ch]; | |
1195 | else | |
1196 | dc->ch_regs = &__txx9dmac_regs32(dc->ddev)->CHAN[ch]; | |
1197 | spin_lock_init(&dc->lock); | |
1198 | ||
1199 | INIT_LIST_HEAD(&dc->active_list); | |
1200 | INIT_LIST_HEAD(&dc->queue); | |
1201 | INIT_LIST_HEAD(&dc->free_list); | |
1202 | ||
1203 | txx9dmac_reset_chan(dc); | |
1204 | ||
1205 | platform_set_drvdata(pdev, dc); | |
1206 | ||
1207 | err = dma_async_device_register(&dc->dma); | |
1208 | if (err) | |
1209 | return err; | |
1210 | dev_dbg(&pdev->dev, "TXx9 DMA Channel (dma%d%s%s)\n", | |
1211 | dc->dma.dev_id, | |
1212 | dma_has_cap(DMA_MEMCPY, dc->dma.cap_mask) ? " memcpy" : "", | |
1213 | dma_has_cap(DMA_SLAVE, dc->dma.cap_mask) ? " slave" : ""); | |
1214 | ||
1215 | return 0; | |
1216 | } | |
1217 | ||
1218 | static int __exit txx9dmac_chan_remove(struct platform_device *pdev) | |
1219 | { | |
1220 | struct txx9dmac_chan *dc = platform_get_drvdata(pdev); | |
1221 | ||
1222 | dma_async_device_unregister(&dc->dma); | |
1223 | if (dc->irq >= 0) | |
1224 | tasklet_kill(&dc->tasklet); | |
1225 | dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL; | |
1226 | return 0; | |
1227 | } | |
1228 | ||
1229 | static int __init txx9dmac_probe(struct platform_device *pdev) | |
1230 | { | |
1231 | struct txx9dmac_platform_data *pdata = pdev->dev.platform_data; | |
1232 | struct resource *io; | |
1233 | struct txx9dmac_dev *ddev; | |
1234 | u32 mcr; | |
1235 | int err; | |
1236 | ||
1237 | io = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
1238 | if (!io) | |
1239 | return -EINVAL; | |
1240 | ||
1241 | ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL); | |
1242 | if (!ddev) | |
1243 | return -ENOMEM; | |
1244 | ||
1245 | if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io), | |
1246 | dev_name(&pdev->dev))) | |
1247 | return -EBUSY; | |
1248 | ||
1249 | ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io)); | |
1250 | if (!ddev->regs) | |
1251 | return -ENOMEM; | |
1252 | ddev->have_64bit_regs = pdata->have_64bit_regs; | |
1253 | if (__is_dmac64(ddev)) | |
1254 | ddev->descsize = sizeof(struct txx9dmac_hwdesc); | |
1255 | else | |
1256 | ddev->descsize = sizeof(struct txx9dmac_hwdesc32); | |
1257 | ||
1258 | /* force dma off, just in case */ | |
1259 | txx9dmac_off(ddev); | |
1260 | ||
1261 | ddev->irq = platform_get_irq(pdev, 0); | |
1262 | if (ddev->irq >= 0) { | |
1263 | tasklet_init(&ddev->tasklet, txx9dmac_tasklet, | |
1264 | (unsigned long)ddev); | |
1265 | err = devm_request_irq(&pdev->dev, ddev->irq, | |
1266 | txx9dmac_interrupt, 0, dev_name(&pdev->dev), ddev); | |
1267 | if (err) | |
1268 | return err; | |
1269 | } | |
1270 | ||
1271 | mcr = TXX9_DMA_MCR_MSTEN | MCR_LE; | |
1272 | if (pdata && pdata->memcpy_chan >= 0) | |
1273 | mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan); | |
1274 | dma_writel(ddev, MCR, mcr); | |
1275 | ||
1276 | platform_set_drvdata(pdev, ddev); | |
1277 | return 0; | |
1278 | } | |
1279 | ||
1280 | static int __exit txx9dmac_remove(struct platform_device *pdev) | |
1281 | { | |
1282 | struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); | |
1283 | ||
1284 | txx9dmac_off(ddev); | |
1285 | if (ddev->irq >= 0) | |
1286 | tasklet_kill(&ddev->tasklet); | |
1287 | return 0; | |
1288 | } | |
1289 | ||
1290 | static void txx9dmac_shutdown(struct platform_device *pdev) | |
1291 | { | |
1292 | struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); | |
1293 | ||
1294 | txx9dmac_off(ddev); | |
1295 | } | |
1296 | ||
4aebac2f | 1297 | static int txx9dmac_suspend_noirq(struct device *dev) |
ea76f0b3 | 1298 | { |
4aebac2f | 1299 | struct platform_device *pdev = to_platform_device(dev); |
ea76f0b3 AN |
1300 | struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); |
1301 | ||
1302 | txx9dmac_off(ddev); | |
1303 | return 0; | |
1304 | } | |
1305 | ||
4aebac2f | 1306 | static int txx9dmac_resume_noirq(struct device *dev) |
ea76f0b3 | 1307 | { |
4aebac2f | 1308 | struct platform_device *pdev = to_platform_device(dev); |
ea76f0b3 AN |
1309 | struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); |
1310 | struct txx9dmac_platform_data *pdata = pdev->dev.platform_data; | |
1311 | u32 mcr; | |
1312 | ||
1313 | mcr = TXX9_DMA_MCR_MSTEN | MCR_LE; | |
1314 | if (pdata && pdata->memcpy_chan >= 0) | |
1315 | mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan); | |
1316 | dma_writel(ddev, MCR, mcr); | |
1317 | return 0; | |
1318 | ||
1319 | } | |
1320 | ||
47145210 | 1321 | static const struct dev_pm_ops txx9dmac_dev_pm_ops = { |
4aebac2f MD |
1322 | .suspend_noirq = txx9dmac_suspend_noirq, |
1323 | .resume_noirq = txx9dmac_resume_noirq, | |
1324 | }; | |
1325 | ||
ea76f0b3 AN |
1326 | static struct platform_driver txx9dmac_chan_driver = { |
1327 | .remove = __exit_p(txx9dmac_chan_remove), | |
1328 | .driver = { | |
1329 | .name = "txx9dmac-chan", | |
1330 | }, | |
1331 | }; | |
1332 | ||
1333 | static struct platform_driver txx9dmac_driver = { | |
1334 | .remove = __exit_p(txx9dmac_remove), | |
1335 | .shutdown = txx9dmac_shutdown, | |
ea76f0b3 AN |
1336 | .driver = { |
1337 | .name = "txx9dmac", | |
4aebac2f | 1338 | .pm = &txx9dmac_dev_pm_ops, |
ea76f0b3 AN |
1339 | }, |
1340 | }; | |
1341 | ||
1342 | static int __init txx9dmac_init(void) | |
1343 | { | |
1344 | int rc; | |
1345 | ||
1346 | rc = platform_driver_probe(&txx9dmac_driver, txx9dmac_probe); | |
1347 | if (!rc) { | |
1348 | rc = platform_driver_probe(&txx9dmac_chan_driver, | |
1349 | txx9dmac_chan_probe); | |
1350 | if (rc) | |
1351 | platform_driver_unregister(&txx9dmac_driver); | |
1352 | } | |
1353 | return rc; | |
1354 | } | |
1355 | module_init(txx9dmac_init); | |
1356 | ||
1357 | static void __exit txx9dmac_exit(void) | |
1358 | { | |
1359 | platform_driver_unregister(&txx9dmac_chan_driver); | |
1360 | platform_driver_unregister(&txx9dmac_driver); | |
1361 | } | |
1362 | module_exit(txx9dmac_exit); | |
1363 | ||
1364 | MODULE_LICENSE("GPL"); | |
1365 | MODULE_DESCRIPTION("TXx9 DMA Controller driver"); | |
1366 | MODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>"); | |
b0b4ce38 GU |
1367 | MODULE_ALIAS("platform:txx9dmac"); |
1368 | MODULE_ALIAS("platform:txx9dmac-chan"); |