Commit | Line | Data |
---|---|---|
5fd54ace | 1 | // SPDX-License-Identifier: GPL-2.0 |
550a7375 FB |
2 | /* |
3 | * Copyright (C) 2005-2006 by Texas Instruments | |
4 | * | |
5 | * This file implements a DMA interface using TI's CPPI DMA. | |
6 | * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB. | |
7 | * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci. | |
8 | */ | |
9 | ||
61af9c3f | 10 | #include <linux/module.h> |
91e9c4fe | 11 | #include <linux/platform_device.h> |
5a0e3ad6 | 12 | #include <linux/slab.h> |
550a7375 FB |
13 | #include <linux/usb.h> |
14 | ||
15 | #include "musb_core.h" | |
704a1485 | 16 | #include "musb_debug.h" |
550a7375 | 17 | #include "cppi_dma.h" |
460ddbec | 18 | #include "davinci.h" |
550a7375 FB |
19 | |
20 | ||
21 | /* CPPI DMA status 7-mar-2006: | |
22 | * | |
23 | * - See musb_{host,gadget}.c for more info | |
24 | * | |
25 | * - Correct RX DMA generally forces the engine into irq-per-packet mode, | |
26 | * which can easily saturate the CPU under non-mass-storage loads. | |
27 | * | |
28 | * NOTES 24-aug-2006 (2.6.18-rc4): | |
29 | * | |
30 | * - peripheral RXDMA wedged in a test with packets of length 512/512/1. | |
31 | * evidently after the 1 byte packet was received and acked, the queue | |
32 | * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003, | |
33 | * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401 | |
34 | * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx | |
35 | * of its next (512 byte) packet. IRQ issues? | |
36 | * | |
37 | * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will | |
38 | * evidently also directly update the RX and TX CSRs ... so audit all | |
39 | * host and peripheral side DMA code to avoid CSR access after DMA has | |
40 | * been started. | |
41 | */ | |
42 | ||
43 | /* REVISIT now we can avoid preallocating these descriptors; or | |
44 | * more simply, switch to a global freelist not per-channel ones. | |
45 | * Note: at full speed, 64 descriptors == 4K bulk data. | |
46 | */ | |
47 | #define NUM_TXCHAN_BD 64 | |
48 | #define NUM_RXCHAN_BD 64 | |
49 | ||
50 | static inline void cpu_drain_writebuffer(void) | |
51 | { | |
52 | wmb(); | |
53 | #ifdef CONFIG_CPU_ARM926T | |
54 | /* REVISIT this "should not be needed", | |
55 | * but lack of it sure seemed to hurt ... | |
56 | */ | |
57 | asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n"); | |
58 | #endif | |
59 | } | |
60 | ||
61 | static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c) | |
62 | { | |
63 | struct cppi_descriptor *bd = c->freelist; | |
64 | ||
65 | if (bd) | |
66 | c->freelist = bd->next; | |
67 | return bd; | |
68 | } | |
69 | ||
70 | static inline void | |
71 | cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd) | |
72 | { | |
73 | if (!bd) | |
74 | return; | |
75 | bd->next = c->freelist; | |
76 | c->freelist = bd; | |
77 | } | |
78 | ||
79 | /* | |
80 | * Start DMA controller | |
81 | * | |
82 | * Initialize the DMA controller as necessary. | |
83 | */ | |
84 | ||
85 | /* zero out entire rx state RAM entry for the channel */ | |
86 | static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) | |
87 | { | |
88 | musb_writel(&rx->rx_skipbytes, 0, 0); | |
89 | musb_writel(&rx->rx_head, 0, 0); | |
90 | musb_writel(&rx->rx_sop, 0, 0); | |
91 | musb_writel(&rx->rx_current, 0, 0); | |
92 | musb_writel(&rx->rx_buf_current, 0, 0); | |
93 | musb_writel(&rx->rx_len_len, 0, 0); | |
94 | musb_writel(&rx->rx_cnt_cnt, 0, 0); | |
95 | } | |
96 | ||
97 | /* zero out entire tx state RAM entry for the channel */ | |
98 | static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) | |
99 | { | |
100 | musb_writel(&tx->tx_head, 0, 0); | |
101 | musb_writel(&tx->tx_buf, 0, 0); | |
102 | musb_writel(&tx->tx_current, 0, 0); | |
103 | musb_writel(&tx->tx_buf_current, 0, 0); | |
104 | musb_writel(&tx->tx_info, 0, 0); | |
105 | musb_writel(&tx->tx_rem_len, 0, 0); | |
106 | /* musb_writel(&tx->tx_dummy, 0, 0); */ | |
107 | musb_writel(&tx->tx_complete, 0, ptr); | |
108 | } | |
109 | ||
091a62c9 | 110 | static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c) |
550a7375 FB |
111 | { |
112 | int j; | |
113 | ||
114 | /* initialize channel fields */ | |
115 | c->head = NULL; | |
116 | c->tail = NULL; | |
117 | c->last_processed = NULL; | |
118 | c->channel.status = MUSB_DMA_STATUS_UNKNOWN; | |
119 | c->controller = cppi; | |
120 | c->is_rndis = 0; | |
121 | c->freelist = NULL; | |
122 | ||
123 | /* build the BD Free list for the channel */ | |
124 | for (j = 0; j < NUM_TXCHAN_BD + 1; j++) { | |
125 | struct cppi_descriptor *bd; | |
126 | dma_addr_t dma; | |
127 | ||
128 | bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); | |
129 | bd->dma = dma; | |
130 | cppi_bd_free(c, bd); | |
131 | } | |
132 | } | |
133 | ||
134 | static int cppi_channel_abort(struct dma_channel *); | |
135 | ||
136 | static void cppi_pool_free(struct cppi_channel *c) | |
137 | { | |
138 | struct cppi *cppi = c->controller; | |
139 | struct cppi_descriptor *bd; | |
140 | ||
141 | (void) cppi_channel_abort(&c->channel); | |
142 | c->channel.status = MUSB_DMA_STATUS_UNKNOWN; | |
143 | c->controller = NULL; | |
144 | ||
145 | /* free all its bds */ | |
146 | bd = c->last_processed; | |
147 | do { | |
148 | if (bd) | |
149 | dma_pool_free(cppi->pool, bd, bd->dma); | |
150 | bd = cppi_bd_alloc(c); | |
151 | } while (bd); | |
152 | c->last_processed = NULL; | |
153 | } | |
154 | ||
66c01883 | 155 | static void cppi_controller_start(struct cppi *controller) |
550a7375 | 156 | { |
550a7375 FB |
157 | void __iomem *tibase; |
158 | int i; | |
159 | ||
550a7375 FB |
160 | /* do whatever is necessary to start controller */ |
161 | for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { | |
162 | controller->tx[i].transmit = true; | |
163 | controller->tx[i].index = i; | |
164 | } | |
165 | for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { | |
166 | controller->rx[i].transmit = false; | |
167 | controller->rx[i].index = i; | |
168 | } | |
169 | ||
170 | /* setup BD list on a per channel basis */ | |
171 | for (i = 0; i < ARRAY_SIZE(controller->tx); i++) | |
172 | cppi_pool_init(controller, controller->tx + i); | |
173 | for (i = 0; i < ARRAY_SIZE(controller->rx); i++) | |
174 | cppi_pool_init(controller, controller->rx + i); | |
175 | ||
176 | tibase = controller->tibase; | |
177 | INIT_LIST_HEAD(&controller->tx_complete); | |
178 | ||
179 | /* initialise tx/rx channel head pointers to zero */ | |
180 | for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { | |
181 | struct cppi_channel *tx_ch = controller->tx + i; | |
182 | struct cppi_tx_stateram __iomem *tx; | |
183 | ||
184 | INIT_LIST_HEAD(&tx_ch->tx_complete); | |
185 | ||
186 | tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); | |
187 | tx_ch->state_ram = tx; | |
188 | cppi_reset_tx(tx, 0); | |
189 | } | |
190 | for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { | |
191 | struct cppi_channel *rx_ch = controller->rx + i; | |
192 | struct cppi_rx_stateram __iomem *rx; | |
193 | ||
194 | INIT_LIST_HEAD(&rx_ch->tx_complete); | |
195 | ||
196 | rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); | |
197 | rx_ch->state_ram = rx; | |
198 | cppi_reset_rx(rx); | |
199 | } | |
200 | ||
201 | /* enable individual cppi channels */ | |
202 | musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, | |
203 | DAVINCI_DMA_ALL_CHANNELS_ENABLE); | |
204 | musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG, | |
205 | DAVINCI_DMA_ALL_CHANNELS_ENABLE); | |
206 | ||
207 | /* enable tx/rx CPPI control */ | |
208 | musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); | |
209 | musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); | |
210 | ||
211 | /* disable RNDIS mode, also host rx RNDIS autorequest */ | |
212 | musb_writel(tibase, DAVINCI_RNDIS_REG, 0); | |
213 | musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0); | |
550a7375 FB |
214 | } |
215 | ||
216 | /* | |
217 | * Stop DMA controller | |
218 | * | |
219 | * De-Init the DMA controller as necessary. | |
220 | */ | |
221 | ||
66c01883 | 222 | static void cppi_controller_stop(struct cppi *controller) |
550a7375 | 223 | { |
550a7375 FB |
224 | void __iomem *tibase; |
225 | int i; | |
f847a79a | 226 | struct musb *musb; |
550a7375 | 227 | |
03158f90 | 228 | musb = controller->controller.musb; |
550a7375 FB |
229 | |
230 | tibase = controller->tibase; | |
231 | /* DISABLE INDIVIDUAL CHANNEL Interrupts */ | |
232 | musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, | |
233 | DAVINCI_DMA_ALL_CHANNELS_ENABLE); | |
234 | musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG, | |
235 | DAVINCI_DMA_ALL_CHANNELS_ENABLE); | |
236 | ||
b99d3659 | 237 | musb_dbg(musb, "Tearing down RX and TX Channels"); |
550a7375 FB |
238 | for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { |
239 | /* FIXME restructure of txdma to use bds like rxdma */ | |
240 | controller->tx[i].last_processed = NULL; | |
241 | cppi_pool_free(controller->tx + i); | |
242 | } | |
243 | for (i = 0; i < ARRAY_SIZE(controller->rx); i++) | |
244 | cppi_pool_free(controller->rx + i); | |
245 | ||
246 | /* in Tx Case proper teardown is supported. We resort to disabling | |
247 | * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is | |
248 | * complete TX CPPI cannot be disabled. | |
249 | */ | |
250 | /*disable tx/rx cppi */ | |
251 | musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); | |
252 | musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); | |
550a7375 FB |
253 | } |
254 | ||
255 | /* While dma channel is allocated, we only want the core irqs active | |
256 | * for fault reports, otherwise we'd get irqs that we don't care about. | |
257 | * Except for TX irqs, where dma done != fifo empty and reusable ... | |
258 | * | |
259 | * NOTE: docs don't say either way, but irq masking **enables** irqs. | |
260 | * | |
261 | * REVISIT same issue applies to pure PIO usage too, and non-cppi dma... | |
262 | */ | |
263 | static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum) | |
264 | { | |
265 | musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8)); | |
266 | } | |
267 | ||
268 | static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum) | |
269 | { | |
270 | musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8)); | |
271 | } | |
272 | ||
273 | ||
274 | /* | |
275 | * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to | |
276 | * each transfer direction of a non-control endpoint, so allocating | |
277 | * (and deallocating) is mostly a way to notice bad housekeeping on | |
278 | * the software side. We assume the irqs are always active. | |
279 | */ | |
280 | static struct dma_channel * | |
281 | cppi_channel_allocate(struct dma_controller *c, | |
282 | struct musb_hw_ep *ep, u8 transmit) | |
283 | { | |
284 | struct cppi *controller; | |
285 | u8 index; | |
286 | struct cppi_channel *cppi_ch; | |
287 | void __iomem *tibase; | |
f847a79a | 288 | struct musb *musb; |
550a7375 FB |
289 | |
290 | controller = container_of(c, struct cppi, controller); | |
291 | tibase = controller->tibase; | |
03158f90 | 292 | musb = c->musb; |
550a7375 FB |
293 | |
294 | /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ | |
295 | index = ep->epnum - 1; | |
296 | ||
297 | /* return the corresponding CPPI Channel Handle, and | |
298 | * probably disable the non-CPPI irq until we need it. | |
299 | */ | |
300 | if (transmit) { | |
301 | if (index >= ARRAY_SIZE(controller->tx)) { | |
b99d3659 | 302 | musb_dbg(musb, "no %cX%d CPPI channel", 'T', index); |
550a7375 FB |
303 | return NULL; |
304 | } | |
305 | cppi_ch = controller->tx + index; | |
306 | } else { | |
307 | if (index >= ARRAY_SIZE(controller->rx)) { | |
b99d3659 | 308 | musb_dbg(musb, "no %cX%d CPPI channel", 'R', index); |
550a7375 FB |
309 | return NULL; |
310 | } | |
311 | cppi_ch = controller->rx + index; | |
312 | core_rxirq_disable(tibase, ep->epnum); | |
313 | } | |
314 | ||
315 | /* REVISIT make this an error later once the same driver code works | |
316 | * with the other DMA engine too | |
317 | */ | |
318 | if (cppi_ch->hw_ep) | |
b99d3659 | 319 | musb_dbg(musb, "re-allocating DMA%d %cX channel %p", |
550a7375 FB |
320 | index, transmit ? 'T' : 'R', cppi_ch); |
321 | cppi_ch->hw_ep = ep; | |
322 | cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; | |
66af83dd | 323 | cppi_ch->channel.max_len = 0x7fffffff; |
550a7375 | 324 | |
b99d3659 | 325 | musb_dbg(musb, "Allocate CPPI%d %cX", index, transmit ? 'T' : 'R'); |
550a7375 FB |
326 | return &cppi_ch->channel; |
327 | } | |
328 | ||
329 | /* Release a CPPI Channel. */ | |
330 | static void cppi_channel_release(struct dma_channel *channel) | |
331 | { | |
332 | struct cppi_channel *c; | |
333 | void __iomem *tibase; | |
334 | ||
335 | /* REVISIT: for paranoia, check state and abort if needed... */ | |
336 | ||
337 | c = container_of(channel, struct cppi_channel, channel); | |
338 | tibase = c->controller->tibase; | |
339 | if (!c->hw_ep) | |
03158f90 | 340 | musb_dbg(c->controller->controller.musb, |
b99d3659 | 341 | "releasing idle DMA channel %p", c); |
550a7375 FB |
342 | else if (!c->transmit) |
343 | core_rxirq_enable(tibase, c->index + 1); | |
344 | ||
345 | /* for now, leave its cppi IRQ enabled (we won't trigger it) */ | |
346 | c->hw_ep = NULL; | |
347 | channel->status = MUSB_DMA_STATUS_UNKNOWN; | |
348 | } | |
349 | ||
350 | /* Context: controller irqlocked */ | |
351 | static void | |
352 | cppi_dump_rx(int level, struct cppi_channel *c, const char *tag) | |
353 | { | |
354 | void __iomem *base = c->controller->mregs; | |
355 | struct cppi_rx_stateram __iomem *rx = c->state_ram; | |
356 | ||
357 | musb_ep_select(base, c->index + 1); | |
358 | ||
03158f90 | 359 | musb_dbg(c->controller->controller.musb, |
f847a79a PF |
360 | "RX DMA%d%s: %d left, csr %04x, " |
361 | "%08x H%08x S%08x C%08x, " | |
b99d3659 | 362 | "B%08x L%08x %08x .. %08x", |
550a7375 FB |
363 | c->index, tag, |
364 | musb_readl(c->controller->tibase, | |
365 | DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), | |
366 | musb_readw(c->hw_ep->regs, MUSB_RXCSR), | |
367 | ||
368 | musb_readl(&rx->rx_skipbytes, 0), | |
369 | musb_readl(&rx->rx_head, 0), | |
370 | musb_readl(&rx->rx_sop, 0), | |
371 | musb_readl(&rx->rx_current, 0), | |
372 | ||
373 | musb_readl(&rx->rx_buf_current, 0), | |
374 | musb_readl(&rx->rx_len_len, 0), | |
375 | musb_readl(&rx->rx_cnt_cnt, 0), | |
376 | musb_readl(&rx->rx_complete, 0) | |
377 | ); | |
378 | } | |
379 | ||
380 | /* Context: controller irqlocked */ | |
381 | static void | |
382 | cppi_dump_tx(int level, struct cppi_channel *c, const char *tag) | |
383 | { | |
384 | void __iomem *base = c->controller->mregs; | |
385 | struct cppi_tx_stateram __iomem *tx = c->state_ram; | |
386 | ||
387 | musb_ep_select(base, c->index + 1); | |
388 | ||
03158f90 | 389 | musb_dbg(c->controller->controller.musb, |
f847a79a PF |
390 | "TX DMA%d%s: csr %04x, " |
391 | "H%08x S%08x C%08x %08x, " | |
b99d3659 | 392 | "F%08x L%08x .. %08x", |
550a7375 FB |
393 | c->index, tag, |
394 | musb_readw(c->hw_ep->regs, MUSB_TXCSR), | |
395 | ||
396 | musb_readl(&tx->tx_head, 0), | |
397 | musb_readl(&tx->tx_buf, 0), | |
398 | musb_readl(&tx->tx_current, 0), | |
399 | musb_readl(&tx->tx_buf_current, 0), | |
400 | ||
401 | musb_readl(&tx->tx_info, 0), | |
402 | musb_readl(&tx->tx_rem_len, 0), | |
403 | /* dummy/unused word 6 */ | |
404 | musb_readl(&tx->tx_complete, 0) | |
405 | ); | |
406 | } | |
407 | ||
408 | /* Context: controller irqlocked */ | |
409 | static inline void | |
410 | cppi_rndis_update(struct cppi_channel *c, int is_rx, | |
411 | void __iomem *tibase, int is_rndis) | |
412 | { | |
413 | /* we may need to change the rndis flag for this cppi channel */ | |
414 | if (c->is_rndis != is_rndis) { | |
415 | u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG); | |
416 | u32 temp = 1 << (c->index); | |
417 | ||
418 | if (is_rx) | |
419 | temp <<= 16; | |
420 | if (is_rndis) | |
421 | value |= temp; | |
422 | else | |
423 | value &= ~temp; | |
424 | musb_writel(tibase, DAVINCI_RNDIS_REG, value); | |
425 | c->is_rndis = is_rndis; | |
426 | } | |
427 | } | |
428 | ||
429 | static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd) | |
430 | { | |
431 | pr_debug("RXBD/%s %08x: " | |
432 | "nxt %08x buf %08x off.blen %08x opt.plen %08x\n", | |
433 | tag, bd->dma, | |
434 | bd->hw_next, bd->hw_bufp, bd->hw_off_len, | |
435 | bd->hw_options); | |
436 | } | |
437 | ||
438 | static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) | |
439 | { | |
550a7375 FB |
440 | struct cppi_descriptor *bd; |
441 | ||
550a7375 FB |
442 | cppi_dump_rx(level, rx, tag); |
443 | if (rx->last_processed) | |
444 | cppi_dump_rxbd("last", rx->last_processed); | |
445 | for (bd = rx->head; bd; bd = bd->next) | |
446 | cppi_dump_rxbd("active", bd); | |
550a7375 FB |
447 | } |
448 | ||
449 | ||
450 | /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX; | |
451 | * so we won't ever use it (see "CPPI RX Woes" below). | |
452 | */ | |
453 | static inline int cppi_autoreq_update(struct cppi_channel *rx, | |
454 | void __iomem *tibase, int onepacket, unsigned n_bds) | |
455 | { | |
456 | u32 val; | |
457 | ||
458 | #ifdef RNDIS_RX_IS_USABLE | |
459 | u32 tmp; | |
460 | /* assert(is_host_active(musb)) */ | |
461 | ||
462 | /* start from "AutoReq never" */ | |
463 | tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); | |
464 | val = tmp & ~((0x3) << (rx->index * 2)); | |
465 | ||
466 | /* HCD arranged reqpkt for packet #1. we arrange int | |
467 | * for all but the last one, maybe in two segments. | |
468 | */ | |
469 | if (!onepacket) { | |
470 | #if 0 | |
471 | /* use two segments, autoreq "all" then the last "never" */ | |
472 | val |= ((0x3) << (rx->index * 2)); | |
473 | n_bds--; | |
474 | #else | |
475 | /* one segment, autoreq "all-but-last" */ | |
476 | val |= ((0x1) << (rx->index * 2)); | |
477 | #endif | |
478 | } | |
479 | ||
480 | if (val != tmp) { | |
481 | int n = 100; | |
482 | ||
483 | /* make sure that autoreq is updated before continuing */ | |
484 | musb_writel(tibase, DAVINCI_AUTOREQ_REG, val); | |
485 | do { | |
486 | tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); | |
487 | if (tmp == val) | |
488 | break; | |
489 | cpu_relax(); | |
490 | } while (n-- > 0); | |
491 | } | |
492 | #endif | |
493 | ||
494 | /* REQPKT is turned off after each segment */ | |
495 | if (n_bds && rx->channel.actual_len) { | |
496 | void __iomem *regs = rx->hw_ep->regs; | |
497 | ||
498 | val = musb_readw(regs, MUSB_RXCSR); | |
499 | if (!(val & MUSB_RXCSR_H_REQPKT)) { | |
500 | val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS; | |
501 | musb_writew(regs, MUSB_RXCSR, val); | |
90802ed9 | 502 | /* flush writebuffer */ |
550a7375 FB |
503 | val = musb_readw(regs, MUSB_RXCSR); |
504 | } | |
505 | } | |
506 | return n_bds; | |
507 | } | |
508 | ||
509 | ||
510 | /* Buffer enqueuing Logic: | |
511 | * | |
512 | * - RX builds new queues each time, to help handle routine "early | |
513 | * termination" cases (faults, including errors and short reads) | |
514 | * more correctly. | |
515 | * | |
516 | * - for now, TX reuses the same queue of BDs every time | |
517 | * | |
518 | * REVISIT long term, we want a normal dynamic model. | |
519 | * ... the goal will be to append to the | |
520 | * existing queue, processing completed "dma buffers" (segments) on the fly. | |
521 | * | |
522 | * Otherwise we force an IRQ latency between requests, which slows us a lot | |
523 | * (especially in "transparent" dma). Unfortunately that model seems to be | |
524 | * inherent in the DMA model from the Mentor code, except in the rare case | |
525 | * of transfers big enough (~128+ KB) that we could append "middle" segments | |
526 | * in the TX paths. (RX can't do this, see below.) | |
527 | * | |
528 | * That's true even in the CPPI- friendly iso case, where most urbs have | |
529 | * several small segments provided in a group and where the "packet at a time" | |
530 | * "transparent" DMA model is always correct, even on the RX side. | |
531 | */ | |
532 | ||
533 | /* | |
534 | * CPPI TX: | |
535 | * ======== | |
536 | * TX is a lot more reasonable than RX; it doesn't need to run in | |
537 | * irq-per-packet mode very often. RNDIS mode seems to behave too | |
538 | * (except how it handles the exactly-N-packets case). Building a | |
539 | * txdma queue with multiple requests (urb or usb_request) looks | |
540 | * like it would work ... but fault handling would need much testing. | |
541 | * | |
542 | * The main issue with TX mode RNDIS relates to transfer lengths that | |
543 | * are an exact multiple of the packet length. It appears that there's | |
544 | * a hiccup in that case (maybe the DMA completes before the ZLP gets | |
545 | * written?) boiling down to not being able to rely on CPPI writing any | |
546 | * terminating zero length packet before the next transfer is written. | |
547 | * So that's punted to PIO; better yet, gadget drivers can avoid it. | |
548 | * | |
549 | * Plus, there's allegedly an undocumented constraint that rndis transfer | |
550 | * length be a multiple of 64 bytes ... but the chip doesn't act that | |
551 | * way, and we really don't _want_ that behavior anyway. | |
552 | * | |
553 | * On TX, "transparent" mode works ... although experiments have shown | |
554 | * problems trying to use the SOP/EOP bits in different USB packets. | |
555 | * | |
556 | * REVISIT try to handle terminating zero length packets using CPPI | |
557 | * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet | |
558 | * links avoid that issue by forcing them to avoid zlps.) | |
559 | */ | |
560 | static void | |
561 | cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) | |
562 | { | |
563 | unsigned maxpacket = tx->maxpacket; | |
564 | dma_addr_t addr = tx->buf_dma + tx->offset; | |
565 | size_t length = tx->buf_len - tx->offset; | |
566 | struct cppi_descriptor *bd; | |
567 | unsigned n_bds; | |
568 | unsigned i; | |
569 | struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; | |
570 | int rndis; | |
571 | ||
572 | /* TX can use the CPPI "rndis" mode, where we can probably fit this | |
573 | * transfer in one BD and one IRQ. The only time we would NOT want | |
574 | * to use it is when hardware constraints prevent it, or if we'd | |
575 | * trigger the "send a ZLP?" confusion. | |
576 | */ | |
577 | rndis = (maxpacket & 0x3f) == 0 | |
6b6e9710 | 578 | && length > maxpacket |
550a7375 FB |
579 | && length < 0xffff |
580 | && (length % maxpacket) != 0; | |
581 | ||
582 | if (rndis) { | |
583 | maxpacket = length; | |
584 | n_bds = 1; | |
585 | } else { | |
f5332b00 IS |
586 | if (length) |
587 | n_bds = DIV_ROUND_UP(length, maxpacket); | |
588 | else | |
589 | n_bds = 1; | |
550a7375 FB |
590 | n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD); |
591 | length = min(n_bds * maxpacket, length); | |
592 | } | |
593 | ||
b99d3659 | 594 | musb_dbg(musb, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u", |
550a7375 FB |
595 | tx->index, |
596 | maxpacket, | |
597 | rndis ? "rndis" : "transparent", | |
598 | n_bds, | |
2fbcf3fa | 599 | (unsigned long long)addr, length); |
550a7375 FB |
600 | |
601 | cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); | |
602 | ||
603 | /* assuming here that channel_program is called during | |
604 | * transfer initiation ... current code maintains state | |
605 | * for one outstanding request only (no queues, not even | |
606 | * the implicit ones of an iso urb). | |
607 | */ | |
608 | ||
609 | bd = tx->freelist; | |
610 | tx->head = bd; | |
611 | tx->last_processed = NULL; | |
612 | ||
613 | /* FIXME use BD pool like RX side does, and just queue | |
614 | * the minimum number for this request. | |
615 | */ | |
616 | ||
617 | /* Prepare queue of BDs first, then hand it to hardware. | |
618 | * All BDs except maybe the last should be of full packet | |
619 | * size; for RNDIS there _is_ only that last packet. | |
620 | */ | |
621 | for (i = 0; i < n_bds; ) { | |
622 | if (++i < n_bds && bd->next) | |
623 | bd->hw_next = bd->next->dma; | |
624 | else | |
625 | bd->hw_next = 0; | |
626 | ||
627 | bd->hw_bufp = tx->buf_dma + tx->offset; | |
628 | ||
629 | /* FIXME set EOP only on the last packet, | |
630 | * SOP only on the first ... avoid IRQs | |
631 | */ | |
632 | if ((tx->offset + maxpacket) <= tx->buf_len) { | |
633 | tx->offset += maxpacket; | |
634 | bd->hw_off_len = maxpacket; | |
635 | bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET | |
636 | | CPPI_OWN_SET | maxpacket; | |
637 | } else { | |
638 | /* only this one may be a partial USB Packet */ | |
639 | u32 partial_len; | |
640 | ||
641 | partial_len = tx->buf_len - tx->offset; | |
642 | tx->offset = tx->buf_len; | |
643 | bd->hw_off_len = partial_len; | |
644 | ||
645 | bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET | |
646 | | CPPI_OWN_SET | partial_len; | |
647 | if (partial_len == 0) | |
648 | bd->hw_options |= CPPI_ZERO_SET; | |
649 | } | |
650 | ||
b99d3659 | 651 | musb_dbg(musb, "TXBD %p: nxt %08x buf %08x len %04x opt %08x", |
550a7375 FB |
652 | bd, bd->hw_next, bd->hw_bufp, |
653 | bd->hw_off_len, bd->hw_options); | |
654 | ||
655 | /* update the last BD enqueued to the list */ | |
656 | tx->tail = bd; | |
657 | bd = bd->next; | |
658 | } | |
659 | ||
660 | /* BDs live in DMA-coherent memory, but writes might be pending */ | |
661 | cpu_drain_writebuffer(); | |
662 | ||
663 | /* Write to the HeadPtr in state RAM to trigger */ | |
664 | musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); | |
665 | ||
666 | cppi_dump_tx(5, tx, "/S"); | |
667 | } | |
668 | ||
669 | /* | |
670 | * CPPI RX Woes: | |
671 | * ============= | |
672 | * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte | |
673 | * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back. | |
674 | * (Full speed transfers have similar scenarios.) | |
675 | * | |
676 | * The correct behavior for Linux is that (a) fills the buffer with 300 bytes, | |
677 | * and the next packet goes into a buffer that's queued later; while (b) fills | |
678 | * the buffer with 1024 bytes. How to do that with CPPI? | |
679 | * | |
680 | * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but | |
681 | * (b) loses **BADLY** because nothing (!) happens when that second packet | |
682 | * fills the buffer, much less when a third one arrives. (Which makes this | |
683 | * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination | |
684 | * is optional, and it's fine if peripherals -- not hosts! -- pad messages | |
685 | * out to end-of-buffer. Standard PCI host controller DMA descriptors | |
686 | * implement that mode by default ... which is no accident.) | |
687 | * | |
688 | * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have | |
689 | * converse problems: (b) is handled right, but (a) loses badly. CPPI RX | |
690 | * ignores SOP/EOP markings and processes both of those BDs; so both packets | |
691 | * are loaded into the buffer (with a 212 byte gap between them), and the next | |
692 | * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP | |
693 | * are intended as outputs for RX queues, not inputs...) | |
694 | * | |
695 | * - A variant of "transparent" mode -- one BD at a time -- is the only way to | |
696 | * reliably make both cases work, with software handling both cases correctly | |
697 | * and at the significant penalty of needing an IRQ per packet. (The lack of | |
698 | * I/O overlap can be slightly ameliorated by enabling double buffering.) | |
699 | * | |
700 | * So how to get rid of IRQ-per-packet? The transparent multi-BD case could | |
701 | * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK | |
702 | * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors | |
703 | * with guaranteed driver level fault recovery and scrubbing out what's left | |
704 | * of that garbaged datastream. | |
705 | * | |
706 | * But there seems to be no way to identify the cases where CPPI RNDIS mode | |
707 | * is appropriate -- which do NOT include RNDIS host drivers, but do include | |
708 | * the CDC Ethernet driver! -- and the documentation is incomplete/wrong. | |
709 | * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic | |
710 | * that applies best on the peripheral side (and which could fail rudely). | |
711 | * | |
712 | * Leaving only "transparent" mode; we avoid multi-bd modes in almost all | |
713 | * cases other than mass storage class. Otherwise we're correct but slow, | |
714 | * since CPPI penalizes our need for a "true RNDIS" default mode. | |
715 | */ | |
716 | ||
717 | ||
718 | /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY | |
719 | * | |
720 | * IFF | |
721 | * (a) peripheral mode ... since rndis peripherals could pad their | |
722 | * writes to hosts, causing i/o failure; or we'd have to cope with | |
723 | * a largely unknowable variety of host side protocol variants | |
724 | * (b) and short reads are NOT errors ... since full reads would | |
725 | * cause those same i/o failures | |
726 | * (c) and read length is | |
727 | * - less than 64KB (max per cppi descriptor) | |
728 | * - not a multiple of 4096 (g_zero default, full reads typical) | |
729 | * - N (>1) packets long, ditto (full reads not EXPECTED) | |
730 | * THEN | |
731 | * try rx rndis mode | |
732 | * | |
733 | * Cost of heuristic failing: RXDMA wedges at the end of transfers that | |
734 | * fill out the whole buffer. Buggy host side usb network drivers could | |
735 | * trigger that, but "in the field" such bugs seem to be all but unknown. | |
736 | * | |
737 | * So this module parameter lets the heuristic be disabled. When using | |
738 | * gadgetfs, the heuristic will probably need to be disabled. | |
739 | */ | |
90ab5ee9 | 740 | static bool cppi_rx_rndis = 1; |
550a7375 FB |
741 | |
742 | module_param(cppi_rx_rndis, bool, 0); | |
743 | MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic"); | |
744 | ||
745 | ||
746 | /** | |
747 | * cppi_next_rx_segment - dma read for the next chunk of a buffer | |
748 | * @musb: the controller | |
749 | * @rx: dma channel | |
750 | * @onepacket: true unless caller treats short reads as errors, and | |
751 | * performs fault recovery above usbcore. | |
752 | * Context: controller irqlocked | |
753 | * | |
754 | * See above notes about why we can't use multi-BD RX queues except in | |
755 | * rare cases (mass storage class), and can never use the hardware "rndis" | |
756 | * mode (since it's not a "true" RNDIS mode) with complete safety.. | |
757 | * | |
758 | * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in | |
759 | * code to recover from corrupted datastreams after each short transfer. | |
760 | */ | |
761 | static void | |
762 | cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) | |
763 | { | |
764 | unsigned maxpacket = rx->maxpacket; | |
765 | dma_addr_t addr = rx->buf_dma + rx->offset; | |
766 | size_t length = rx->buf_len - rx->offset; | |
767 | struct cppi_descriptor *bd, *tail; | |
768 | unsigned n_bds; | |
769 | unsigned i; | |
770 | void __iomem *tibase = musb->ctrl_base; | |
771 | int is_rndis = 0; | |
772 | struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; | |
2daf5966 | 773 | struct cppi_descriptor *d; |
550a7375 FB |
774 | |
775 | if (onepacket) { | |
776 | /* almost every USB driver, host or peripheral side */ | |
777 | n_bds = 1; | |
778 | ||
779 | /* maybe apply the heuristic above */ | |
780 | if (cppi_rx_rndis | |
781 | && is_peripheral_active(musb) | |
782 | && length > maxpacket | |
783 | && (length & ~0xffff) == 0 | |
784 | && (length & 0x0fff) != 0 | |
785 | && (length & (maxpacket - 1)) == 0) { | |
786 | maxpacket = length; | |
787 | is_rndis = 1; | |
788 | } | |
789 | } else { | |
790 | /* virtually nothing except mass storage class */ | |
791 | if (length > 0xffff) { | |
792 | n_bds = 0xffff / maxpacket; | |
793 | length = n_bds * maxpacket; | |
794 | } else { | |
f5332b00 | 795 | n_bds = DIV_ROUND_UP(length, maxpacket); |
550a7375 FB |
796 | } |
797 | if (n_bds == 1) | |
798 | onepacket = 1; | |
799 | else | |
800 | n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); | |
801 | } | |
802 | ||
803 | /* In host mode, autorequest logic can generate some IN tokens; it's | |
804 | * tricky since we can't leave REQPKT set in RXCSR after the transfer | |
805 | * finishes. So: multipacket transfers involve two or more segments. | |
806 | * And always at least two IRQs ... RNDIS mode is not an option. | |
807 | */ | |
808 | if (is_host_active(musb)) | |
809 | n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); | |
810 | ||
811 | cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); | |
812 | ||
813 | length = min(n_bds * maxpacket, length); | |
814 | ||
b99d3659 BL |
815 | musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " |
816 | "dma 0x%llx len %u %u/%u", | |
550a7375 FB |
817 | rx->index, maxpacket, |
818 | onepacket | |
819 | ? (is_rndis ? "rndis" : "onepacket") | |
820 | : "multipacket", | |
821 | n_bds, | |
822 | musb_readl(tibase, | |
823 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) | |
824 | & 0xffff, | |
2fbcf3fa DC |
825 | (unsigned long long)addr, length, |
826 | rx->channel.actual_len, rx->buf_len); | |
550a7375 FB |
827 | |
828 | /* only queue one segment at a time, since the hardware prevents | |
829 | * correct queue shutdown after unexpected short packets | |
830 | */ | |
831 | bd = cppi_bd_alloc(rx); | |
832 | rx->head = bd; | |
833 | ||
834 | /* Build BDs for all packets in this segment */ | |
835 | for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { | |
836 | u32 bd_len; | |
837 | ||
838 | if (i) { | |
839 | bd = cppi_bd_alloc(rx); | |
840 | if (!bd) | |
841 | break; | |
842 | tail->next = bd; | |
843 | tail->hw_next = bd->dma; | |
844 | } | |
845 | bd->hw_next = 0; | |
846 | ||
847 | /* all but the last packet will be maxpacket size */ | |
848 | if (maxpacket < length) | |
849 | bd_len = maxpacket; | |
850 | else | |
851 | bd_len = length; | |
852 | ||
853 | bd->hw_bufp = addr; | |
854 | addr += bd_len; | |
855 | rx->offset += bd_len; | |
856 | ||
857 | bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; | |
858 | bd->buflen = bd_len; | |
859 | ||
860 | bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); | |
861 | length -= bd_len; | |
862 | } | |
863 | ||
864 | /* we always expect at least one reusable BD! */ | |
865 | if (!tail) { | |
866 | WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); | |
867 | return; | |
868 | } else if (i < n_bds) | |
869 | WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); | |
870 | ||
871 | tail->next = NULL; | |
872 | tail->hw_next = 0; | |
873 | ||
874 | bd = rx->head; | |
875 | rx->tail = tail; | |
876 | ||
877 | /* short reads and other faults should terminate this entire | |
878 | * dma segment. we want one "dma packet" per dma segment, not | |
879 | * one per USB packet, terminating the whole queue at once... | |
880 | * NOTE that current hardware seems to ignore SOP and EOP. | |
881 | */ | |
882 | bd->hw_options |= CPPI_SOP_SET; | |
883 | tail->hw_options |= CPPI_EOP_SET; | |
884 | ||
2daf5966 FB |
885 | for (d = rx->head; d; d = d->next) |
886 | cppi_dump_rxbd("S", d); | |
550a7375 FB |
887 | |
888 | /* in case the preceding transfer left some state... */ | |
889 | tail = rx->last_processed; | |
890 | if (tail) { | |
891 | tail->next = bd; | |
892 | tail->hw_next = bd->dma; | |
893 | } | |
894 | ||
895 | core_rxirq_enable(tibase, rx->index + 1); | |
896 | ||
897 | /* BDs live in DMA-coherent memory, but writes might be pending */ | |
898 | cpu_drain_writebuffer(); | |
899 | ||
900 | /* REVISIT specs say to write this AFTER the BUFCNT register | |
901 | * below ... but that loses badly. | |
902 | */ | |
903 | musb_writel(&rx_ram->rx_head, 0, bd->dma); | |
904 | ||
905 | /* bufferCount must be at least 3, and zeroes on completion | |
906 | * unless it underflows below zero, or stops at two, or keeps | |
907 | * growing ... grr. | |
908 | */ | |
909 | i = musb_readl(tibase, | |
910 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) | |
911 | & 0xffff; | |
912 | ||
913 | if (!i) | |
914 | musb_writel(tibase, | |
915 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), | |
916 | n_bds + 2); | |
917 | else if (n_bds > (i - 3)) | |
918 | musb_writel(tibase, | |
919 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), | |
920 | n_bds - (i - 3)); | |
921 | ||
922 | i = musb_readl(tibase, | |
923 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) | |
924 | & 0xffff; | |
925 | if (i < (2 + n_bds)) { | |
b99d3659 | 926 | musb_dbg(musb, "bufcnt%d underrun - %d (for %d)", |
550a7375 FB |
927 | rx->index, i, n_bds); |
928 | musb_writel(tibase, | |
929 | DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), | |
930 | n_bds + 2); | |
931 | } | |
932 | ||
933 | cppi_dump_rx(4, rx, "/S"); | |
934 | } | |
935 | ||
936 | /** | |
937 | * cppi_channel_program - program channel for data transfer | |
938 | * @ch: the channel | |
939 | * @maxpacket: max packet size | |
940 | * @mode: For RX, 1 unless the usb protocol driver promised to treat | |
941 | * all short reads as errors and kick in high level fault recovery. | |
942 | * For TX, ignored because of RNDIS mode races/glitches. | |
943 | * @dma_addr: dma address of buffer | |
944 | * @len: length of buffer | |
945 | * Context: controller irqlocked | |
946 | */ | |
947 | static int cppi_channel_program(struct dma_channel *ch, | |
948 | u16 maxpacket, u8 mode, | |
949 | dma_addr_t dma_addr, u32 len) | |
950 | { | |
951 | struct cppi_channel *cppi_ch; | |
952 | struct cppi *controller; | |
953 | struct musb *musb; | |
954 | ||
955 | cppi_ch = container_of(ch, struct cppi_channel, channel); | |
956 | controller = cppi_ch->controller; | |
03158f90 | 957 | musb = controller->controller.musb; |
550a7375 FB |
958 | |
959 | switch (ch->status) { | |
960 | case MUSB_DMA_STATUS_BUS_ABORT: | |
961 | case MUSB_DMA_STATUS_CORE_ABORT: | |
962 | /* fault irq handler should have handled cleanup */ | |
963 | WARNING("%cX DMA%d not cleaned up after abort!\n", | |
964 | cppi_ch->transmit ? 'T' : 'R', | |
965 | cppi_ch->index); | |
966 | /* WARN_ON(1); */ | |
967 | break; | |
968 | case MUSB_DMA_STATUS_BUSY: | |
969 | WARNING("program active channel? %cX DMA%d\n", | |
970 | cppi_ch->transmit ? 'T' : 'R', | |
971 | cppi_ch->index); | |
972 | /* WARN_ON(1); */ | |
973 | break; | |
974 | case MUSB_DMA_STATUS_UNKNOWN: | |
b99d3659 | 975 | musb_dbg(musb, "%cX DMA%d not allocated!", |
550a7375 FB |
976 | cppi_ch->transmit ? 'T' : 'R', |
977 | cppi_ch->index); | |
978 | /* FALLTHROUGH */ | |
979 | case MUSB_DMA_STATUS_FREE: | |
980 | break; | |
981 | } | |
982 | ||
983 | ch->status = MUSB_DMA_STATUS_BUSY; | |
984 | ||
985 | /* set transfer parameters, then queue up its first segment */ | |
986 | cppi_ch->buf_dma = dma_addr; | |
987 | cppi_ch->offset = 0; | |
988 | cppi_ch->maxpacket = maxpacket; | |
989 | cppi_ch->buf_len = len; | |
191b7766 | 990 | cppi_ch->channel.actual_len = 0; |
550a7375 FB |
991 | |
992 | /* TX channel? or RX? */ | |
993 | if (cppi_ch->transmit) | |
994 | cppi_next_tx_segment(musb, cppi_ch); | |
995 | else | |
996 | cppi_next_rx_segment(musb, cppi_ch, mode); | |
997 | ||
998 | return true; | |
999 | } | |
1000 | ||
1001 | static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) | |
1002 | { | |
1003 | struct cppi_channel *rx = &cppi->rx[ch]; | |
1004 | struct cppi_rx_stateram __iomem *state = rx->state_ram; | |
1005 | struct cppi_descriptor *bd; | |
1006 | struct cppi_descriptor *last = rx->last_processed; | |
1007 | bool completed = false; | |
1008 | bool acked = false; | |
1009 | int i; | |
1010 | dma_addr_t safe2ack; | |
1011 | void __iomem *regs = rx->hw_ep->regs; | |
03158f90 | 1012 | struct musb *musb = cppi->controller.musb; |
550a7375 FB |
1013 | |
1014 | cppi_dump_rx(6, rx, "/K"); | |
1015 | ||
1016 | bd = last ? last->next : rx->head; | |
1017 | if (!bd) | |
1018 | return false; | |
1019 | ||
1020 | /* run through all completed BDs */ | |
1021 | for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); | |
1022 | (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; | |
1023 | i++, bd = bd->next) { | |
1024 | u16 len; | |
1025 | ||
1026 | /* catch latest BD writes from CPPI */ | |
1027 | rmb(); | |
1028 | if (!completed && (bd->hw_options & CPPI_OWN_SET)) | |
1029 | break; | |
1030 | ||
b99d3659 BL |
1031 | musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x " |
1032 | "off.len %08x opt.len %08x (%d)", | |
2fbcf3fa | 1033 | (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, |
550a7375 FB |
1034 | bd->hw_off_len, bd->hw_options, |
1035 | rx->channel.actual_len); | |
1036 | ||
1037 | /* actual packet received length */ | |
1038 | if ((bd->hw_options & CPPI_SOP_SET) && !completed) | |
1039 | len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; | |
1040 | else | |
1041 | len = 0; | |
1042 | ||
1043 | if (bd->hw_options & CPPI_EOQ_MASK) | |
1044 | completed = true; | |
1045 | ||
1046 | if (!completed && len < bd->buflen) { | |
1047 | /* NOTE: when we get a short packet, RXCSR_H_REQPKT | |
1048 | * must have been cleared, and no more DMA packets may | |
1049 | * active be in the queue... TI docs didn't say, but | |
1050 | * CPPI ignores those BDs even though OWN is still set. | |
1051 | */ | |
1052 | completed = true; | |
b99d3659 | 1053 | musb_dbg(musb, "rx short %d/%d (%d)", |
550a7375 FB |
1054 | len, bd->buflen, |
1055 | rx->channel.actual_len); | |
1056 | } | |
1057 | ||
1058 | /* If we got here, we expect to ack at least one BD; meanwhile | |
1059 | * CPPI may completing other BDs while we scan this list... | |
1060 | * | |
1061 | * RACE: we can notice OWN cleared before CPPI raises the | |
1062 | * matching irq by writing that BD as the completion pointer. | |
1063 | * In such cases, stop scanning and wait for the irq, avoiding | |
1064 | * lost acks and states where BD ownership is unclear. | |
1065 | */ | |
1066 | if (bd->dma == safe2ack) { | |
1067 | musb_writel(&state->rx_complete, 0, safe2ack); | |
1068 | safe2ack = musb_readl(&state->rx_complete, 0); | |
1069 | acked = true; | |
1070 | if (bd->dma == safe2ack) | |
1071 | safe2ack = 0; | |
1072 | } | |
1073 | ||
1074 | rx->channel.actual_len += len; | |
1075 | ||
1076 | cppi_bd_free(rx, last); | |
1077 | last = bd; | |
1078 | ||
1079 | /* stop scanning on end-of-segment */ | |
1080 | if (bd->hw_next == 0) | |
1081 | completed = true; | |
1082 | } | |
1083 | rx->last_processed = last; | |
1084 | ||
1085 | /* dma abort, lost ack, or ... */ | |
1086 | if (!acked && last) { | |
1087 | int csr; | |
1088 | ||
1089 | if (safe2ack == 0 || safe2ack == rx->last_processed->dma) | |
1090 | musb_writel(&state->rx_complete, 0, safe2ack); | |
1091 | if (safe2ack == 0) { | |
1092 | cppi_bd_free(rx, last); | |
1093 | rx->last_processed = NULL; | |
1094 | ||
1095 | /* if we land here on the host side, H_REQPKT will | |
1096 | * be clear and we need to restart the queue... | |
1097 | */ | |
1098 | WARN_ON(rx->head); | |
1099 | } | |
1100 | musb_ep_select(cppi->mregs, rx->index + 1); | |
1101 | csr = musb_readw(regs, MUSB_RXCSR); | |
1102 | if (csr & MUSB_RXCSR_DMAENAB) { | |
b99d3659 | 1103 | musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x", |
550a7375 FB |
1104 | rx->index, |
1105 | rx->head, rx->tail, | |
1106 | rx->last_processed | |
2fbcf3fa DC |
1107 | ? (unsigned long long) |
1108 | rx->last_processed->dma | |
550a7375 FB |
1109 | : 0, |
1110 | completed ? ", completed" : "", | |
1111 | csr); | |
1112 | cppi_dump_rxq(4, "/what?", rx); | |
1113 | } | |
1114 | } | |
1115 | if (!completed) { | |
1116 | int csr; | |
1117 | ||
1118 | rx->head = bd; | |
1119 | ||
1120 | /* REVISIT seems like "autoreq all but EOP" doesn't... | |
1121 | * setting it here "should" be racey, but seems to work | |
1122 | */ | |
1123 | csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); | |
03158f90 | 1124 | if (is_host_active(cppi->controller.musb) |
550a7375 FB |
1125 | && bd |
1126 | && !(csr & MUSB_RXCSR_H_REQPKT)) { | |
1127 | csr |= MUSB_RXCSR_H_REQPKT; | |
1128 | musb_writew(regs, MUSB_RXCSR, | |
1129 | MUSB_RXCSR_H_WZC_BITS | csr); | |
1130 | csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); | |
1131 | } | |
1132 | } else { | |
1133 | rx->head = NULL; | |
1134 | rx->tail = NULL; | |
1135 | } | |
1136 | ||
1137 | cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); | |
1138 | return completed; | |
1139 | } | |
1140 | ||
91e9c4fe | 1141 | irqreturn_t cppi_interrupt(int irq, void *dev_id) |
550a7375 | 1142 | { |
91e9c4fe | 1143 | struct musb *musb = dev_id; |
550a7375 | 1144 | struct cppi *cppi; |
91e9c4fe | 1145 | void __iomem *tibase; |
550a7375 | 1146 | struct musb_hw_ep *hw_ep = NULL; |
91e9c4fe SS |
1147 | u32 rx, tx; |
1148 | int i, index; | |
2f8d5cd6 | 1149 | unsigned long uninitialized_var(flags); |
550a7375 FB |
1150 | |
1151 | cppi = container_of(musb->dma_controller, struct cppi, controller); | |
93aa3dab S |
1152 | if (cppi->irq) |
1153 | spin_lock_irqsave(&musb->lock, flags); | |
550a7375 FB |
1154 | |
1155 | tibase = musb->ctrl_base; | |
1156 | ||
91e9c4fe SS |
1157 | tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); |
1158 | rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); | |
1159 | ||
ec63bf6c DC |
1160 | if (!tx && !rx) { |
1161 | if (cppi->irq) | |
1162 | spin_unlock_irqrestore(&musb->lock, flags); | |
91e9c4fe | 1163 | return IRQ_NONE; |
ec63bf6c | 1164 | } |
91e9c4fe | 1165 | |
b99d3659 | 1166 | musb_dbg(musb, "CPPI IRQ Tx%x Rx%x", tx, rx); |
91e9c4fe | 1167 | |
550a7375 FB |
1168 | /* process TX channels */ |
1169 | for (index = 0; tx; tx = tx >> 1, index++) { | |
1170 | struct cppi_channel *tx_ch; | |
1171 | struct cppi_tx_stateram __iomem *tx_ram; | |
1172 | bool completed = false; | |
1173 | struct cppi_descriptor *bd; | |
1174 | ||
1175 | if (!(tx & 1)) | |
1176 | continue; | |
1177 | ||
1178 | tx_ch = cppi->tx + index; | |
1179 | tx_ram = tx_ch->state_ram; | |
1180 | ||
1181 | /* FIXME need a cppi_tx_scan() routine, which | |
1182 | * can also be called from abort code | |
1183 | */ | |
1184 | ||
1185 | cppi_dump_tx(5, tx_ch, "/E"); | |
1186 | ||
1187 | bd = tx_ch->head; | |
1188 | ||
56596923 S |
1189 | /* |
1190 | * If Head is null then this could mean that a abort interrupt | |
1191 | * that needs to be acknowledged. | |
1192 | */ | |
550a7375 | 1193 | if (NULL == bd) { |
b99d3659 | 1194 | musb_dbg(musb, "null BD"); |
2e10f5e7 | 1195 | musb_writel(&tx_ram->tx_complete, 0, 0); |
550a7375 FB |
1196 | continue; |
1197 | } | |
1198 | ||
1199 | /* run through all completed BDs */ | |
1200 | for (i = 0; !completed && bd && i < NUM_TXCHAN_BD; | |
1201 | i++, bd = bd->next) { | |
1202 | u16 len; | |
1203 | ||
1204 | /* catch latest BD writes from CPPI */ | |
1205 | rmb(); | |
1206 | if (bd->hw_options & CPPI_OWN_SET) | |
1207 | break; | |
1208 | ||
b99d3659 | 1209 | musb_dbg(musb, "C/TXBD %p n %x b %x off %x opt %x", |
550a7375 FB |
1210 | bd, bd->hw_next, bd->hw_bufp, |
1211 | bd->hw_off_len, bd->hw_options); | |
1212 | ||
1213 | len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; | |
1214 | tx_ch->channel.actual_len += len; | |
1215 | ||
1216 | tx_ch->last_processed = bd; | |
1217 | ||
1218 | /* write completion register to acknowledge | |
1219 | * processing of completed BDs, and possibly | |
1220 | * release the IRQ; EOQ might not be set ... | |
1221 | * | |
1222 | * REVISIT use the same ack strategy as rx | |
1223 | * | |
1224 | * REVISIT have observed bit 18 set; huh?? | |
1225 | */ | |
1226 | /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ | |
1227 | musb_writel(&tx_ram->tx_complete, 0, bd->dma); | |
1228 | ||
1229 | /* stop scanning on end-of-segment */ | |
1230 | if (bd->hw_next == 0) | |
1231 | completed = true; | |
1232 | } | |
1233 | ||
1234 | /* on end of segment, maybe go to next one */ | |
1235 | if (completed) { | |
1236 | /* cppi_dump_tx(4, tx_ch, "/complete"); */ | |
1237 | ||
1238 | /* transfer more, or report completion */ | |
1239 | if (tx_ch->offset >= tx_ch->buf_len) { | |
1240 | tx_ch->head = NULL; | |
1241 | tx_ch->tail = NULL; | |
1242 | tx_ch->channel.status = MUSB_DMA_STATUS_FREE; | |
1243 | ||
1244 | hw_ep = tx_ch->hw_ep; | |
1245 | ||
c7bbc056 | 1246 | musb_dma_completion(musb, index + 1, 1); |
550a7375 FB |
1247 | |
1248 | } else { | |
1249 | /* Bigger transfer than we could fit in | |
1250 | * that first batch of descriptors... | |
1251 | */ | |
1252 | cppi_next_tx_segment(musb, tx_ch); | |
1253 | } | |
1254 | } else | |
1255 | tx_ch->head = bd; | |
1256 | } | |
1257 | ||
1258 | /* Start processing the RX block */ | |
1259 | for (index = 0; rx; rx = rx >> 1, index++) { | |
1260 | ||
1261 | if (rx & 1) { | |
1262 | struct cppi_channel *rx_ch; | |
1263 | ||
1264 | rx_ch = cppi->rx + index; | |
1265 | ||
1266 | /* let incomplete dma segments finish */ | |
1267 | if (!cppi_rx_scan(cppi, index)) | |
1268 | continue; | |
1269 | ||
1270 | /* start another dma segment if needed */ | |
1271 | if (rx_ch->channel.actual_len != rx_ch->buf_len | |
1272 | && rx_ch->channel.actual_len | |
1273 | == rx_ch->offset) { | |
1274 | cppi_next_rx_segment(musb, rx_ch, 1); | |
1275 | continue; | |
1276 | } | |
1277 | ||
1278 | /* all segments completed! */ | |
1279 | rx_ch->channel.status = MUSB_DMA_STATUS_FREE; | |
1280 | ||
1281 | hw_ep = rx_ch->hw_ep; | |
1282 | ||
1283 | core_rxirq_disable(tibase, index + 1); | |
1284 | musb_dma_completion(musb, index + 1, 0); | |
1285 | } | |
1286 | } | |
1287 | ||
1288 | /* write to CPPI EOI register to re-enable interrupts */ | |
1289 | musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0); | |
91e9c4fe | 1290 | |
93aa3dab S |
1291 | if (cppi->irq) |
1292 | spin_unlock_irqrestore(&musb->lock, flags); | |
1293 | ||
91e9c4fe | 1294 | return IRQ_HANDLED; |
550a7375 | 1295 | } |
8b416b0b | 1296 | EXPORT_SYMBOL_GPL(cppi_interrupt); |
550a7375 FB |
1297 | |
1298 | /* Instantiate a software object representing a DMA controller. */ | |
7f6283ed TL |
1299 | struct dma_controller * |
1300 | cppi_dma_controller_create(struct musb *musb, void __iomem *mregs) | |
550a7375 FB |
1301 | { |
1302 | struct cppi *controller; | |
91e9c4fe SS |
1303 | struct device *dev = musb->controller; |
1304 | struct platform_device *pdev = to_platform_device(dev); | |
fcf173e4 | 1305 | int irq = platform_get_irq_byname(pdev, "dma"); |
550a7375 FB |
1306 | |
1307 | controller = kzalloc(sizeof *controller, GFP_KERNEL); | |
1308 | if (!controller) | |
1309 | return NULL; | |
1310 | ||
1311 | controller->mregs = mregs; | |
1312 | controller->tibase = mregs - DAVINCI_BASE_OFFSET; | |
1313 | ||
03158f90 | 1314 | controller->controller.musb = musb; |
550a7375 FB |
1315 | controller->controller.channel_alloc = cppi_channel_allocate; |
1316 | controller->controller.channel_release = cppi_channel_release; | |
1317 | controller->controller.channel_program = cppi_channel_program; | |
1318 | controller->controller.channel_abort = cppi_channel_abort; | |
1319 | ||
1320 | /* NOTE: allocating from on-chip SRAM would give the least | |
1321 | * contention for memory access, if that ever matters here. | |
1322 | */ | |
1323 | ||
1324 | /* setup BufferPool */ | |
1325 | controller->pool = dma_pool_create("cppi", | |
03158f90 | 1326 | controller->controller.musb->controller, |
550a7375 FB |
1327 | sizeof(struct cppi_descriptor), |
1328 | CPPI_DESCRIPTOR_ALIGN, 0); | |
1329 | if (!controller->pool) { | |
1330 | kfree(controller); | |
1331 | return NULL; | |
1332 | } | |
1333 | ||
91e9c4fe SS |
1334 | if (irq > 0) { |
1335 | if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { | |
1336 | dev_err(dev, "request_irq %d failed!\n", irq); | |
7f6283ed | 1337 | musb_dma_controller_destroy(&controller->controller); |
91e9c4fe SS |
1338 | return NULL; |
1339 | } | |
1340 | controller->irq = irq; | |
1341 | } | |
1342 | ||
66c01883 | 1343 | cppi_controller_start(controller); |
550a7375 FB |
1344 | return &controller->controller; |
1345 | } | |
7f6283ed | 1346 | EXPORT_SYMBOL_GPL(cppi_dma_controller_create); |
550a7375 FB |
1347 | |
1348 | /* | |
1349 | * Destroy a previously-instantiated DMA controller. | |
1350 | */ | |
7f6283ed | 1351 | void cppi_dma_controller_destroy(struct dma_controller *c) |
550a7375 FB |
1352 | { |
1353 | struct cppi *cppi; | |
1354 | ||
1355 | cppi = container_of(c, struct cppi, controller); | |
1356 | ||
66c01883 SAS |
1357 | cppi_controller_stop(cppi); |
1358 | ||
91e9c4fe | 1359 | if (cppi->irq) |
03158f90 | 1360 | free_irq(cppi->irq, cppi->controller.musb); |
91e9c4fe | 1361 | |
550a7375 FB |
1362 | /* assert: caller stopped the controller first */ |
1363 | dma_pool_destroy(cppi->pool); | |
1364 | ||
1365 | kfree(cppi); | |
1366 | } | |
7f6283ed | 1367 | EXPORT_SYMBOL_GPL(cppi_dma_controller_destroy); |
550a7375 FB |
1368 | |
1369 | /* | |
1370 | * Context: controller irqlocked, endpoint selected | |
1371 | */ | |
1372 | static int cppi_channel_abort(struct dma_channel *channel) | |
1373 | { | |
1374 | struct cppi_channel *cppi_ch; | |
1375 | struct cppi *controller; | |
1376 | void __iomem *mbase; | |
1377 | void __iomem *tibase; | |
1378 | void __iomem *regs; | |
1379 | u32 value; | |
1380 | struct cppi_descriptor *queue; | |
1381 | ||
1382 | cppi_ch = container_of(channel, struct cppi_channel, channel); | |
1383 | ||
1384 | controller = cppi_ch->controller; | |
1385 | ||
1386 | switch (channel->status) { | |
1387 | case MUSB_DMA_STATUS_BUS_ABORT: | |
1388 | case MUSB_DMA_STATUS_CORE_ABORT: | |
1389 | /* from RX or TX fault irq handler */ | |
1390 | case MUSB_DMA_STATUS_BUSY: | |
1391 | /* the hardware needs shutting down */ | |
1392 | regs = cppi_ch->hw_ep->regs; | |
1393 | break; | |
1394 | case MUSB_DMA_STATUS_UNKNOWN: | |
1395 | case MUSB_DMA_STATUS_FREE: | |
1396 | return 0; | |
1397 | default: | |
1398 | return -EINVAL; | |
1399 | } | |
1400 | ||
1401 | if (!cppi_ch->transmit && cppi_ch->head) | |
1402 | cppi_dump_rxq(3, "/abort", cppi_ch); | |
1403 | ||
1404 | mbase = controller->mregs; | |
1405 | tibase = controller->tibase; | |
1406 | ||
1407 | queue = cppi_ch->head; | |
1408 | cppi_ch->head = NULL; | |
1409 | cppi_ch->tail = NULL; | |
1410 | ||
1411 | /* REVISIT should rely on caller having done this, | |
1412 | * and caller should rely on us not changing it. | |
1413 | * peripheral code is safe ... check host too. | |
1414 | */ | |
1415 | musb_ep_select(mbase, cppi_ch->index + 1); | |
1416 | ||
1417 | if (cppi_ch->transmit) { | |
1418 | struct cppi_tx_stateram __iomem *tx_ram; | |
550a7375 FB |
1419 | /* REVISIT put timeouts on these controller handshakes */ |
1420 | ||
1421 | cppi_dump_tx(6, cppi_ch, " (teardown)"); | |
1422 | ||
1423 | /* teardown DMA engine then usb core */ | |
1424 | do { | |
1425 | value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); | |
1426 | } while (!(value & CPPI_TEAR_READY)); | |
1427 | musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); | |
1428 | ||
1429 | tx_ram = cppi_ch->state_ram; | |
1430 | do { | |
1431 | value = musb_readl(&tx_ram->tx_complete, 0); | |
1432 | } while (0xFFFFFFFC != value); | |
550a7375 FB |
1433 | |
1434 | /* FIXME clean up the transfer state ... here? | |
1435 | * the completion routine should get called with | |
1436 | * an appropriate status code. | |
1437 | */ | |
1438 | ||
1439 | value = musb_readw(regs, MUSB_TXCSR); | |
1440 | value &= ~MUSB_TXCSR_DMAENAB; | |
1441 | value |= MUSB_TXCSR_FLUSHFIFO; | |
1442 | musb_writew(regs, MUSB_TXCSR, value); | |
1443 | musb_writew(regs, MUSB_TXCSR, value); | |
1444 | ||
56596923 | 1445 | /* |
550a7375 FB |
1446 | * 1. Write to completion Ptr value 0x1(bit 0 set) |
1447 | * (write back mode) | |
56596923 S |
1448 | * 2. Wait for abort interrupt and then put the channel in |
1449 | * compare mode by writing 1 to the tx_complete register. | |
550a7375 FB |
1450 | */ |
1451 | cppi_reset_tx(tx_ram, 1); | |
aca7f353 | 1452 | cppi_ch->head = NULL; |
56596923 | 1453 | musb_writel(&tx_ram->tx_complete, 0, 1); |
550a7375 FB |
1454 | cppi_dump_tx(5, cppi_ch, " (done teardown)"); |
1455 | ||
1456 | /* REVISIT tx side _should_ clean up the same way | |
1457 | * as the RX side ... this does no cleanup at all! | |
1458 | */ | |
1459 | ||
1460 | } else /* RX */ { | |
1461 | u16 csr; | |
1462 | ||
1463 | /* NOTE: docs don't guarantee any of this works ... we | |
1464 | * expect that if the usb core stops telling the cppi core | |
1465 | * to pull more data from it, then it'll be safe to flush | |
1466 | * current RX DMA state iff any pending fifo transfer is done. | |
1467 | */ | |
1468 | ||
1469 | core_rxirq_disable(tibase, cppi_ch->index + 1); | |
1470 | ||
1471 | /* for host, ensure ReqPkt is never set again */ | |
03158f90 | 1472 | if (is_host_active(cppi_ch->controller->controller.musb)) { |
550a7375 FB |
1473 | value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); |
1474 | value &= ~((0x3) << (cppi_ch->index * 2)); | |
1475 | musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); | |
1476 | } | |
1477 | ||
1478 | csr = musb_readw(regs, MUSB_RXCSR); | |
1479 | ||
1480 | /* for host, clear (just) ReqPkt at end of current packet(s) */ | |
03158f90 | 1481 | if (is_host_active(cppi_ch->controller->controller.musb)) { |
550a7375 FB |
1482 | csr |= MUSB_RXCSR_H_WZC_BITS; |
1483 | csr &= ~MUSB_RXCSR_H_REQPKT; | |
1484 | } else | |
1485 | csr |= MUSB_RXCSR_P_WZC_BITS; | |
1486 | ||
1487 | /* clear dma enable */ | |
1488 | csr &= ~(MUSB_RXCSR_DMAENAB); | |
1489 | musb_writew(regs, MUSB_RXCSR, csr); | |
1490 | csr = musb_readw(regs, MUSB_RXCSR); | |
1491 | ||
1492 | /* Quiesce: wait for current dma to finish (if not cleanup). | |
1493 | * We can't use bit zero of stateram->rx_sop, since that | |
1494 | * refers to an entire "DMA packet" not just emptying the | |
1495 | * current fifo. Most segments need multiple usb packets. | |
1496 | */ | |
1497 | if (channel->status == MUSB_DMA_STATUS_BUSY) | |
1498 | udelay(50); | |
1499 | ||
1500 | /* scan the current list, reporting any data that was | |
1501 | * transferred and acking any IRQ | |
1502 | */ | |
1503 | cppi_rx_scan(controller, cppi_ch->index); | |
1504 | ||
1505 | /* clobber the existing state once it's idle | |
1506 | * | |
1507 | * NOTE: arguably, we should also wait for all the other | |
1508 | * RX channels to quiesce (how??) and then temporarily | |
1509 | * disable RXCPPI_CTRL_REG ... but it seems that we can | |
1510 | * rely on the controller restarting from state ram, with | |
1511 | * only RXCPPI_BUFCNT state being bogus. BUFCNT will | |
1512 | * correct itself after the next DMA transfer though. | |
1513 | * | |
1514 | * REVISIT does using rndis mode change that? | |
1515 | */ | |
1516 | cppi_reset_rx(cppi_ch->state_ram); | |
1517 | ||
1518 | /* next DMA request _should_ load cppi head ptr */ | |
1519 | ||
1520 | /* ... we don't "free" that list, only mutate it in place. */ | |
1521 | cppi_dump_rx(5, cppi_ch, " (done abort)"); | |
1522 | ||
1523 | /* clean up previously pending bds */ | |
1524 | cppi_bd_free(cppi_ch, cppi_ch->last_processed); | |
1525 | cppi_ch->last_processed = NULL; | |
1526 | ||
1527 | while (queue) { | |
1528 | struct cppi_descriptor *tmp = queue->next; | |
1529 | ||
1530 | cppi_bd_free(cppi_ch, queue); | |
1531 | queue = tmp; | |
1532 | } | |
1533 | } | |
1534 | ||
1535 | channel->status = MUSB_DMA_STATUS_FREE; | |
1536 | cppi_ch->buf_dma = 0; | |
1537 | cppi_ch->offset = 0; | |
1538 | cppi_ch->buf_len = 0; | |
1539 | cppi_ch->maxpacket = 0; | |
1540 | return 0; | |
1541 | } | |
1542 | ||
1543 | /* TBD Queries: | |
1544 | * | |
1545 | * Power Management ... probably turn off cppi during suspend, restart; | |
1546 | * check state ram? Clocking is presumably shared with usb core. | |
1547 | */ |