| 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | #include <linux/device.h> |
| 3 | #include <linux/dma-mapping.h> |
| 4 | #include <linux/dmaengine.h> |
| 5 | #include <linux/sizes.h> |
| 6 | #include <linux/platform_device.h> |
| 7 | #include <linux/of.h> |
| 8 | |
| 9 | #include "cppi_dma.h" |
| 10 | #include "musb_core.h" |
| 11 | #include "musb_trace.h" |
| 12 | |
| 13 | #define RNDIS_REG(x) (0x80 + ((x - 1) * 4)) |
| 14 | |
| 15 | #define EP_MODE_AUTOREQ_NONE 0 |
| 16 | #define EP_MODE_AUTOREQ_ALL_NEOP 1 |
| 17 | #define EP_MODE_AUTOREQ_ALWAYS 3 |
| 18 | |
| 19 | #define EP_MODE_DMA_TRANSPARENT 0 |
| 20 | #define EP_MODE_DMA_RNDIS 1 |
| 21 | #define EP_MODE_DMA_GEN_RNDIS 3 |
| 22 | |
| 23 | #define USB_CTRL_TX_MODE 0x70 |
| 24 | #define USB_CTRL_RX_MODE 0x74 |
| 25 | #define USB_CTRL_AUTOREQ 0xd0 |
| 26 | #define USB_TDOWN 0xd8 |
| 27 | |
| 28 | #define MUSB_DMA_NUM_CHANNELS 15 |
| 29 | |
| 30 | #define DA8XX_USB_MODE 0x10 |
| 31 | #define DA8XX_USB_AUTOREQ 0x14 |
| 32 | #define DA8XX_USB_TEARDOWN 0x1c |
| 33 | |
| 34 | #define DA8XX_DMA_NUM_CHANNELS 4 |
| 35 | |
| 36 | struct cppi41_dma_controller { |
| 37 | struct dma_controller controller; |
| 38 | struct cppi41_dma_channel *rx_channel; |
| 39 | struct cppi41_dma_channel *tx_channel; |
| 40 | struct hrtimer early_tx; |
| 41 | struct list_head early_tx_list; |
| 42 | u32 rx_mode; |
| 43 | u32 tx_mode; |
| 44 | u32 auto_req; |
| 45 | |
| 46 | u32 tdown_reg; |
| 47 | u32 autoreq_reg; |
| 48 | |
| 49 | void (*set_dma_mode)(struct cppi41_dma_channel *cppi41_channel, |
| 50 | unsigned int mode); |
| 51 | u8 num_channels; |
| 52 | }; |
| 53 | |
| 54 | static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel) |
| 55 | { |
| 56 | u16 csr; |
| 57 | u8 toggle; |
| 58 | |
| 59 | if (cppi41_channel->is_tx) |
| 60 | return; |
| 61 | if (!is_host_active(cppi41_channel->controller->controller.musb)) |
| 62 | return; |
| 63 | |
| 64 | csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR); |
| 65 | toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; |
| 66 | |
| 67 | cppi41_channel->usb_toggle = toggle; |
| 68 | } |
| 69 | |
| 70 | static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel) |
| 71 | { |
| 72 | struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; |
| 73 | struct musb *musb = hw_ep->musb; |
| 74 | u16 csr; |
| 75 | u8 toggle; |
| 76 | |
| 77 | if (cppi41_channel->is_tx) |
| 78 | return; |
| 79 | if (!is_host_active(musb)) |
| 80 | return; |
| 81 | |
| 82 | musb_ep_select(musb->mregs, hw_ep->epnum); |
| 83 | csr = musb_readw(hw_ep->regs, MUSB_RXCSR); |
| 84 | toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0; |
| 85 | |
| 86 | /* |
| 87 | * AM335x Advisory 1.0.13: Due to internal synchronisation error the |
| 88 | * data toggle may reset from DATA1 to DATA0 during receiving data from |
| 89 | * more than one endpoint. |
| 90 | */ |
| 91 | if (!toggle && toggle == cppi41_channel->usb_toggle) { |
| 92 | csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE; |
| 93 | musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr); |
| 94 | musb_dbg(musb, "Restoring DATA1 toggle."); |
| 95 | } |
| 96 | |
| 97 | cppi41_channel->usb_toggle = toggle; |
| 98 | } |
| 99 | |
| 100 | static bool musb_is_tx_fifo_empty(struct musb_hw_ep *hw_ep) |
| 101 | { |
| 102 | u8 epnum = hw_ep->epnum; |
| 103 | struct musb *musb = hw_ep->musb; |
| 104 | void __iomem *epio = musb->endpoints[epnum].regs; |
| 105 | u16 csr; |
| 106 | |
| 107 | musb_ep_select(musb->mregs, hw_ep->epnum); |
| 108 | csr = musb_readw(epio, MUSB_TXCSR); |
| 109 | if (csr & MUSB_TXCSR_TXPKTRDY) |
| 110 | return false; |
| 111 | return true; |
| 112 | } |
| 113 | |
| 114 | static void cppi41_dma_callback(void *private_data, |
| 115 | const struct dmaengine_result *result); |
| 116 | |
| 117 | static void cppi41_trans_done(struct cppi41_dma_channel *cppi41_channel) |
| 118 | { |
| 119 | struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; |
| 120 | struct musb *musb = hw_ep->musb; |
| 121 | void __iomem *epio = hw_ep->regs; |
| 122 | u16 csr; |
| 123 | |
| 124 | if (!cppi41_channel->prog_len || |
| 125 | (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)) { |
| 126 | |
| 127 | /* done, complete */ |
| 128 | cppi41_channel->channel.actual_len = |
| 129 | cppi41_channel->transferred; |
| 130 | cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; |
| 131 | cppi41_channel->channel.rx_packet_done = true; |
| 132 | |
| 133 | /* |
| 134 | * transmit ZLP using PIO mode for transfers which size is |
| 135 | * multiple of EP packet size. |
| 136 | */ |
| 137 | if (cppi41_channel->tx_zlp && (cppi41_channel->transferred % |
| 138 | cppi41_channel->packet_sz) == 0) { |
| 139 | musb_ep_select(musb->mregs, hw_ep->epnum); |
| 140 | csr = MUSB_TXCSR_MODE | MUSB_TXCSR_TXPKTRDY; |
| 141 | musb_writew(epio, MUSB_TXCSR, csr); |
| 142 | } |
| 143 | |
| 144 | trace_musb_cppi41_done(cppi41_channel); |
| 145 | musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx); |
| 146 | } else { |
| 147 | /* next iteration, reload */ |
| 148 | struct dma_chan *dc = cppi41_channel->dc; |
| 149 | struct dma_async_tx_descriptor *dma_desc; |
| 150 | enum dma_transfer_direction direction; |
| 151 | u32 remain_bytes; |
| 152 | |
| 153 | cppi41_channel->buf_addr += cppi41_channel->packet_sz; |
| 154 | |
| 155 | remain_bytes = cppi41_channel->total_len; |
| 156 | remain_bytes -= cppi41_channel->transferred; |
| 157 | remain_bytes = min(remain_bytes, cppi41_channel->packet_sz); |
| 158 | cppi41_channel->prog_len = remain_bytes; |
| 159 | |
| 160 | direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV |
| 161 | : DMA_DEV_TO_MEM; |
| 162 | dma_desc = dmaengine_prep_slave_single(dc, |
| 163 | cppi41_channel->buf_addr, |
| 164 | remain_bytes, |
| 165 | direction, |
| 166 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
| 167 | if (WARN_ON(!dma_desc)) |
| 168 | return; |
| 169 | |
| 170 | dma_desc->callback_result = cppi41_dma_callback; |
| 171 | dma_desc->callback_param = &cppi41_channel->channel; |
| 172 | cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); |
| 173 | trace_musb_cppi41_cont(cppi41_channel); |
| 174 | dma_async_issue_pending(dc); |
| 175 | |
| 176 | if (!cppi41_channel->is_tx) { |
| 177 | musb_ep_select(musb->mregs, hw_ep->epnum); |
| 178 | csr = musb_readw(epio, MUSB_RXCSR); |
| 179 | csr |= MUSB_RXCSR_H_REQPKT; |
| 180 | musb_writew(epio, MUSB_RXCSR, csr); |
| 181 | } |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | static enum hrtimer_restart cppi41_recheck_tx_req(struct hrtimer *timer) |
| 186 | { |
| 187 | struct cppi41_dma_controller *controller; |
| 188 | struct cppi41_dma_channel *cppi41_channel, *n; |
| 189 | struct musb *musb; |
| 190 | unsigned long flags; |
| 191 | enum hrtimer_restart ret = HRTIMER_NORESTART; |
| 192 | |
| 193 | controller = container_of(timer, struct cppi41_dma_controller, |
| 194 | early_tx); |
| 195 | musb = controller->controller.musb; |
| 196 | |
| 197 | spin_lock_irqsave(&musb->lock, flags); |
| 198 | list_for_each_entry_safe(cppi41_channel, n, &controller->early_tx_list, |
| 199 | tx_check) { |
| 200 | bool empty; |
| 201 | struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; |
| 202 | |
| 203 | empty = musb_is_tx_fifo_empty(hw_ep); |
| 204 | if (empty) { |
| 205 | list_del_init(&cppi41_channel->tx_check); |
| 206 | cppi41_trans_done(cppi41_channel); |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | if (!list_empty(&controller->early_tx_list) && |
| 211 | !hrtimer_is_queued(&controller->early_tx)) { |
| 212 | ret = HRTIMER_RESTART; |
| 213 | hrtimer_forward_now(&controller->early_tx, 20 * NSEC_PER_USEC); |
| 214 | } |
| 215 | |
| 216 | spin_unlock_irqrestore(&musb->lock, flags); |
| 217 | return ret; |
| 218 | } |
| 219 | |
| 220 | static void cppi41_dma_callback(void *private_data, |
| 221 | const struct dmaengine_result *result) |
| 222 | { |
| 223 | struct dma_channel *channel = private_data; |
| 224 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 225 | struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep; |
| 226 | struct cppi41_dma_controller *controller; |
| 227 | struct musb *musb = hw_ep->musb; |
| 228 | unsigned long flags; |
| 229 | struct dma_tx_state txstate; |
| 230 | u32 transferred; |
| 231 | int is_hs = 0; |
| 232 | bool empty; |
| 233 | |
| 234 | controller = cppi41_channel->controller; |
| 235 | if (controller->controller.dma_callback) |
| 236 | controller->controller.dma_callback(&controller->controller); |
| 237 | |
| 238 | if (result->result == DMA_TRANS_ABORTED) |
| 239 | return; |
| 240 | |
| 241 | spin_lock_irqsave(&musb->lock, flags); |
| 242 | |
| 243 | dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie, |
| 244 | &txstate); |
| 245 | transferred = cppi41_channel->prog_len - txstate.residue; |
| 246 | cppi41_channel->transferred += transferred; |
| 247 | |
| 248 | trace_musb_cppi41_gb(cppi41_channel); |
| 249 | update_rx_toggle(cppi41_channel); |
| 250 | |
| 251 | if (cppi41_channel->transferred == cppi41_channel->total_len || |
| 252 | transferred < cppi41_channel->packet_sz) |
| 253 | cppi41_channel->prog_len = 0; |
| 254 | |
| 255 | if (cppi41_channel->is_tx) { |
| 256 | u8 type; |
| 257 | |
| 258 | if (is_host_active(musb)) |
| 259 | type = hw_ep->out_qh->type; |
| 260 | else |
| 261 | type = hw_ep->ep_in.type; |
| 262 | |
| 263 | if (type == USB_ENDPOINT_XFER_ISOC) |
| 264 | /* |
| 265 | * Don't use the early-TX-interrupt workaround below |
| 266 | * for Isoch transfter. Since Isoch are periodic |
| 267 | * transfer, by the time the next transfer is |
| 268 | * scheduled, the current one should be done already. |
| 269 | * |
| 270 | * This avoids audio playback underrun issue. |
| 271 | */ |
| 272 | empty = true; |
| 273 | else |
| 274 | empty = musb_is_tx_fifo_empty(hw_ep); |
| 275 | } |
| 276 | |
| 277 | if (!cppi41_channel->is_tx || empty) { |
| 278 | cppi41_trans_done(cppi41_channel); |
| 279 | goto out; |
| 280 | } |
| 281 | |
| 282 | /* |
| 283 | * On AM335x it has been observed that the TX interrupt fires |
| 284 | * too early that means the TXFIFO is not yet empty but the DMA |
| 285 | * engine says that it is done with the transfer. We don't |
| 286 | * receive a FIFO empty interrupt so the only thing we can do is |
| 287 | * to poll for the bit. On HS it usually takes 2us, on FS around |
| 288 | * 110us - 150us depending on the transfer size. |
| 289 | * We spin on HS (no longer than 25us and setup a timer on |
| 290 | * FS to check for the bit and complete the transfer. |
| 291 | */ |
| 292 | if (is_host_active(musb)) { |
| 293 | if (musb->port1_status & USB_PORT_STAT_HIGH_SPEED) |
| 294 | is_hs = 1; |
| 295 | } else { |
| 296 | if (musb->g.speed == USB_SPEED_HIGH) |
| 297 | is_hs = 1; |
| 298 | } |
| 299 | if (is_hs) { |
| 300 | unsigned wait = 25; |
| 301 | |
| 302 | do { |
| 303 | empty = musb_is_tx_fifo_empty(hw_ep); |
| 304 | if (empty) { |
| 305 | cppi41_trans_done(cppi41_channel); |
| 306 | goto out; |
| 307 | } |
| 308 | wait--; |
| 309 | if (!wait) |
| 310 | break; |
| 311 | cpu_relax(); |
| 312 | } while (1); |
| 313 | } |
| 314 | list_add_tail(&cppi41_channel->tx_check, |
| 315 | &controller->early_tx_list); |
| 316 | if (!hrtimer_is_queued(&controller->early_tx)) { |
| 317 | unsigned long usecs = cppi41_channel->total_len / 10; |
| 318 | |
| 319 | hrtimer_start_range_ns(&controller->early_tx, |
| 320 | usecs * NSEC_PER_USEC, |
| 321 | 20 * NSEC_PER_USEC, |
| 322 | HRTIMER_MODE_REL); |
| 323 | } |
| 324 | |
| 325 | out: |
| 326 | spin_unlock_irqrestore(&musb->lock, flags); |
| 327 | } |
| 328 | |
| 329 | static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old) |
| 330 | { |
| 331 | unsigned shift; |
| 332 | |
| 333 | shift = (ep - 1) * 2; |
| 334 | old &= ~(3 << shift); |
| 335 | old |= mode << shift; |
| 336 | return old; |
| 337 | } |
| 338 | |
| 339 | static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel, |
| 340 | unsigned mode) |
| 341 | { |
| 342 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 343 | struct musb *musb = controller->controller.musb; |
| 344 | u32 port; |
| 345 | u32 new_mode; |
| 346 | u32 old_mode; |
| 347 | |
| 348 | if (cppi41_channel->is_tx) |
| 349 | old_mode = controller->tx_mode; |
| 350 | else |
| 351 | old_mode = controller->rx_mode; |
| 352 | port = cppi41_channel->port_num; |
| 353 | new_mode = update_ep_mode(port, mode, old_mode); |
| 354 | |
| 355 | if (new_mode == old_mode) |
| 356 | return; |
| 357 | if (cppi41_channel->is_tx) { |
| 358 | controller->tx_mode = new_mode; |
| 359 | musb_writel(musb->ctrl_base, USB_CTRL_TX_MODE, new_mode); |
| 360 | } else { |
| 361 | controller->rx_mode = new_mode; |
| 362 | musb_writel(musb->ctrl_base, USB_CTRL_RX_MODE, new_mode); |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | static void da8xx_set_dma_mode(struct cppi41_dma_channel *cppi41_channel, |
| 367 | unsigned int mode) |
| 368 | { |
| 369 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 370 | struct musb *musb = controller->controller.musb; |
| 371 | unsigned int shift; |
| 372 | u32 port; |
| 373 | u32 new_mode; |
| 374 | u32 old_mode; |
| 375 | |
| 376 | old_mode = controller->tx_mode; |
| 377 | port = cppi41_channel->port_num; |
| 378 | |
| 379 | shift = (port - 1) * 4; |
| 380 | if (!cppi41_channel->is_tx) |
| 381 | shift += 16; |
| 382 | new_mode = old_mode & ~(3 << shift); |
| 383 | new_mode |= mode << shift; |
| 384 | |
| 385 | if (new_mode == old_mode) |
| 386 | return; |
| 387 | controller->tx_mode = new_mode; |
| 388 | musb_writel(musb->ctrl_base, DA8XX_USB_MODE, new_mode); |
| 389 | } |
| 390 | |
| 391 | |
| 392 | static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel, |
| 393 | unsigned mode) |
| 394 | { |
| 395 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 396 | u32 port; |
| 397 | u32 new_mode; |
| 398 | u32 old_mode; |
| 399 | |
| 400 | old_mode = controller->auto_req; |
| 401 | port = cppi41_channel->port_num; |
| 402 | new_mode = update_ep_mode(port, mode, old_mode); |
| 403 | |
| 404 | if (new_mode == old_mode) |
| 405 | return; |
| 406 | controller->auto_req = new_mode; |
| 407 | musb_writel(controller->controller.musb->ctrl_base, |
| 408 | controller->autoreq_reg, new_mode); |
| 409 | } |
| 410 | |
| 411 | static bool cppi41_configure_channel(struct dma_channel *channel, |
| 412 | u16 packet_sz, u8 mode, |
| 413 | dma_addr_t dma_addr, u32 len) |
| 414 | { |
| 415 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 416 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 417 | struct dma_chan *dc = cppi41_channel->dc; |
| 418 | struct dma_async_tx_descriptor *dma_desc; |
| 419 | enum dma_transfer_direction direction; |
| 420 | struct musb *musb = cppi41_channel->controller->controller.musb; |
| 421 | unsigned use_gen_rndis = 0; |
| 422 | |
| 423 | cppi41_channel->buf_addr = dma_addr; |
| 424 | cppi41_channel->total_len = len; |
| 425 | cppi41_channel->transferred = 0; |
| 426 | cppi41_channel->packet_sz = packet_sz; |
| 427 | cppi41_channel->tx_zlp = (cppi41_channel->is_tx && mode) ? 1 : 0; |
| 428 | |
| 429 | /* |
| 430 | * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more |
| 431 | * than max packet size at a time. |
| 432 | */ |
| 433 | if (cppi41_channel->is_tx) |
| 434 | use_gen_rndis = 1; |
| 435 | |
| 436 | if (use_gen_rndis) { |
| 437 | /* RNDIS mode */ |
| 438 | if (len > packet_sz) { |
| 439 | musb_writel(musb->ctrl_base, |
| 440 | RNDIS_REG(cppi41_channel->port_num), len); |
| 441 | /* gen rndis */ |
| 442 | controller->set_dma_mode(cppi41_channel, |
| 443 | EP_MODE_DMA_GEN_RNDIS); |
| 444 | |
| 445 | /* auto req */ |
| 446 | cppi41_set_autoreq_mode(cppi41_channel, |
| 447 | EP_MODE_AUTOREQ_ALL_NEOP); |
| 448 | } else { |
| 449 | musb_writel(musb->ctrl_base, |
| 450 | RNDIS_REG(cppi41_channel->port_num), 0); |
| 451 | controller->set_dma_mode(cppi41_channel, |
| 452 | EP_MODE_DMA_TRANSPARENT); |
| 453 | cppi41_set_autoreq_mode(cppi41_channel, |
| 454 | EP_MODE_AUTOREQ_NONE); |
| 455 | } |
| 456 | } else { |
| 457 | /* fallback mode */ |
| 458 | controller->set_dma_mode(cppi41_channel, |
| 459 | EP_MODE_DMA_TRANSPARENT); |
| 460 | cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE); |
| 461 | len = min_t(u32, packet_sz, len); |
| 462 | } |
| 463 | cppi41_channel->prog_len = len; |
| 464 | direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; |
| 465 | dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction, |
| 466 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
| 467 | if (!dma_desc) |
| 468 | return false; |
| 469 | |
| 470 | dma_desc->callback_result = cppi41_dma_callback; |
| 471 | dma_desc->callback_param = channel; |
| 472 | cppi41_channel->cookie = dma_desc->tx_submit(dma_desc); |
| 473 | cppi41_channel->channel.rx_packet_done = false; |
| 474 | |
| 475 | trace_musb_cppi41_config(cppi41_channel); |
| 476 | |
| 477 | save_rx_toggle(cppi41_channel); |
| 478 | dma_async_issue_pending(dc); |
| 479 | return true; |
| 480 | } |
| 481 | |
| 482 | static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c, |
| 483 | struct musb_hw_ep *hw_ep, u8 is_tx) |
| 484 | { |
| 485 | struct cppi41_dma_controller *controller = container_of(c, |
| 486 | struct cppi41_dma_controller, controller); |
| 487 | struct cppi41_dma_channel *cppi41_channel = NULL; |
| 488 | u8 ch_num = hw_ep->epnum - 1; |
| 489 | |
| 490 | if (ch_num >= controller->num_channels) |
| 491 | return NULL; |
| 492 | |
| 493 | if (is_tx) |
| 494 | cppi41_channel = &controller->tx_channel[ch_num]; |
| 495 | else |
| 496 | cppi41_channel = &controller->rx_channel[ch_num]; |
| 497 | |
| 498 | if (!cppi41_channel->dc) |
| 499 | return NULL; |
| 500 | |
| 501 | if (cppi41_channel->is_allocated) |
| 502 | return NULL; |
| 503 | |
| 504 | cppi41_channel->hw_ep = hw_ep; |
| 505 | cppi41_channel->is_allocated = 1; |
| 506 | |
| 507 | trace_musb_cppi41_alloc(cppi41_channel); |
| 508 | return &cppi41_channel->channel; |
| 509 | } |
| 510 | |
| 511 | static void cppi41_dma_channel_release(struct dma_channel *channel) |
| 512 | { |
| 513 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 514 | |
| 515 | trace_musb_cppi41_free(cppi41_channel); |
| 516 | if (cppi41_channel->is_allocated) { |
| 517 | cppi41_channel->is_allocated = 0; |
| 518 | channel->status = MUSB_DMA_STATUS_FREE; |
| 519 | channel->actual_len = 0; |
| 520 | } |
| 521 | } |
| 522 | |
| 523 | static int cppi41_dma_channel_program(struct dma_channel *channel, |
| 524 | u16 packet_sz, u8 mode, |
| 525 | dma_addr_t dma_addr, u32 len) |
| 526 | { |
| 527 | int ret; |
| 528 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 529 | int hb_mult = 0; |
| 530 | |
| 531 | BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN || |
| 532 | channel->status == MUSB_DMA_STATUS_BUSY); |
| 533 | |
| 534 | if (is_host_active(cppi41_channel->controller->controller.musb)) { |
| 535 | if (cppi41_channel->is_tx) |
| 536 | hb_mult = cppi41_channel->hw_ep->out_qh->hb_mult; |
| 537 | else |
| 538 | hb_mult = cppi41_channel->hw_ep->in_qh->hb_mult; |
| 539 | } |
| 540 | |
| 541 | channel->status = MUSB_DMA_STATUS_BUSY; |
| 542 | channel->actual_len = 0; |
| 543 | |
| 544 | if (hb_mult) |
| 545 | packet_sz = hb_mult * (packet_sz & 0x7FF); |
| 546 | |
| 547 | ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len); |
| 548 | if (!ret) |
| 549 | channel->status = MUSB_DMA_STATUS_FREE; |
| 550 | |
| 551 | return ret; |
| 552 | } |
| 553 | |
| 554 | static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket, |
| 555 | void *buf, u32 length) |
| 556 | { |
| 557 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 558 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 559 | struct musb *musb = controller->controller.musb; |
| 560 | |
| 561 | if (is_host_active(musb)) { |
| 562 | WARN_ON(1); |
| 563 | return 1; |
| 564 | } |
| 565 | if (cppi41_channel->hw_ep->ep_in.type != USB_ENDPOINT_XFER_BULK) |
| 566 | return 0; |
| 567 | if (cppi41_channel->is_tx) |
| 568 | return 1; |
| 569 | /* AM335x Advisory 1.0.13. No workaround for device RX mode */ |
| 570 | return 0; |
| 571 | } |
| 572 | |
| 573 | static int cppi41_dma_channel_abort(struct dma_channel *channel) |
| 574 | { |
| 575 | struct cppi41_dma_channel *cppi41_channel = channel->private_data; |
| 576 | struct cppi41_dma_controller *controller = cppi41_channel->controller; |
| 577 | struct musb *musb = controller->controller.musb; |
| 578 | void __iomem *epio = cppi41_channel->hw_ep->regs; |
| 579 | int tdbit; |
| 580 | int ret; |
| 581 | unsigned is_tx; |
| 582 | u16 csr; |
| 583 | |
| 584 | is_tx = cppi41_channel->is_tx; |
| 585 | trace_musb_cppi41_abort(cppi41_channel); |
| 586 | |
| 587 | if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE) |
| 588 | return 0; |
| 589 | |
| 590 | list_del_init(&cppi41_channel->tx_check); |
| 591 | if (is_tx) { |
| 592 | csr = musb_readw(epio, MUSB_TXCSR); |
| 593 | csr &= ~MUSB_TXCSR_DMAENAB; |
| 594 | musb_writew(epio, MUSB_TXCSR, csr); |
| 595 | } else { |
| 596 | cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREQ_NONE); |
| 597 | |
| 598 | /* delay to drain to cppi dma pipeline for isoch */ |
| 599 | udelay(250); |
| 600 | |
| 601 | csr = musb_readw(epio, MUSB_RXCSR); |
| 602 | csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB); |
| 603 | musb_writew(epio, MUSB_RXCSR, csr); |
| 604 | |
| 605 | /* wait to drain cppi dma pipe line */ |
| 606 | udelay(50); |
| 607 | |
| 608 | csr = musb_readw(epio, MUSB_RXCSR); |
| 609 | if (csr & MUSB_RXCSR_RXPKTRDY) { |
| 610 | csr |= MUSB_RXCSR_FLUSHFIFO; |
| 611 | musb_writew(epio, MUSB_RXCSR, csr); |
| 612 | musb_writew(epio, MUSB_RXCSR, csr); |
| 613 | } |
| 614 | } |
| 615 | |
| 616 | /* DA8xx Advisory 2.3.27: wait 250 ms before to start the teardown */ |
| 617 | if (musb->ops->quirks & MUSB_DA8XX) |
| 618 | mdelay(250); |
| 619 | |
| 620 | tdbit = 1 << cppi41_channel->port_num; |
| 621 | if (is_tx) |
| 622 | tdbit <<= 16; |
| 623 | |
| 624 | do { |
| 625 | if (is_tx) |
| 626 | musb_writel(musb->ctrl_base, controller->tdown_reg, |
| 627 | tdbit); |
| 628 | ret = dmaengine_terminate_all(cppi41_channel->dc); |
| 629 | } while (ret == -EAGAIN); |
| 630 | |
| 631 | if (is_tx) { |
| 632 | musb_writel(musb->ctrl_base, controller->tdown_reg, tdbit); |
| 633 | |
| 634 | csr = musb_readw(epio, MUSB_TXCSR); |
| 635 | if (csr & MUSB_TXCSR_TXPKTRDY) { |
| 636 | csr |= MUSB_TXCSR_FLUSHFIFO; |
| 637 | musb_writew(epio, MUSB_TXCSR, csr); |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE; |
| 642 | return 0; |
| 643 | } |
| 644 | |
| 645 | static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl) |
| 646 | { |
| 647 | struct dma_chan *dc; |
| 648 | int i; |
| 649 | |
| 650 | for (i = 0; i < ctrl->num_channels; i++) { |
| 651 | dc = ctrl->tx_channel[i].dc; |
| 652 | if (dc) |
| 653 | dma_release_channel(dc); |
| 654 | dc = ctrl->rx_channel[i].dc; |
| 655 | if (dc) |
| 656 | dma_release_channel(dc); |
| 657 | } |
| 658 | } |
| 659 | |
| 660 | static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller) |
| 661 | { |
| 662 | cppi41_release_all_dma_chans(controller); |
| 663 | } |
| 664 | |
| 665 | static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller) |
| 666 | { |
| 667 | struct musb *musb = controller->controller.musb; |
| 668 | struct device *dev = musb->controller; |
| 669 | struct device_node *np = dev->parent->of_node; |
| 670 | struct cppi41_dma_channel *cppi41_channel; |
| 671 | int count; |
| 672 | int i; |
| 673 | int ret; |
| 674 | |
| 675 | count = of_property_count_strings(np, "dma-names"); |
| 676 | if (count < 0) |
| 677 | return count; |
| 678 | |
| 679 | for (i = 0; i < count; i++) { |
| 680 | struct dma_chan *dc; |
| 681 | struct dma_channel *musb_dma; |
| 682 | const char *str; |
| 683 | unsigned is_tx; |
| 684 | unsigned int port; |
| 685 | |
| 686 | ret = of_property_read_string_index(np, "dma-names", i, &str); |
| 687 | if (ret) |
| 688 | goto err; |
| 689 | if (strstarts(str, "tx")) |
| 690 | is_tx = 1; |
| 691 | else if (strstarts(str, "rx")) |
| 692 | is_tx = 0; |
| 693 | else { |
| 694 | dev_err(dev, "Wrong dmatype %s\n", str); |
| 695 | goto err; |
| 696 | } |
| 697 | ret = kstrtouint(str + 2, 0, &port); |
| 698 | if (ret) |
| 699 | goto err; |
| 700 | |
| 701 | ret = -EINVAL; |
| 702 | if (port > controller->num_channels || !port) |
| 703 | goto err; |
| 704 | if (is_tx) |
| 705 | cppi41_channel = &controller->tx_channel[port - 1]; |
| 706 | else |
| 707 | cppi41_channel = &controller->rx_channel[port - 1]; |
| 708 | |
| 709 | cppi41_channel->controller = controller; |
| 710 | cppi41_channel->port_num = port; |
| 711 | cppi41_channel->is_tx = is_tx; |
| 712 | INIT_LIST_HEAD(&cppi41_channel->tx_check); |
| 713 | |
| 714 | musb_dma = &cppi41_channel->channel; |
| 715 | musb_dma->private_data = cppi41_channel; |
| 716 | musb_dma->status = MUSB_DMA_STATUS_FREE; |
| 717 | musb_dma->max_len = SZ_4M; |
| 718 | |
| 719 | dc = dma_request_chan(dev->parent, str); |
| 720 | if (IS_ERR(dc)) { |
| 721 | ret = dev_err_probe(dev, PTR_ERR(dc), |
| 722 | "Failed to request %s.\n", str); |
| 723 | goto err; |
| 724 | } |
| 725 | |
| 726 | cppi41_channel->dc = dc; |
| 727 | } |
| 728 | return 0; |
| 729 | err: |
| 730 | cppi41_release_all_dma_chans(controller); |
| 731 | return ret; |
| 732 | } |
| 733 | |
| 734 | void cppi41_dma_controller_destroy(struct dma_controller *c) |
| 735 | { |
| 736 | struct cppi41_dma_controller *controller = container_of(c, |
| 737 | struct cppi41_dma_controller, controller); |
| 738 | |
| 739 | hrtimer_cancel(&controller->early_tx); |
| 740 | cppi41_dma_controller_stop(controller); |
| 741 | kfree(controller->rx_channel); |
| 742 | kfree(controller->tx_channel); |
| 743 | kfree(controller); |
| 744 | } |
| 745 | EXPORT_SYMBOL_GPL(cppi41_dma_controller_destroy); |
| 746 | |
| 747 | struct dma_controller * |
| 748 | cppi41_dma_controller_create(struct musb *musb, void __iomem *base) |
| 749 | { |
| 750 | struct cppi41_dma_controller *controller; |
| 751 | int channel_size; |
| 752 | int ret = 0; |
| 753 | |
| 754 | if (!musb->controller->parent->of_node) { |
| 755 | dev_err(musb->controller, "Need DT for the DMA engine.\n"); |
| 756 | return NULL; |
| 757 | } |
| 758 | |
| 759 | controller = kzalloc(sizeof(*controller), GFP_KERNEL); |
| 760 | if (!controller) |
| 761 | goto kzalloc_fail; |
| 762 | |
| 763 | hrtimer_init(&controller->early_tx, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 764 | controller->early_tx.function = cppi41_recheck_tx_req; |
| 765 | INIT_LIST_HEAD(&controller->early_tx_list); |
| 766 | |
| 767 | controller->controller.channel_alloc = cppi41_dma_channel_allocate; |
| 768 | controller->controller.channel_release = cppi41_dma_channel_release; |
| 769 | controller->controller.channel_program = cppi41_dma_channel_program; |
| 770 | controller->controller.channel_abort = cppi41_dma_channel_abort; |
| 771 | controller->controller.is_compatible = cppi41_is_compatible; |
| 772 | controller->controller.musb = musb; |
| 773 | |
| 774 | if (musb->ops->quirks & MUSB_DA8XX) { |
| 775 | controller->tdown_reg = DA8XX_USB_TEARDOWN; |
| 776 | controller->autoreq_reg = DA8XX_USB_AUTOREQ; |
| 777 | controller->set_dma_mode = da8xx_set_dma_mode; |
| 778 | controller->num_channels = DA8XX_DMA_NUM_CHANNELS; |
| 779 | } else { |
| 780 | controller->tdown_reg = USB_TDOWN; |
| 781 | controller->autoreq_reg = USB_CTRL_AUTOREQ; |
| 782 | controller->set_dma_mode = cppi41_set_dma_mode; |
| 783 | controller->num_channels = MUSB_DMA_NUM_CHANNELS; |
| 784 | } |
| 785 | |
| 786 | channel_size = controller->num_channels * |
| 787 | sizeof(struct cppi41_dma_channel); |
| 788 | controller->rx_channel = kzalloc(channel_size, GFP_KERNEL); |
| 789 | if (!controller->rx_channel) |
| 790 | goto rx_channel_alloc_fail; |
| 791 | controller->tx_channel = kzalloc(channel_size, GFP_KERNEL); |
| 792 | if (!controller->tx_channel) |
| 793 | goto tx_channel_alloc_fail; |
| 794 | |
| 795 | ret = cppi41_dma_controller_start(controller); |
| 796 | if (ret) |
| 797 | goto plat_get_fail; |
| 798 | return &controller->controller; |
| 799 | |
| 800 | plat_get_fail: |
| 801 | kfree(controller->tx_channel); |
| 802 | tx_channel_alloc_fail: |
| 803 | kfree(controller->rx_channel); |
| 804 | rx_channel_alloc_fail: |
| 805 | kfree(controller); |
| 806 | kzalloc_fail: |
| 807 | if (ret == -EPROBE_DEFER) |
| 808 | return ERR_PTR(ret); |
| 809 | return NULL; |
| 810 | } |
| 811 | EXPORT_SYMBOL_GPL(cppi41_dma_controller_create); |