| 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | /* |
| 3 | * Special handling for DW DMA core |
| 4 | * |
| 5 | * Copyright (c) 2009, 2014 Intel Corporation. |
| 6 | */ |
| 7 | |
| 8 | #include <linux/completion.h> |
| 9 | #include <linux/dma-mapping.h> |
| 10 | #include <linux/dmaengine.h> |
| 11 | #include <linux/irqreturn.h> |
| 12 | #include <linux/jiffies.h> |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/pci.h> |
| 15 | #include <linux/platform_data/dma-dw.h> |
| 16 | #include <linux/spi/spi.h> |
| 17 | #include <linux/types.h> |
| 18 | |
| 19 | #include "spi-dw.h" |
| 20 | |
| 21 | #define DW_SPI_RX_BUSY 0 |
| 22 | #define DW_SPI_RX_BURST_LEVEL 16 |
| 23 | #define DW_SPI_TX_BUSY 1 |
| 24 | #define DW_SPI_TX_BURST_LEVEL 16 |
| 25 | |
| 26 | static bool dw_spi_dma_chan_filter(struct dma_chan *chan, void *param) |
| 27 | { |
| 28 | struct dw_dma_slave *s = param; |
| 29 | |
| 30 | if (s->dma_dev != chan->device->dev) |
| 31 | return false; |
| 32 | |
| 33 | chan->private = s; |
| 34 | return true; |
| 35 | } |
| 36 | |
| 37 | static void dw_spi_dma_maxburst_init(struct dw_spi *dws) |
| 38 | { |
| 39 | struct dma_slave_caps caps; |
| 40 | u32 max_burst, def_burst; |
| 41 | int ret; |
| 42 | |
| 43 | def_burst = dws->fifo_len / 2; |
| 44 | |
| 45 | ret = dma_get_slave_caps(dws->rxchan, &caps); |
| 46 | if (!ret && caps.max_burst) |
| 47 | max_burst = caps.max_burst; |
| 48 | else |
| 49 | max_burst = DW_SPI_RX_BURST_LEVEL; |
| 50 | |
| 51 | dws->rxburst = min(max_burst, def_burst); |
| 52 | dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1); |
| 53 | |
| 54 | ret = dma_get_slave_caps(dws->txchan, &caps); |
| 55 | if (!ret && caps.max_burst) |
| 56 | max_burst = caps.max_burst; |
| 57 | else |
| 58 | max_burst = DW_SPI_TX_BURST_LEVEL; |
| 59 | |
| 60 | /* |
| 61 | * Having a Rx DMA channel serviced with higher priority than a Tx DMA |
| 62 | * channel might not be enough to provide a well balanced DMA-based |
| 63 | * SPI transfer interface. There might still be moments when the Tx DMA |
| 64 | * channel is occasionally handled faster than the Rx DMA channel. |
| 65 | * That in its turn will eventually cause the SPI Rx FIFO overflow if |
| 66 | * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's |
| 67 | * cleared by the Rx DMA channel. In order to fix the problem the Tx |
| 68 | * DMA activity is intentionally slowed down by limiting the SPI Tx |
| 69 | * FIFO depth with a value twice bigger than the Tx burst length. |
| 70 | */ |
| 71 | dws->txburst = min(max_burst, def_burst); |
| 72 | dw_writel(dws, DW_SPI_DMATDLR, dws->txburst); |
| 73 | } |
| 74 | |
| 75 | static int dw_spi_dma_caps_init(struct dw_spi *dws) |
| 76 | { |
| 77 | struct dma_slave_caps tx, rx; |
| 78 | int ret; |
| 79 | |
| 80 | ret = dma_get_slave_caps(dws->txchan, &tx); |
| 81 | if (ret) |
| 82 | return ret; |
| 83 | |
| 84 | ret = dma_get_slave_caps(dws->rxchan, &rx); |
| 85 | if (ret) |
| 86 | return ret; |
| 87 | |
| 88 | if (!(tx.directions & BIT(DMA_MEM_TO_DEV) && |
| 89 | rx.directions & BIT(DMA_DEV_TO_MEM))) |
| 90 | return -ENXIO; |
| 91 | |
| 92 | if (tx.max_sg_burst > 0 && rx.max_sg_burst > 0) |
| 93 | dws->dma_sg_burst = min(tx.max_sg_burst, rx.max_sg_burst); |
| 94 | else if (tx.max_sg_burst > 0) |
| 95 | dws->dma_sg_burst = tx.max_sg_burst; |
| 96 | else if (rx.max_sg_burst > 0) |
| 97 | dws->dma_sg_burst = rx.max_sg_burst; |
| 98 | else |
| 99 | dws->dma_sg_burst = 0; |
| 100 | |
| 101 | /* |
| 102 | * Assuming both channels belong to the same DMA controller hence the |
| 103 | * peripheral side address width capabilities most likely would be |
| 104 | * the same. |
| 105 | */ |
| 106 | dws->dma_addr_widths = tx.dst_addr_widths & rx.src_addr_widths; |
| 107 | |
| 108 | return 0; |
| 109 | } |
| 110 | |
| 111 | static int dw_spi_dma_init_mfld(struct device *dev, struct dw_spi *dws) |
| 112 | { |
| 113 | struct dw_dma_slave dma_tx = { .dst_id = 1 }, *tx = &dma_tx; |
| 114 | struct dw_dma_slave dma_rx = { .src_id = 0 }, *rx = &dma_rx; |
| 115 | struct pci_dev *dma_dev; |
| 116 | dma_cap_mask_t mask; |
| 117 | int ret = -EBUSY; |
| 118 | |
| 119 | /* |
| 120 | * Get pci device for DMA controller, currently it could only |
| 121 | * be the DMA controller of Medfield |
| 122 | */ |
| 123 | dma_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x0827, NULL); |
| 124 | if (!dma_dev) |
| 125 | return -ENODEV; |
| 126 | |
| 127 | dma_cap_zero(mask); |
| 128 | dma_cap_set(DMA_SLAVE, mask); |
| 129 | |
| 130 | /* 1. Init rx channel */ |
| 131 | rx->dma_dev = &dma_dev->dev; |
| 132 | dws->rxchan = dma_request_channel(mask, dw_spi_dma_chan_filter, rx); |
| 133 | if (!dws->rxchan) |
| 134 | goto err_exit; |
| 135 | |
| 136 | /* 2. Init tx channel */ |
| 137 | tx->dma_dev = &dma_dev->dev; |
| 138 | dws->txchan = dma_request_channel(mask, dw_spi_dma_chan_filter, tx); |
| 139 | if (!dws->txchan) |
| 140 | goto free_rxchan; |
| 141 | |
| 142 | dws->host->dma_rx = dws->rxchan; |
| 143 | dws->host->dma_tx = dws->txchan; |
| 144 | |
| 145 | init_completion(&dws->dma_completion); |
| 146 | |
| 147 | ret = dw_spi_dma_caps_init(dws); |
| 148 | if (ret) |
| 149 | goto free_txchan; |
| 150 | |
| 151 | dw_spi_dma_maxburst_init(dws); |
| 152 | |
| 153 | pci_dev_put(dma_dev); |
| 154 | |
| 155 | return 0; |
| 156 | |
| 157 | free_txchan: |
| 158 | dma_release_channel(dws->txchan); |
| 159 | dws->txchan = NULL; |
| 160 | free_rxchan: |
| 161 | dma_release_channel(dws->rxchan); |
| 162 | dws->rxchan = NULL; |
| 163 | err_exit: |
| 164 | pci_dev_put(dma_dev); |
| 165 | return ret; |
| 166 | } |
| 167 | |
| 168 | static int dw_spi_dma_init_generic(struct device *dev, struct dw_spi *dws) |
| 169 | { |
| 170 | int ret; |
| 171 | |
| 172 | dws->rxchan = dma_request_chan(dev, "rx"); |
| 173 | if (IS_ERR(dws->rxchan)) { |
| 174 | ret = PTR_ERR(dws->rxchan); |
| 175 | dws->rxchan = NULL; |
| 176 | goto err_exit; |
| 177 | } |
| 178 | |
| 179 | dws->txchan = dma_request_chan(dev, "tx"); |
| 180 | if (IS_ERR(dws->txchan)) { |
| 181 | ret = PTR_ERR(dws->txchan); |
| 182 | dws->txchan = NULL; |
| 183 | goto free_rxchan; |
| 184 | } |
| 185 | |
| 186 | dws->host->dma_rx = dws->rxchan; |
| 187 | dws->host->dma_tx = dws->txchan; |
| 188 | |
| 189 | init_completion(&dws->dma_completion); |
| 190 | |
| 191 | ret = dw_spi_dma_caps_init(dws); |
| 192 | if (ret) |
| 193 | goto free_txchan; |
| 194 | |
| 195 | dw_spi_dma_maxburst_init(dws); |
| 196 | |
| 197 | return 0; |
| 198 | |
| 199 | free_txchan: |
| 200 | dma_release_channel(dws->txchan); |
| 201 | dws->txchan = NULL; |
| 202 | free_rxchan: |
| 203 | dma_release_channel(dws->rxchan); |
| 204 | dws->rxchan = NULL; |
| 205 | err_exit: |
| 206 | return ret; |
| 207 | } |
| 208 | |
| 209 | static void dw_spi_dma_exit(struct dw_spi *dws) |
| 210 | { |
| 211 | if (dws->txchan) { |
| 212 | dmaengine_terminate_sync(dws->txchan); |
| 213 | dma_release_channel(dws->txchan); |
| 214 | } |
| 215 | |
| 216 | if (dws->rxchan) { |
| 217 | dmaengine_terminate_sync(dws->rxchan); |
| 218 | dma_release_channel(dws->rxchan); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | static irqreturn_t dw_spi_dma_transfer_handler(struct dw_spi *dws) |
| 223 | { |
| 224 | dw_spi_check_status(dws, false); |
| 225 | |
| 226 | complete(&dws->dma_completion); |
| 227 | |
| 228 | return IRQ_HANDLED; |
| 229 | } |
| 230 | |
| 231 | static enum dma_slave_buswidth dw_spi_dma_convert_width(u8 n_bytes) |
| 232 | { |
| 233 | switch (n_bytes) { |
| 234 | case 1: |
| 235 | return DMA_SLAVE_BUSWIDTH_1_BYTE; |
| 236 | case 2: |
| 237 | return DMA_SLAVE_BUSWIDTH_2_BYTES; |
| 238 | case 4: |
| 239 | return DMA_SLAVE_BUSWIDTH_4_BYTES; |
| 240 | default: |
| 241 | return DMA_SLAVE_BUSWIDTH_UNDEFINED; |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | static bool dw_spi_can_dma(struct spi_controller *host, |
| 246 | struct spi_device *spi, struct spi_transfer *xfer) |
| 247 | { |
| 248 | struct dw_spi *dws = spi_controller_get_devdata(host); |
| 249 | enum dma_slave_buswidth dma_bus_width; |
| 250 | |
| 251 | if (xfer->len <= dws->fifo_len) |
| 252 | return false; |
| 253 | |
| 254 | dma_bus_width = dw_spi_dma_convert_width(dws->n_bytes); |
| 255 | |
| 256 | return dws->dma_addr_widths & BIT(dma_bus_width); |
| 257 | } |
| 258 | |
| 259 | static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed) |
| 260 | { |
| 261 | unsigned long long ms; |
| 262 | |
| 263 | ms = len * MSEC_PER_SEC * BITS_PER_BYTE; |
| 264 | do_div(ms, speed); |
| 265 | ms += ms + 200; |
| 266 | |
| 267 | if (ms > UINT_MAX) |
| 268 | ms = UINT_MAX; |
| 269 | |
| 270 | ms = wait_for_completion_timeout(&dws->dma_completion, |
| 271 | msecs_to_jiffies(ms)); |
| 272 | |
| 273 | if (ms == 0) { |
| 274 | dev_err(&dws->host->cur_msg->spi->dev, |
| 275 | "DMA transaction timed out\n"); |
| 276 | return -ETIMEDOUT; |
| 277 | } |
| 278 | |
| 279 | return 0; |
| 280 | } |
| 281 | |
| 282 | static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws) |
| 283 | { |
| 284 | return !(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_TF_EMPT); |
| 285 | } |
| 286 | |
| 287 | static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, |
| 288 | struct spi_transfer *xfer) |
| 289 | { |
| 290 | int retry = DW_SPI_WAIT_RETRIES; |
| 291 | struct spi_delay delay; |
| 292 | u32 nents; |
| 293 | |
| 294 | nents = dw_readl(dws, DW_SPI_TXFLR); |
| 295 | delay.unit = SPI_DELAY_UNIT_SCK; |
| 296 | delay.value = nents * dws->n_bytes * BITS_PER_BYTE; |
| 297 | |
| 298 | while (dw_spi_dma_tx_busy(dws) && retry--) |
| 299 | spi_delay_exec(&delay, xfer); |
| 300 | |
| 301 | if (retry < 0) { |
| 302 | dev_err(&dws->host->dev, "Tx hanged up\n"); |
| 303 | return -EIO; |
| 304 | } |
| 305 | |
| 306 | return 0; |
| 307 | } |
| 308 | |
| 309 | /* |
| 310 | * dws->dma_chan_busy is set before the dma transfer starts, callback for tx |
| 311 | * channel will clear a corresponding bit. |
| 312 | */ |
| 313 | static void dw_spi_dma_tx_done(void *arg) |
| 314 | { |
| 315 | struct dw_spi *dws = arg; |
| 316 | |
| 317 | clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy); |
| 318 | if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) |
| 319 | return; |
| 320 | |
| 321 | complete(&dws->dma_completion); |
| 322 | } |
| 323 | |
| 324 | static int dw_spi_dma_config_tx(struct dw_spi *dws) |
| 325 | { |
| 326 | struct dma_slave_config txconf; |
| 327 | |
| 328 | memset(&txconf, 0, sizeof(txconf)); |
| 329 | txconf.direction = DMA_MEM_TO_DEV; |
| 330 | txconf.dst_addr = dws->dma_addr; |
| 331 | txconf.dst_maxburst = dws->txburst; |
| 332 | txconf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; |
| 333 | txconf.dst_addr_width = dw_spi_dma_convert_width(dws->n_bytes); |
| 334 | txconf.device_fc = false; |
| 335 | |
| 336 | return dmaengine_slave_config(dws->txchan, &txconf); |
| 337 | } |
| 338 | |
| 339 | static int dw_spi_dma_submit_tx(struct dw_spi *dws, struct scatterlist *sgl, |
| 340 | unsigned int nents) |
| 341 | { |
| 342 | struct dma_async_tx_descriptor *txdesc; |
| 343 | dma_cookie_t cookie; |
| 344 | int ret; |
| 345 | |
| 346 | txdesc = dmaengine_prep_slave_sg(dws->txchan, sgl, nents, |
| 347 | DMA_MEM_TO_DEV, |
| 348 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
| 349 | if (!txdesc) |
| 350 | return -ENOMEM; |
| 351 | |
| 352 | txdesc->callback = dw_spi_dma_tx_done; |
| 353 | txdesc->callback_param = dws; |
| 354 | |
| 355 | cookie = dmaengine_submit(txdesc); |
| 356 | ret = dma_submit_error(cookie); |
| 357 | if (ret) { |
| 358 | dmaengine_terminate_sync(dws->txchan); |
| 359 | return ret; |
| 360 | } |
| 361 | |
| 362 | set_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy); |
| 363 | |
| 364 | return 0; |
| 365 | } |
| 366 | |
| 367 | static inline bool dw_spi_dma_rx_busy(struct dw_spi *dws) |
| 368 | { |
| 369 | return !!(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_RF_NOT_EMPT); |
| 370 | } |
| 371 | |
| 372 | static int dw_spi_dma_wait_rx_done(struct dw_spi *dws) |
| 373 | { |
| 374 | int retry = DW_SPI_WAIT_RETRIES; |
| 375 | struct spi_delay delay; |
| 376 | unsigned long ns, us; |
| 377 | u32 nents; |
| 378 | |
| 379 | /* |
| 380 | * It's unlikely that DMA engine is still doing the data fetching, but |
| 381 | * if it's let's give it some reasonable time. The timeout calculation |
| 382 | * is based on the synchronous APB/SSI reference clock rate, on a |
| 383 | * number of data entries left in the Rx FIFO, times a number of clock |
| 384 | * periods normally needed for a single APB read/write transaction |
| 385 | * without PREADY signal utilized (which is true for the DW APB SSI |
| 386 | * controller). |
| 387 | */ |
| 388 | nents = dw_readl(dws, DW_SPI_RXFLR); |
| 389 | ns = 4U * NSEC_PER_SEC / dws->max_freq * nents; |
| 390 | if (ns <= NSEC_PER_USEC) { |
| 391 | delay.unit = SPI_DELAY_UNIT_NSECS; |
| 392 | delay.value = ns; |
| 393 | } else { |
| 394 | us = DIV_ROUND_UP(ns, NSEC_PER_USEC); |
| 395 | delay.unit = SPI_DELAY_UNIT_USECS; |
| 396 | delay.value = clamp_val(us, 0, USHRT_MAX); |
| 397 | } |
| 398 | |
| 399 | while (dw_spi_dma_rx_busy(dws) && retry--) |
| 400 | spi_delay_exec(&delay, NULL); |
| 401 | |
| 402 | if (retry < 0) { |
| 403 | dev_err(&dws->host->dev, "Rx hanged up\n"); |
| 404 | return -EIO; |
| 405 | } |
| 406 | |
| 407 | return 0; |
| 408 | } |
| 409 | |
| 410 | /* |
| 411 | * dws->dma_chan_busy is set before the dma transfer starts, callback for rx |
| 412 | * channel will clear a corresponding bit. |
| 413 | */ |
| 414 | static void dw_spi_dma_rx_done(void *arg) |
| 415 | { |
| 416 | struct dw_spi *dws = arg; |
| 417 | |
| 418 | clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy); |
| 419 | if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) |
| 420 | return; |
| 421 | |
| 422 | complete(&dws->dma_completion); |
| 423 | } |
| 424 | |
| 425 | static int dw_spi_dma_config_rx(struct dw_spi *dws) |
| 426 | { |
| 427 | struct dma_slave_config rxconf; |
| 428 | |
| 429 | memset(&rxconf, 0, sizeof(rxconf)); |
| 430 | rxconf.direction = DMA_DEV_TO_MEM; |
| 431 | rxconf.src_addr = dws->dma_addr; |
| 432 | rxconf.src_maxburst = dws->rxburst; |
| 433 | rxconf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; |
| 434 | rxconf.src_addr_width = dw_spi_dma_convert_width(dws->n_bytes); |
| 435 | rxconf.device_fc = false; |
| 436 | |
| 437 | return dmaengine_slave_config(dws->rxchan, &rxconf); |
| 438 | } |
| 439 | |
| 440 | static int dw_spi_dma_submit_rx(struct dw_spi *dws, struct scatterlist *sgl, |
| 441 | unsigned int nents) |
| 442 | { |
| 443 | struct dma_async_tx_descriptor *rxdesc; |
| 444 | dma_cookie_t cookie; |
| 445 | int ret; |
| 446 | |
| 447 | rxdesc = dmaengine_prep_slave_sg(dws->rxchan, sgl, nents, |
| 448 | DMA_DEV_TO_MEM, |
| 449 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
| 450 | if (!rxdesc) |
| 451 | return -ENOMEM; |
| 452 | |
| 453 | rxdesc->callback = dw_spi_dma_rx_done; |
| 454 | rxdesc->callback_param = dws; |
| 455 | |
| 456 | cookie = dmaengine_submit(rxdesc); |
| 457 | ret = dma_submit_error(cookie); |
| 458 | if (ret) { |
| 459 | dmaengine_terminate_sync(dws->rxchan); |
| 460 | return ret; |
| 461 | } |
| 462 | |
| 463 | set_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy); |
| 464 | |
| 465 | return 0; |
| 466 | } |
| 467 | |
| 468 | static int dw_spi_dma_setup(struct dw_spi *dws, struct spi_transfer *xfer) |
| 469 | { |
| 470 | u16 imr, dma_ctrl; |
| 471 | int ret; |
| 472 | |
| 473 | if (!xfer->tx_buf) |
| 474 | return -EINVAL; |
| 475 | |
| 476 | /* Setup DMA channels */ |
| 477 | ret = dw_spi_dma_config_tx(dws); |
| 478 | if (ret) |
| 479 | return ret; |
| 480 | |
| 481 | if (xfer->rx_buf) { |
| 482 | ret = dw_spi_dma_config_rx(dws); |
| 483 | if (ret) |
| 484 | return ret; |
| 485 | } |
| 486 | |
| 487 | /* Set the DMA handshaking interface */ |
| 488 | dma_ctrl = DW_SPI_DMACR_TDMAE; |
| 489 | if (xfer->rx_buf) |
| 490 | dma_ctrl |= DW_SPI_DMACR_RDMAE; |
| 491 | dw_writel(dws, DW_SPI_DMACR, dma_ctrl); |
| 492 | |
| 493 | /* Set the interrupt mask */ |
| 494 | imr = DW_SPI_INT_TXOI; |
| 495 | if (xfer->rx_buf) |
| 496 | imr |= DW_SPI_INT_RXUI | DW_SPI_INT_RXOI; |
| 497 | dw_spi_umask_intr(dws, imr); |
| 498 | |
| 499 | reinit_completion(&dws->dma_completion); |
| 500 | |
| 501 | dws->transfer_handler = dw_spi_dma_transfer_handler; |
| 502 | |
| 503 | return 0; |
| 504 | } |
| 505 | |
| 506 | static int dw_spi_dma_transfer_all(struct dw_spi *dws, |
| 507 | struct spi_transfer *xfer) |
| 508 | { |
| 509 | int ret; |
| 510 | |
| 511 | /* Submit the DMA Tx transfer */ |
| 512 | ret = dw_spi_dma_submit_tx(dws, xfer->tx_sg.sgl, xfer->tx_sg.nents); |
| 513 | if (ret) |
| 514 | goto err_clear_dmac; |
| 515 | |
| 516 | /* Submit the DMA Rx transfer if required */ |
| 517 | if (xfer->rx_buf) { |
| 518 | ret = dw_spi_dma_submit_rx(dws, xfer->rx_sg.sgl, |
| 519 | xfer->rx_sg.nents); |
| 520 | if (ret) |
| 521 | goto err_clear_dmac; |
| 522 | |
| 523 | /* rx must be started before tx due to spi instinct */ |
| 524 | dma_async_issue_pending(dws->rxchan); |
| 525 | } |
| 526 | |
| 527 | dma_async_issue_pending(dws->txchan); |
| 528 | |
| 529 | ret = dw_spi_dma_wait(dws, xfer->len, xfer->effective_speed_hz); |
| 530 | |
| 531 | err_clear_dmac: |
| 532 | dw_writel(dws, DW_SPI_DMACR, 0); |
| 533 | |
| 534 | return ret; |
| 535 | } |
| 536 | |
| 537 | /* |
| 538 | * In case if at least one of the requested DMA channels doesn't support the |
| 539 | * hardware accelerated SG list entries traverse, the DMA driver will most |
| 540 | * likely work that around by performing the IRQ-based SG list entries |
| 541 | * resubmission. That might and will cause a problem if the DMA Tx channel is |
| 542 | * recharged and re-executed before the Rx DMA channel. Due to |
| 543 | * non-deterministic IRQ-handler execution latency the DMA Tx channel will |
| 544 | * start pushing data to the SPI bus before the Rx DMA channel is even |
| 545 | * reinitialized with the next inbound SG list entry. By doing so the DMA Tx |
| 546 | * channel will implicitly start filling the DW APB SSI Rx FIFO up, which while |
| 547 | * the DMA Rx channel being recharged and re-executed will eventually be |
| 548 | * overflown. |
| 549 | * |
| 550 | * In order to solve the problem we have to feed the DMA engine with SG list |
| 551 | * entries one-by-one. It shall keep the DW APB SSI Tx and Rx FIFOs |
| 552 | * synchronized and prevent the Rx FIFO overflow. Since in general the tx_sg |
| 553 | * and rx_sg lists may have different number of entries of different lengths |
| 554 | * (though total length should match) let's virtually split the SG-lists to the |
| 555 | * set of DMA transfers, which length is a minimum of the ordered SG-entries |
| 556 | * lengths. An ASCII-sketch of the implemented algo is following: |
| 557 | * xfer->len |
| 558 | * |___________| |
| 559 | * tx_sg list: |___|____|__| |
| 560 | * rx_sg list: |_|____|____| |
| 561 | * DMA transfers: |_|_|__|_|__| |
| 562 | * |
| 563 | * Note in order to have this workaround solving the denoted problem the DMA |
| 564 | * engine driver should properly initialize the max_sg_burst capability and set |
| 565 | * the DMA device max segment size parameter with maximum data block size the |
| 566 | * DMA engine supports. |
| 567 | */ |
| 568 | |
| 569 | static int dw_spi_dma_transfer_one(struct dw_spi *dws, |
| 570 | struct spi_transfer *xfer) |
| 571 | { |
| 572 | struct scatterlist *tx_sg = NULL, *rx_sg = NULL, tx_tmp, rx_tmp; |
| 573 | unsigned int tx_len = 0, rx_len = 0; |
| 574 | unsigned int base, len; |
| 575 | int ret; |
| 576 | |
| 577 | sg_init_table(&tx_tmp, 1); |
| 578 | sg_init_table(&rx_tmp, 1); |
| 579 | |
| 580 | for (base = 0; base < xfer->len; base += len) { |
| 581 | /* Fetch next Tx DMA data chunk */ |
| 582 | if (!tx_len) { |
| 583 | tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg); |
| 584 | sg_dma_address(&tx_tmp) = sg_dma_address(tx_sg); |
| 585 | tx_len = sg_dma_len(tx_sg); |
| 586 | } |
| 587 | |
| 588 | /* Fetch next Rx DMA data chunk */ |
| 589 | if (!rx_len) { |
| 590 | rx_sg = !rx_sg ? &xfer->rx_sg.sgl[0] : sg_next(rx_sg); |
| 591 | sg_dma_address(&rx_tmp) = sg_dma_address(rx_sg); |
| 592 | rx_len = sg_dma_len(rx_sg); |
| 593 | } |
| 594 | |
| 595 | len = min(tx_len, rx_len); |
| 596 | |
| 597 | sg_dma_len(&tx_tmp) = len; |
| 598 | sg_dma_len(&rx_tmp) = len; |
| 599 | |
| 600 | /* Submit DMA Tx transfer */ |
| 601 | ret = dw_spi_dma_submit_tx(dws, &tx_tmp, 1); |
| 602 | if (ret) |
| 603 | break; |
| 604 | |
| 605 | /* Submit DMA Rx transfer */ |
| 606 | ret = dw_spi_dma_submit_rx(dws, &rx_tmp, 1); |
| 607 | if (ret) |
| 608 | break; |
| 609 | |
| 610 | /* Rx must be started before Tx due to SPI instinct */ |
| 611 | dma_async_issue_pending(dws->rxchan); |
| 612 | |
| 613 | dma_async_issue_pending(dws->txchan); |
| 614 | |
| 615 | /* |
| 616 | * Here we only need to wait for the DMA transfer to be |
| 617 | * finished since SPI controller is kept enabled during the |
| 618 | * procedure this loop implements and there is no risk to lose |
| 619 | * data left in the Tx/Rx FIFOs. |
| 620 | */ |
| 621 | ret = dw_spi_dma_wait(dws, len, xfer->effective_speed_hz); |
| 622 | if (ret) |
| 623 | break; |
| 624 | |
| 625 | reinit_completion(&dws->dma_completion); |
| 626 | |
| 627 | sg_dma_address(&tx_tmp) += len; |
| 628 | sg_dma_address(&rx_tmp) += len; |
| 629 | tx_len -= len; |
| 630 | rx_len -= len; |
| 631 | } |
| 632 | |
| 633 | dw_writel(dws, DW_SPI_DMACR, 0); |
| 634 | |
| 635 | return ret; |
| 636 | } |
| 637 | |
| 638 | static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer) |
| 639 | { |
| 640 | unsigned int nents; |
| 641 | int ret; |
| 642 | |
| 643 | nents = max(xfer->tx_sg.nents, xfer->rx_sg.nents); |
| 644 | |
| 645 | /* |
| 646 | * Execute normal DMA-based transfer (which submits the Rx and Tx SG |
| 647 | * lists directly to the DMA engine at once) if either full hardware |
| 648 | * accelerated SG list traverse is supported by both channels, or the |
| 649 | * Tx-only SPI transfer is requested, or the DMA engine is capable to |
| 650 | * handle both SG lists on hardware accelerated basis. |
| 651 | */ |
| 652 | if (!dws->dma_sg_burst || !xfer->rx_buf || nents <= dws->dma_sg_burst) |
| 653 | ret = dw_spi_dma_transfer_all(dws, xfer); |
| 654 | else |
| 655 | ret = dw_spi_dma_transfer_one(dws, xfer); |
| 656 | if (ret) |
| 657 | return ret; |
| 658 | |
| 659 | if (dws->host->cur_msg->status == -EINPROGRESS) { |
| 660 | ret = dw_spi_dma_wait_tx_done(dws, xfer); |
| 661 | if (ret) |
| 662 | return ret; |
| 663 | } |
| 664 | |
| 665 | if (xfer->rx_buf && dws->host->cur_msg->status == -EINPROGRESS) |
| 666 | ret = dw_spi_dma_wait_rx_done(dws); |
| 667 | |
| 668 | return ret; |
| 669 | } |
| 670 | |
| 671 | static void dw_spi_dma_stop(struct dw_spi *dws) |
| 672 | { |
| 673 | if (test_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy)) { |
| 674 | dmaengine_terminate_sync(dws->txchan); |
| 675 | clear_bit(DW_SPI_TX_BUSY, &dws->dma_chan_busy); |
| 676 | } |
| 677 | if (test_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy)) { |
| 678 | dmaengine_terminate_sync(dws->rxchan); |
| 679 | clear_bit(DW_SPI_RX_BUSY, &dws->dma_chan_busy); |
| 680 | } |
| 681 | } |
| 682 | |
| 683 | static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = { |
| 684 | .dma_init = dw_spi_dma_init_mfld, |
| 685 | .dma_exit = dw_spi_dma_exit, |
| 686 | .dma_setup = dw_spi_dma_setup, |
| 687 | .can_dma = dw_spi_can_dma, |
| 688 | .dma_transfer = dw_spi_dma_transfer, |
| 689 | .dma_stop = dw_spi_dma_stop, |
| 690 | }; |
| 691 | |
| 692 | void dw_spi_dma_setup_mfld(struct dw_spi *dws) |
| 693 | { |
| 694 | dws->dma_ops = &dw_spi_dma_mfld_ops; |
| 695 | } |
| 696 | EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_mfld, SPI_DW_CORE); |
| 697 | |
| 698 | static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = { |
| 699 | .dma_init = dw_spi_dma_init_generic, |
| 700 | .dma_exit = dw_spi_dma_exit, |
| 701 | .dma_setup = dw_spi_dma_setup, |
| 702 | .can_dma = dw_spi_can_dma, |
| 703 | .dma_transfer = dw_spi_dma_transfer, |
| 704 | .dma_stop = dw_spi_dma_stop, |
| 705 | }; |
| 706 | |
| 707 | void dw_spi_dma_setup_generic(struct dw_spi *dws) |
| 708 | { |
| 709 | dws->dma_ops = &dw_spi_dma_generic_ops; |
| 710 | } |
| 711 | EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_generic, SPI_DW_CORE); |