Commit | Line | Data |
---|---|---|
2025cf9e | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7063c0d9 | 2 | /* |
6c710c0c | 3 | * Special handling for DW DMA core |
7063c0d9 | 4 | * |
197e96b4 | 5 | * Copyright (c) 2009, 2014 Intel Corporation. |
7063c0d9 FT |
6 | */ |
7 | ||
bdbdf0f0 | 8 | #include <linux/completion.h> |
e7940952 AS |
9 | #include <linux/dma-mapping.h> |
10 | #include <linux/dmaengine.h> | |
e62a15d9 | 11 | #include <linux/irqreturn.h> |
bdbdf0f0 | 12 | #include <linux/jiffies.h> |
7063c0d9 | 13 | #include <linux/pci.h> |
d744f826 | 14 | #include <linux/platform_data/dma-dw.h> |
6c710c0c SS |
15 | #include <linux/spi/spi.h> |
16 | #include <linux/types.h> | |
17 | ||
18 | #include "spi-dw.h" | |
7063c0d9 | 19 | |
30c8eb52 | 20 | #define RX_BUSY 0 |
c534df9d | 21 | #define RX_BURST_LEVEL 16 |
30c8eb52 | 22 | #define TX_BUSY 1 |
c534df9d | 23 | #define TX_BURST_LEVEL 16 |
30c8eb52 | 24 | |
57784411 | 25 | static bool dw_spi_dma_chan_filter(struct dma_chan *chan, void *param) |
7063c0d9 | 26 | { |
d744f826 AS |
27 | struct dw_dma_slave *s = param; |
28 | ||
29 | if (s->dma_dev != chan->device->dev) | |
30 | return false; | |
7063c0d9 | 31 | |
d744f826 AS |
32 | chan->private = s; |
33 | return true; | |
7063c0d9 FT |
34 | } |
35 | ||
57784411 | 36 | static void dw_spi_dma_maxburst_init(struct dw_spi *dws) |
0b2b6651 SS |
37 | { |
38 | struct dma_slave_caps caps; | |
39 | u32 max_burst, def_burst; | |
40 | int ret; | |
41 | ||
42 | def_burst = dws->fifo_len / 2; | |
43 | ||
44 | ret = dma_get_slave_caps(dws->rxchan, &caps); | |
45 | if (!ret && caps.max_burst) | |
46 | max_burst = caps.max_burst; | |
47 | else | |
48 | max_burst = RX_BURST_LEVEL; | |
49 | ||
50 | dws->rxburst = min(max_burst, def_burst); | |
01ddbbb0 | 51 | dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1); |
0b2b6651 SS |
52 | |
53 | ret = dma_get_slave_caps(dws->txchan, &caps); | |
54 | if (!ret && caps.max_burst) | |
55 | max_burst = caps.max_burst; | |
56 | else | |
57 | max_burst = TX_BURST_LEVEL; | |
58 | ||
01ddbbb0 SS |
59 | /* |
60 | * Having a Rx DMA channel serviced with higher priority than a Tx DMA | |
61 | * channel might not be enough to provide a well balanced DMA-based | |
62 | * SPI transfer interface. There might still be moments when the Tx DMA | |
63 | * channel is occasionally handled faster than the Rx DMA channel. | |
64 | * That in its turn will eventually cause the SPI Rx FIFO overflow if | |
65 | * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's | |
66 | * cleared by the Rx DMA channel. In order to fix the problem the Tx | |
67 | * DMA activity is intentionally slowed down by limiting the SPI Tx | |
68 | * FIFO depth with a value twice bigger than the Tx burst length. | |
69 | */ | |
0b2b6651 | 70 | dws->txburst = min(max_burst, def_burst); |
01ddbbb0 | 71 | dw_writel(dws, DW_SPI_DMATDLR, dws->txburst); |
0b2b6651 SS |
72 | } |
73 | ||
ad4fe126 SS |
74 | static void dw_spi_dma_sg_burst_init(struct dw_spi *dws) |
75 | { | |
76 | struct dma_slave_caps tx = {0}, rx = {0}; | |
77 | ||
78 | dma_get_slave_caps(dws->txchan, &tx); | |
79 | dma_get_slave_caps(dws->rxchan, &rx); | |
80 | ||
81 | if (tx.max_sg_burst > 0 && rx.max_sg_burst > 0) | |
82 | dws->dma_sg_burst = min(tx.max_sg_burst, rx.max_sg_burst); | |
83 | else if (tx.max_sg_burst > 0) | |
84 | dws->dma_sg_burst = tx.max_sg_burst; | |
85 | else if (rx.max_sg_burst > 0) | |
86 | dws->dma_sg_burst = rx.max_sg_burst; | |
87 | else | |
88 | dws->dma_sg_burst = 0; | |
89 | } | |
90 | ||
57784411 | 91 | static int dw_spi_dma_init_mfld(struct device *dev, struct dw_spi *dws) |
7063c0d9 | 92 | { |
b3f82dc2 AS |
93 | struct dw_dma_slave dma_tx = { .dst_id = 1 }, *tx = &dma_tx; |
94 | struct dw_dma_slave dma_rx = { .src_id = 0 }, *rx = &dma_rx; | |
b89e9c87 | 95 | struct pci_dev *dma_dev; |
7063c0d9 FT |
96 | dma_cap_mask_t mask; |
97 | ||
98 | /* | |
99 | * Get pci device for DMA controller, currently it could only | |
ea092455 | 100 | * be the DMA controller of Medfield |
7063c0d9 | 101 | */ |
b89e9c87 AS |
102 | dma_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x0827, NULL); |
103 | if (!dma_dev) | |
104 | return -ENODEV; | |
105 | ||
7063c0d9 FT |
106 | dma_cap_zero(mask); |
107 | dma_cap_set(DMA_SLAVE, mask); | |
108 | ||
109 | /* 1. Init rx channel */ | |
b3f82dc2 AS |
110 | rx->dma_dev = &dma_dev->dev; |
111 | dws->rxchan = dma_request_channel(mask, dw_spi_dma_chan_filter, rx); | |
7063c0d9 FT |
112 | if (!dws->rxchan) |
113 | goto err_exit; | |
7063c0d9 FT |
114 | |
115 | /* 2. Init tx channel */ | |
b3f82dc2 AS |
116 | tx->dma_dev = &dma_dev->dev; |
117 | dws->txchan = dma_request_channel(mask, dw_spi_dma_chan_filter, tx); | |
7063c0d9 FT |
118 | if (!dws->txchan) |
119 | goto free_rxchan; | |
a041e672 AS |
120 | |
121 | dws->master->dma_rx = dws->rxchan; | |
f89a6d8f | 122 | dws->master->dma_tx = dws->txchan; |
7063c0d9 | 123 | |
bdbdf0f0 SS |
124 | init_completion(&dws->dma_completion); |
125 | ||
57784411 | 126 | dw_spi_dma_maxburst_init(dws); |
0b2b6651 | 127 | |
ad4fe126 SS |
128 | dw_spi_dma_sg_burst_init(dws); |
129 | ||
7063c0d9 FT |
130 | return 0; |
131 | ||
132 | free_rxchan: | |
133 | dma_release_channel(dws->rxchan); | |
a041e672 | 134 | dws->rxchan = NULL; |
7063c0d9 | 135 | err_exit: |
b89e9c87 | 136 | return -EBUSY; |
7063c0d9 FT |
137 | } |
138 | ||
57784411 | 139 | static int dw_spi_dma_init_generic(struct device *dev, struct dw_spi *dws) |
22d48ad7 JN |
140 | { |
141 | dws->rxchan = dma_request_slave_channel(dev, "rx"); | |
142 | if (!dws->rxchan) | |
143 | return -ENODEV; | |
22d48ad7 JN |
144 | |
145 | dws->txchan = dma_request_slave_channel(dev, "tx"); | |
146 | if (!dws->txchan) { | |
147 | dma_release_channel(dws->rxchan); | |
a041e672 | 148 | dws->rxchan = NULL; |
22d48ad7 JN |
149 | return -ENODEV; |
150 | } | |
a041e672 AS |
151 | |
152 | dws->master->dma_rx = dws->rxchan; | |
22d48ad7 JN |
153 | dws->master->dma_tx = dws->txchan; |
154 | ||
bdbdf0f0 SS |
155 | init_completion(&dws->dma_completion); |
156 | ||
57784411 | 157 | dw_spi_dma_maxburst_init(dws); |
0b2b6651 | 158 | |
ad4fe126 SS |
159 | dw_spi_dma_sg_burst_init(dws); |
160 | ||
22d48ad7 JN |
161 | return 0; |
162 | } | |
163 | ||
57784411 | 164 | static void dw_spi_dma_exit(struct dw_spi *dws) |
7063c0d9 | 165 | { |
a041e672 AS |
166 | if (dws->txchan) { |
167 | dmaengine_terminate_sync(dws->txchan); | |
168 | dma_release_channel(dws->txchan); | |
169 | } | |
8e45ef68 | 170 | |
a041e672 AS |
171 | if (dws->rxchan) { |
172 | dmaengine_terminate_sync(dws->rxchan); | |
173 | dma_release_channel(dws->rxchan); | |
174 | } | |
7063c0d9 FT |
175 | } |
176 | ||
57784411 | 177 | static irqreturn_t dw_spi_dma_transfer_handler(struct dw_spi *dws) |
f051fc8f | 178 | { |
bf64b660 | 179 | dw_spi_check_status(dws, false); |
f051fc8f | 180 | |
bdbdf0f0 | 181 | complete(&dws->dma_completion); |
bf64b660 | 182 | |
f051fc8f AS |
183 | return IRQ_HANDLED; |
184 | } | |
185 | ||
57784411 SS |
186 | static bool dw_spi_can_dma(struct spi_controller *master, |
187 | struct spi_device *spi, struct spi_transfer *xfer) | |
f89a6d8f | 188 | { |
721483e2 | 189 | struct dw_spi *dws = spi_controller_get_devdata(master); |
f89a6d8f | 190 | |
f89a6d8f AS |
191 | return xfer->len > dws->fifo_len; |
192 | } | |
193 | ||
57784411 SS |
194 | static enum dma_slave_buswidth dw_spi_dma_convert_width(u8 n_bytes) |
195 | { | |
4fdc03a9 | 196 | if (n_bytes == 1) |
e31abce7 | 197 | return DMA_SLAVE_BUSWIDTH_1_BYTE; |
4fdc03a9 | 198 | else if (n_bytes == 2) |
e31abce7 AS |
199 | return DMA_SLAVE_BUSWIDTH_2_BYTES; |
200 | ||
201 | return DMA_SLAVE_BUSWIDTH_UNDEFINED; | |
202 | } | |
203 | ||
917ce29e | 204 | static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed) |
bdbdf0f0 SS |
205 | { |
206 | unsigned long long ms; | |
207 | ||
917ce29e SS |
208 | ms = len * MSEC_PER_SEC * BITS_PER_BYTE; |
209 | do_div(ms, speed); | |
bdbdf0f0 SS |
210 | ms += ms + 200; |
211 | ||
212 | if (ms > UINT_MAX) | |
213 | ms = UINT_MAX; | |
214 | ||
215 | ms = wait_for_completion_timeout(&dws->dma_completion, | |
216 | msecs_to_jiffies(ms)); | |
217 | ||
218 | if (ms == 0) { | |
219 | dev_err(&dws->master->cur_msg->spi->dev, | |
220 | "DMA transaction timed out\n"); | |
221 | return -ETIMEDOUT; | |
222 | } | |
223 | ||
224 | return 0; | |
225 | } | |
226 | ||
1ade2d8a SS |
227 | static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws) |
228 | { | |
229 | return !(dw_readl(dws, DW_SPI_SR) & SR_TF_EMPT); | |
230 | } | |
231 | ||
232 | static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, | |
233 | struct spi_transfer *xfer) | |
234 | { | |
cf75baea | 235 | int retry = SPI_WAIT_RETRIES; |
1ade2d8a SS |
236 | struct spi_delay delay; |
237 | u32 nents; | |
238 | ||
239 | nents = dw_readl(dws, DW_SPI_TXFLR); | |
240 | delay.unit = SPI_DELAY_UNIT_SCK; | |
241 | delay.value = nents * dws->n_bytes * BITS_PER_BYTE; | |
242 | ||
243 | while (dw_spi_dma_tx_busy(dws) && retry--) | |
244 | spi_delay_exec(&delay, xfer); | |
245 | ||
246 | if (retry < 0) { | |
247 | dev_err(&dws->master->dev, "Tx hanged up\n"); | |
248 | return -EIO; | |
249 | } | |
250 | ||
251 | return 0; | |
252 | } | |
253 | ||
7063c0d9 | 254 | /* |
30c8eb52 AS |
255 | * dws->dma_chan_busy is set before the dma transfer starts, callback for tx |
256 | * channel will clear a corresponding bit. | |
7063c0d9 | 257 | */ |
30c8eb52 | 258 | static void dw_spi_dma_tx_done(void *arg) |
7063c0d9 FT |
259 | { |
260 | struct dw_spi *dws = arg; | |
261 | ||
854d2f24 AS |
262 | clear_bit(TX_BUSY, &dws->dma_chan_busy); |
263 | if (test_bit(RX_BUSY, &dws->dma_chan_busy)) | |
7063c0d9 | 264 | return; |
0327f0b8 | 265 | |
bdbdf0f0 | 266 | complete(&dws->dma_completion); |
7063c0d9 FT |
267 | } |
268 | ||
a874d811 | 269 | static int dw_spi_dma_config_tx(struct dw_spi *dws) |
7063c0d9 | 270 | { |
a5c2db96 | 271 | struct dma_slave_config txconf; |
7063c0d9 | 272 | |
3cb97e22 | 273 | memset(&txconf, 0, sizeof(txconf)); |
a485df4b | 274 | txconf.direction = DMA_MEM_TO_DEV; |
7063c0d9 | 275 | txconf.dst_addr = dws->dma_addr; |
0b2b6651 | 276 | txconf.dst_maxburst = dws->txburst; |
7063c0d9 | 277 | txconf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; |
57784411 | 278 | txconf.dst_addr_width = dw_spi_dma_convert_width(dws->n_bytes); |
258aea76 | 279 | txconf.device_fc = false; |
7063c0d9 | 280 | |
a874d811 SS |
281 | return dmaengine_slave_config(dws->txchan, &txconf); |
282 | } | |
283 | ||
917ce29e SS |
284 | static int dw_spi_dma_submit_tx(struct dw_spi *dws, struct scatterlist *sgl, |
285 | unsigned int nents) | |
a874d811 SS |
286 | { |
287 | struct dma_async_tx_descriptor *txdesc; | |
9a6471a1 SS |
288 | dma_cookie_t cookie; |
289 | int ret; | |
7063c0d9 | 290 | |
917ce29e SS |
291 | txdesc = dmaengine_prep_slave_sg(dws->txchan, sgl, nents, |
292 | DMA_MEM_TO_DEV, | |
293 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
c9dafb27 | 294 | if (!txdesc) |
7a4d61f1 | 295 | return -ENOMEM; |
c9dafb27 | 296 | |
30c8eb52 | 297 | txdesc->callback = dw_spi_dma_tx_done; |
7063c0d9 FT |
298 | txdesc->callback_param = dws; |
299 | ||
9a6471a1 SS |
300 | cookie = dmaengine_submit(txdesc); |
301 | ret = dma_submit_error(cookie); | |
302 | if (ret) { | |
303 | dmaengine_terminate_sync(dws->txchan); | |
7a4d61f1 | 304 | return ret; |
9a6471a1 SS |
305 | } |
306 | ||
ab7a4d75 SS |
307 | set_bit(TX_BUSY, &dws->dma_chan_busy); |
308 | ||
7a4d61f1 | 309 | return 0; |
a5c2db96 AS |
310 | } |
311 | ||
33726eff SS |
312 | static inline bool dw_spi_dma_rx_busy(struct dw_spi *dws) |
313 | { | |
314 | return !!(dw_readl(dws, DW_SPI_SR) & SR_RF_NOT_EMPT); | |
315 | } | |
316 | ||
317 | static int dw_spi_dma_wait_rx_done(struct dw_spi *dws) | |
318 | { | |
cf75baea | 319 | int retry = SPI_WAIT_RETRIES; |
33726eff SS |
320 | struct spi_delay delay; |
321 | unsigned long ns, us; | |
322 | u32 nents; | |
323 | ||
324 | /* | |
325 | * It's unlikely that DMA engine is still doing the data fetching, but | |
326 | * if it's let's give it some reasonable time. The timeout calculation | |
327 | * is based on the synchronous APB/SSI reference clock rate, on a | |
328 | * number of data entries left in the Rx FIFO, times a number of clock | |
329 | * periods normally needed for a single APB read/write transaction | |
330 | * without PREADY signal utilized (which is true for the DW APB SSI | |
331 | * controller). | |
332 | */ | |
333 | nents = dw_readl(dws, DW_SPI_RXFLR); | |
334 | ns = 4U * NSEC_PER_SEC / dws->max_freq * nents; | |
335 | if (ns <= NSEC_PER_USEC) { | |
336 | delay.unit = SPI_DELAY_UNIT_NSECS; | |
337 | delay.value = ns; | |
338 | } else { | |
339 | us = DIV_ROUND_UP(ns, NSEC_PER_USEC); | |
340 | delay.unit = SPI_DELAY_UNIT_USECS; | |
341 | delay.value = clamp_val(us, 0, USHRT_MAX); | |
342 | } | |
343 | ||
344 | while (dw_spi_dma_rx_busy(dws) && retry--) | |
345 | spi_delay_exec(&delay, NULL); | |
346 | ||
347 | if (retry < 0) { | |
348 | dev_err(&dws->master->dev, "Rx hanged up\n"); | |
349 | return -EIO; | |
350 | } | |
351 | ||
352 | return 0; | |
353 | } | |
354 | ||
30c8eb52 AS |
355 | /* |
356 | * dws->dma_chan_busy is set before the dma transfer starts, callback for rx | |
357 | * channel will clear a corresponding bit. | |
358 | */ | |
359 | static void dw_spi_dma_rx_done(void *arg) | |
360 | { | |
361 | struct dw_spi *dws = arg; | |
362 | ||
854d2f24 AS |
363 | clear_bit(RX_BUSY, &dws->dma_chan_busy); |
364 | if (test_bit(TX_BUSY, &dws->dma_chan_busy)) | |
30c8eb52 | 365 | return; |
0327f0b8 | 366 | |
bdbdf0f0 | 367 | complete(&dws->dma_completion); |
30c8eb52 AS |
368 | } |
369 | ||
a874d811 | 370 | static int dw_spi_dma_config_rx(struct dw_spi *dws) |
a5c2db96 AS |
371 | { |
372 | struct dma_slave_config rxconf; | |
30c8eb52 | 373 | |
3cb97e22 | 374 | memset(&rxconf, 0, sizeof(rxconf)); |
a485df4b | 375 | rxconf.direction = DMA_DEV_TO_MEM; |
7063c0d9 | 376 | rxconf.src_addr = dws->dma_addr; |
0b2b6651 | 377 | rxconf.src_maxburst = dws->rxburst; |
7063c0d9 | 378 | rxconf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; |
57784411 | 379 | rxconf.src_addr_width = dw_spi_dma_convert_width(dws->n_bytes); |
258aea76 | 380 | rxconf.device_fc = false; |
7063c0d9 | 381 | |
a874d811 SS |
382 | return dmaengine_slave_config(dws->rxchan, &rxconf); |
383 | } | |
384 | ||
917ce29e SS |
385 | static int dw_spi_dma_submit_rx(struct dw_spi *dws, struct scatterlist *sgl, |
386 | unsigned int nents) | |
a874d811 SS |
387 | { |
388 | struct dma_async_tx_descriptor *rxdesc; | |
9a6471a1 SS |
389 | dma_cookie_t cookie; |
390 | int ret; | |
a874d811 | 391 | |
917ce29e SS |
392 | rxdesc = dmaengine_prep_slave_sg(dws->rxchan, sgl, nents, |
393 | DMA_DEV_TO_MEM, | |
394 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
c9dafb27 | 395 | if (!rxdesc) |
7a4d61f1 | 396 | return -ENOMEM; |
c9dafb27 | 397 | |
30c8eb52 | 398 | rxdesc->callback = dw_spi_dma_rx_done; |
7063c0d9 FT |
399 | rxdesc->callback_param = dws; |
400 | ||
9a6471a1 SS |
401 | cookie = dmaengine_submit(rxdesc); |
402 | ret = dma_submit_error(cookie); | |
403 | if (ret) { | |
404 | dmaengine_terminate_sync(dws->rxchan); | |
7a4d61f1 | 405 | return ret; |
9a6471a1 SS |
406 | } |
407 | ||
ab7a4d75 SS |
408 | set_bit(RX_BUSY, &dws->dma_chan_busy); |
409 | ||
7a4d61f1 | 410 | return 0; |
a5c2db96 AS |
411 | } |
412 | ||
57784411 | 413 | static int dw_spi_dma_setup(struct dw_spi *dws, struct spi_transfer *xfer) |
a5c2db96 | 414 | { |
7ef30385 | 415 | u16 imr, dma_ctrl; |
a874d811 | 416 | int ret; |
a5c2db96 | 417 | |
7ef30385 SS |
418 | if (!xfer->tx_buf) |
419 | return -EINVAL; | |
420 | ||
a874d811 SS |
421 | /* Setup DMA channels */ |
422 | ret = dw_spi_dma_config_tx(dws); | |
423 | if (ret) | |
424 | return ret; | |
425 | ||
426 | if (xfer->rx_buf) { | |
427 | ret = dw_spi_dma_config_rx(dws); | |
428 | if (ret) | |
429 | return ret; | |
430 | } | |
431 | ||
7ef30385 SS |
432 | /* Set the DMA handshaking interface */ |
433 | dma_ctrl = SPI_DMA_TDMAE; | |
3d7db0f1 | 434 | if (xfer->rx_buf) |
a5c2db96 | 435 | dma_ctrl |= SPI_DMA_RDMAE; |
dd114443 | 436 | dw_writel(dws, DW_SPI_DMACR, dma_ctrl); |
a5c2db96 | 437 | |
f051fc8f | 438 | /* Set the interrupt mask */ |
7ef30385 | 439 | imr = SPI_INT_TXOI; |
3d7db0f1 AS |
440 | if (xfer->rx_buf) |
441 | imr |= SPI_INT_RXUI | SPI_INT_RXOI; | |
43dba9f3 | 442 | spi_umask_intr(dws, imr); |
f051fc8f | 443 | |
bdbdf0f0 SS |
444 | reinit_completion(&dws->dma_completion); |
445 | ||
57784411 | 446 | dws->transfer_handler = dw_spi_dma_transfer_handler; |
f051fc8f | 447 | |
9f14538e | 448 | return 0; |
a5c2db96 AS |
449 | } |
450 | ||
b86fed12 SS |
451 | static int dw_spi_dma_transfer_all(struct dw_spi *dws, |
452 | struct spi_transfer *xfer) | |
a5c2db96 | 453 | { |
bdbdf0f0 | 454 | int ret; |
a5c2db96 | 455 | |
ab7a4d75 | 456 | /* Submit the DMA Tx transfer */ |
917ce29e | 457 | ret = dw_spi_dma_submit_tx(dws, xfer->tx_sg.sgl, xfer->tx_sg.nents); |
7a4d61f1 | 458 | if (ret) |
945b5b60 | 459 | goto err_clear_dmac; |
a5c2db96 | 460 | |
ab7a4d75 | 461 | /* Submit the DMA Rx transfer if required */ |
be3034d9 | 462 | if (xfer->rx_buf) { |
917ce29e SS |
463 | ret = dw_spi_dma_submit_rx(dws, xfer->rx_sg.sgl, |
464 | xfer->rx_sg.nents); | |
7a4d61f1 | 465 | if (ret) |
945b5b60 | 466 | goto err_clear_dmac; |
a5c2db96 | 467 | |
be3034d9 | 468 | /* rx must be started before tx due to spi instinct */ |
30c8eb52 AS |
469 | dma_async_issue_pending(dws->rxchan); |
470 | } | |
471 | ||
7ef30385 | 472 | dma_async_issue_pending(dws->txchan); |
f7477c2b | 473 | |
917ce29e | 474 | ret = dw_spi_dma_wait(dws, xfer->len, xfer->effective_speed_hz); |
945b5b60 SS |
475 | |
476 | err_clear_dmac: | |
477 | dw_writel(dws, DW_SPI_DMACR, 0); | |
478 | ||
479 | return ret; | |
b86fed12 SS |
480 | } |
481 | ||
ad4fe126 SS |
482 | /* |
483 | * In case if at least one of the requested DMA channels doesn't support the | |
484 | * hardware accelerated SG list entries traverse, the DMA driver will most | |
485 | * likely work that around by performing the IRQ-based SG list entries | |
486 | * resubmission. That might and will cause a problem if the DMA Tx channel is | |
487 | * recharged and re-executed before the Rx DMA channel. Due to | |
488 | * non-deterministic IRQ-handler execution latency the DMA Tx channel will | |
489 | * start pushing data to the SPI bus before the Rx DMA channel is even | |
490 | * reinitialized with the next inbound SG list entry. By doing so the DMA Tx | |
491 | * channel will implicitly start filling the DW APB SSI Rx FIFO up, which while | |
492 | * the DMA Rx channel being recharged and re-executed will eventually be | |
493 | * overflown. | |
494 | * | |
495 | * In order to solve the problem we have to feed the DMA engine with SG list | |
496 | * entries one-by-one. It shall keep the DW APB SSI Tx and Rx FIFOs | |
497 | * synchronized and prevent the Rx FIFO overflow. Since in general the tx_sg | |
498 | * and rx_sg lists may have different number of entries of different lengths | |
499 | * (though total length should match) let's virtually split the SG-lists to the | |
500 | * set of DMA transfers, which length is a minimum of the ordered SG-entries | |
501 | * lengths. An ASCII-sketch of the implemented algo is following: | |
502 | * xfer->len | |
503 | * |___________| | |
504 | * tx_sg list: |___|____|__| | |
505 | * rx_sg list: |_|____|____| | |
506 | * DMA transfers: |_|_|__|_|__| | |
507 | * | |
508 | * Note in order to have this workaround solving the denoted problem the DMA | |
509 | * engine driver should properly initialize the max_sg_burst capability and set | |
510 | * the DMA device max segment size parameter with maximum data block size the | |
511 | * DMA engine supports. | |
512 | */ | |
513 | ||
514 | static int dw_spi_dma_transfer_one(struct dw_spi *dws, | |
515 | struct spi_transfer *xfer) | |
516 | { | |
517 | struct scatterlist *tx_sg = NULL, *rx_sg = NULL, tx_tmp, rx_tmp; | |
518 | unsigned int tx_len = 0, rx_len = 0; | |
519 | unsigned int base, len; | |
520 | int ret; | |
521 | ||
522 | sg_init_table(&tx_tmp, 1); | |
523 | sg_init_table(&rx_tmp, 1); | |
524 | ||
525 | for (base = 0, len = 0; base < xfer->len; base += len) { | |
526 | /* Fetch next Tx DMA data chunk */ | |
527 | if (!tx_len) { | |
528 | tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg); | |
529 | sg_dma_address(&tx_tmp) = sg_dma_address(tx_sg); | |
530 | tx_len = sg_dma_len(tx_sg); | |
531 | } | |
532 | ||
533 | /* Fetch next Rx DMA data chunk */ | |
534 | if (!rx_len) { | |
535 | rx_sg = !rx_sg ? &xfer->rx_sg.sgl[0] : sg_next(rx_sg); | |
536 | sg_dma_address(&rx_tmp) = sg_dma_address(rx_sg); | |
537 | rx_len = sg_dma_len(rx_sg); | |
538 | } | |
539 | ||
540 | len = min(tx_len, rx_len); | |
541 | ||
542 | sg_dma_len(&tx_tmp) = len; | |
543 | sg_dma_len(&rx_tmp) = len; | |
544 | ||
545 | /* Submit DMA Tx transfer */ | |
546 | ret = dw_spi_dma_submit_tx(dws, &tx_tmp, 1); | |
547 | if (ret) | |
548 | break; | |
549 | ||
550 | /* Submit DMA Rx transfer */ | |
551 | ret = dw_spi_dma_submit_rx(dws, &rx_tmp, 1); | |
552 | if (ret) | |
553 | break; | |
554 | ||
555 | /* Rx must be started before Tx due to SPI instinct */ | |
556 | dma_async_issue_pending(dws->rxchan); | |
557 | ||
558 | dma_async_issue_pending(dws->txchan); | |
559 | ||
560 | /* | |
561 | * Here we only need to wait for the DMA transfer to be | |
562 | * finished since SPI controller is kept enabled during the | |
563 | * procedure this loop implements and there is no risk to lose | |
564 | * data left in the Tx/Rx FIFOs. | |
565 | */ | |
566 | ret = dw_spi_dma_wait(dws, len, xfer->effective_speed_hz); | |
567 | if (ret) | |
568 | break; | |
569 | ||
570 | reinit_completion(&dws->dma_completion); | |
571 | ||
572 | sg_dma_address(&tx_tmp) += len; | |
573 | sg_dma_address(&rx_tmp) += len; | |
574 | tx_len -= len; | |
575 | rx_len -= len; | |
576 | } | |
577 | ||
578 | dw_writel(dws, DW_SPI_DMACR, 0); | |
579 | ||
580 | return ret; | |
581 | } | |
582 | ||
b86fed12 SS |
583 | static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer) |
584 | { | |
ad4fe126 | 585 | unsigned int nents; |
b86fed12 SS |
586 | int ret; |
587 | ||
ad4fe126 SS |
588 | nents = max(xfer->tx_sg.nents, xfer->rx_sg.nents); |
589 | ||
590 | /* | |
591 | * Execute normal DMA-based transfer (which submits the Rx and Tx SG | |
592 | * lists directly to the DMA engine at once) if either full hardware | |
593 | * accelerated SG list traverse is supported by both channels, or the | |
594 | * Tx-only SPI transfer is requested, or the DMA engine is capable to | |
595 | * handle both SG lists on hardware accelerated basis. | |
596 | */ | |
597 | if (!dws->dma_sg_burst || !xfer->rx_buf || nents <= dws->dma_sg_burst) | |
598 | ret = dw_spi_dma_transfer_all(dws, xfer); | |
599 | else | |
600 | ret = dw_spi_dma_transfer_one(dws, xfer); | |
bdbdf0f0 SS |
601 | if (ret) |
602 | return ret; | |
603 | ||
7ef30385 | 604 | if (dws->master->cur_msg->status == -EINPROGRESS) { |
1ade2d8a SS |
605 | ret = dw_spi_dma_wait_tx_done(dws, xfer); |
606 | if (ret) | |
607 | return ret; | |
608 | } | |
609 | ||
be3034d9 | 610 | if (xfer->rx_buf && dws->master->cur_msg->status == -EINPROGRESS) |
33726eff SS |
611 | ret = dw_spi_dma_wait_rx_done(dws); |
612 | ||
613 | return ret; | |
7063c0d9 FT |
614 | } |
615 | ||
57784411 | 616 | static void dw_spi_dma_stop(struct dw_spi *dws) |
4d5ac1ed AS |
617 | { |
618 | if (test_bit(TX_BUSY, &dws->dma_chan_busy)) { | |
cf1716e9 | 619 | dmaengine_terminate_sync(dws->txchan); |
4d5ac1ed AS |
620 | clear_bit(TX_BUSY, &dws->dma_chan_busy); |
621 | } | |
622 | if (test_bit(RX_BUSY, &dws->dma_chan_busy)) { | |
cf1716e9 | 623 | dmaengine_terminate_sync(dws->rxchan); |
4d5ac1ed AS |
624 | clear_bit(RX_BUSY, &dws->dma_chan_busy); |
625 | } | |
626 | } | |
627 | ||
57784411 SS |
628 | static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = { |
629 | .dma_init = dw_spi_dma_init_mfld, | |
630 | .dma_exit = dw_spi_dma_exit, | |
631 | .dma_setup = dw_spi_dma_setup, | |
632 | .can_dma = dw_spi_can_dma, | |
633 | .dma_transfer = dw_spi_dma_transfer, | |
634 | .dma_stop = dw_spi_dma_stop, | |
7063c0d9 | 635 | }; |
37aa8aa6 | 636 | |
57784411 | 637 | void dw_spi_dma_setup_mfld(struct dw_spi *dws) |
37aa8aa6 | 638 | { |
57784411 | 639 | dws->dma_ops = &dw_spi_dma_mfld_ops; |
37aa8aa6 | 640 | } |
57784411 SS |
641 | EXPORT_SYMBOL_GPL(dw_spi_dma_setup_mfld); |
642 | ||
643 | static const struct dw_spi_dma_ops dw_spi_dma_generic_ops = { | |
644 | .dma_init = dw_spi_dma_init_generic, | |
645 | .dma_exit = dw_spi_dma_exit, | |
646 | .dma_setup = dw_spi_dma_setup, | |
647 | .can_dma = dw_spi_can_dma, | |
648 | .dma_transfer = dw_spi_dma_transfer, | |
649 | .dma_stop = dw_spi_dma_stop, | |
22d48ad7 JN |
650 | }; |
651 | ||
57784411 | 652 | void dw_spi_dma_setup_generic(struct dw_spi *dws) |
22d48ad7 | 653 | { |
57784411 | 654 | dws->dma_ops = &dw_spi_dma_generic_ops; |
22d48ad7 | 655 | } |
57784411 | 656 | EXPORT_SYMBOL_GPL(dw_spi_dma_setup_generic); |