1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * DMA driver for Xilinx ZynqMP DMA Engine
5 * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
8 #include <linux/bitops.h>
9 #include <linux/dmapool.h>
10 #include <linux/dma/xilinx_dma.h>
11 #include <linux/init.h>
12 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_address.h>
16 #include <linux/of_dma.h>
17 #include <linux/of_irq.h>
18 #include <linux/of_platform.h>
19 #include <linux/slab.h>
20 #include <linux/clk.h>
21 #include <linux/io-64-nonatomic-lo-hi.h>
22 #include <linux/pm_runtime.h>
24 #include "../dmaengine.h"
26 /* Register Offsets */
27 #define ZYNQMP_DMA_ISR 0x100
28 #define ZYNQMP_DMA_IMR 0x104
29 #define ZYNQMP_DMA_IER 0x108
30 #define ZYNQMP_DMA_IDS 0x10C
31 #define ZYNQMP_DMA_CTRL0 0x110
32 #define ZYNQMP_DMA_CTRL1 0x114
33 #define ZYNQMP_DMA_DATA_ATTR 0x120
34 #define ZYNQMP_DMA_DSCR_ATTR 0x124
35 #define ZYNQMP_DMA_SRC_DSCR_WRD0 0x128
36 #define ZYNQMP_DMA_SRC_DSCR_WRD1 0x12C
37 #define ZYNQMP_DMA_SRC_DSCR_WRD2 0x130
38 #define ZYNQMP_DMA_SRC_DSCR_WRD3 0x134
39 #define ZYNQMP_DMA_DST_DSCR_WRD0 0x138
40 #define ZYNQMP_DMA_DST_DSCR_WRD1 0x13C
41 #define ZYNQMP_DMA_DST_DSCR_WRD2 0x140
42 #define ZYNQMP_DMA_DST_DSCR_WRD3 0x144
43 #define ZYNQMP_DMA_SRC_START_LSB 0x158
44 #define ZYNQMP_DMA_SRC_START_MSB 0x15C
45 #define ZYNQMP_DMA_DST_START_LSB 0x160
46 #define ZYNQMP_DMA_DST_START_MSB 0x164
47 #define ZYNQMP_DMA_TOTAL_BYTE 0x188
48 #define ZYNQMP_DMA_RATE_CTRL 0x18C
49 #define ZYNQMP_DMA_IRQ_SRC_ACCT 0x190
50 #define ZYNQMP_DMA_IRQ_DST_ACCT 0x194
51 #define ZYNQMP_DMA_CTRL2 0x200
53 /* Interrupt registers bit field definitions */
54 #define ZYNQMP_DMA_DONE BIT(10)
55 #define ZYNQMP_DMA_AXI_WR_DATA BIT(9)
56 #define ZYNQMP_DMA_AXI_RD_DATA BIT(8)
57 #define ZYNQMP_DMA_AXI_RD_DST_DSCR BIT(7)
58 #define ZYNQMP_DMA_AXI_RD_SRC_DSCR BIT(6)
59 #define ZYNQMP_DMA_IRQ_DST_ACCT_ERR BIT(5)
60 #define ZYNQMP_DMA_IRQ_SRC_ACCT_ERR BIT(4)
61 #define ZYNQMP_DMA_BYTE_CNT_OVRFL BIT(3)
62 #define ZYNQMP_DMA_DST_DSCR_DONE BIT(2)
63 #define ZYNQMP_DMA_INV_APB BIT(0)
65 /* Control 0 register bit field definitions */
66 #define ZYNQMP_DMA_OVR_FETCH BIT(7)
67 #define ZYNQMP_DMA_POINT_TYPE_SG BIT(6)
68 #define ZYNQMP_DMA_RATE_CTRL_EN BIT(3)
70 /* Control 1 register bit field definitions */
71 #define ZYNQMP_DMA_SRC_ISSUE GENMASK(4, 0)
73 /* Data Attribute register bit field definitions */
74 #define ZYNQMP_DMA_ARBURST GENMASK(27, 26)
75 #define ZYNQMP_DMA_ARCACHE GENMASK(25, 22)
76 #define ZYNQMP_DMA_ARCACHE_OFST 22
77 #define ZYNQMP_DMA_ARQOS GENMASK(21, 18)
78 #define ZYNQMP_DMA_ARQOS_OFST 18
79 #define ZYNQMP_DMA_ARLEN GENMASK(17, 14)
80 #define ZYNQMP_DMA_ARLEN_OFST 14
81 #define ZYNQMP_DMA_AWBURST GENMASK(13, 12)
82 #define ZYNQMP_DMA_AWCACHE GENMASK(11, 8)
83 #define ZYNQMP_DMA_AWCACHE_OFST 8
84 #define ZYNQMP_DMA_AWQOS GENMASK(7, 4)
85 #define ZYNQMP_DMA_AWQOS_OFST 4
86 #define ZYNQMP_DMA_AWLEN GENMASK(3, 0)
87 #define ZYNQMP_DMA_AWLEN_OFST 0
89 /* Descriptor Attribute register bit field definitions */
90 #define ZYNQMP_DMA_AXCOHRNT BIT(8)
91 #define ZYNQMP_DMA_AXCACHE GENMASK(7, 4)
92 #define ZYNQMP_DMA_AXCACHE_OFST 4
93 #define ZYNQMP_DMA_AXQOS GENMASK(3, 0)
94 #define ZYNQMP_DMA_AXQOS_OFST 0
96 /* Control register 2 bit field definitions */
97 #define ZYNQMP_DMA_ENABLE BIT(0)
99 /* Buffer Descriptor definitions */
100 #define ZYNQMP_DMA_DESC_CTRL_STOP 0x10
101 #define ZYNQMP_DMA_DESC_CTRL_COMP_INT 0x4
102 #define ZYNQMP_DMA_DESC_CTRL_SIZE_256 0x2
103 #define ZYNQMP_DMA_DESC_CTRL_COHRNT 0x1
105 /* Interrupt Mask specific definitions */
106 #define ZYNQMP_DMA_INT_ERR (ZYNQMP_DMA_AXI_RD_DATA | \
107 ZYNQMP_DMA_AXI_WR_DATA | \
108 ZYNQMP_DMA_AXI_RD_DST_DSCR | \
109 ZYNQMP_DMA_AXI_RD_SRC_DSCR | \
111 #define ZYNQMP_DMA_INT_OVRFL (ZYNQMP_DMA_BYTE_CNT_OVRFL | \
112 ZYNQMP_DMA_IRQ_SRC_ACCT_ERR | \
113 ZYNQMP_DMA_IRQ_DST_ACCT_ERR)
114 #define ZYNQMP_DMA_INT_DONE (ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE)
115 #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK (ZYNQMP_DMA_INT_DONE | \
116 ZYNQMP_DMA_INT_ERR | \
117 ZYNQMP_DMA_INT_OVRFL | \
118 ZYNQMP_DMA_DST_DSCR_DONE)
120 /* Max number of descriptors per channel */
121 #define ZYNQMP_DMA_NUM_DESCS 32
123 /* Max transfer size per descriptor */
124 #define ZYNQMP_DMA_MAX_TRANS_LEN 0x40000000
126 /* Reset values for data attributes */
127 #define ZYNQMP_DMA_AXCACHE_VAL 0xF
128 #define ZYNQMP_DMA_ARLEN_RST_VAL 0xF
129 #define ZYNQMP_DMA_AWLEN_RST_VAL 0xF
131 #define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F
133 #define ZYNQMP_DMA_IDS_DEFAULT_MASK 0xFFF
135 /* Bus width in bits */
136 #define ZYNQMP_DMA_BUS_WIDTH_64 64
137 #define ZYNQMP_DMA_BUS_WIDTH_128 128
139 #define ZDMA_PM_TIMEOUT 100
141 #define ZYNQMP_DMA_DESC_SIZE(chan) (chan->desc_size)
143 #define to_chan(chan) container_of(chan, struct zynqmp_dma_chan, \
145 #define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \
149 * struct zynqmp_dma_desc_ll - Hw linked list descriptor
150 * @addr: Buffer address
151 * @size: Size of the buffer
152 * @ctrl: Control word
153 * @nxtdscraddr: Next descriptor base address
154 * @rsvd: Reserved field and for Hw internal use.
156 struct zynqmp_dma_desc_ll {
165 * struct zynqmp_dma_desc_sw - Per Transaction structure
166 * @src: Source address for simple mode dma
167 * @dst: Destination address for simple mode dma
168 * @len: Transfer length for simple mode dma
169 * @node: Node in the channel descriptor list
170 * @tx_list: List head for the current transfer
171 * @async_tx: Async transaction descriptor
172 * @src_v: Virtual address of the src descriptor
173 * @src_p: Physical address of the src descriptor
174 * @dst_v: Virtual address of the dst descriptor
175 * @dst_p: Physical address of the dst descriptor
177 struct zynqmp_dma_desc_sw {
181 struct list_head node;
182 struct list_head tx_list;
183 struct dma_async_tx_descriptor async_tx;
184 struct zynqmp_dma_desc_ll *src_v;
186 struct zynqmp_dma_desc_ll *dst_v;
191 * struct zynqmp_dma_chan - Driver specific DMA channel structure
192 * @zdev: Driver specific device structure
193 * @regs: Control registers offset
194 * @lock: Descriptor operation lock
195 * @pending_list: Descriptors waiting
196 * @free_list: Descriptors free
197 * @active_list: Descriptors active
198 * @sw_desc_pool: SW descriptor pool
199 * @done_list: Complete descriptors
200 * @common: DMA common channel
201 * @desc_pool_v: Statically allocated descriptor base
202 * @desc_pool_p: Physical allocated descriptor base
203 * @desc_free_cnt: Descriptor available count
204 * @dev: The dma device
206 * @is_dmacoherent: Tells whether dma operations are coherent or not
207 * @tasklet: Cleanup work after irq
208 * @idle : Channel status;
209 * @desc_size: Size of the low level descriptor
210 * @err: Channel has errors
211 * @bus_width: Bus width
212 * @src_burst_len: Source burst length
213 * @dst_burst_len: Dest burst length
215 struct zynqmp_dma_chan {
216 struct zynqmp_dma_device *zdev;
219 struct list_head pending_list;
220 struct list_head free_list;
221 struct list_head active_list;
222 struct zynqmp_dma_desc_sw *sw_desc_pool;
223 struct list_head done_list;
224 struct dma_chan common;
226 dma_addr_t desc_pool_p;
231 struct tasklet_struct tasklet;
241 * struct zynqmp_dma_device - DMA device structure
242 * @dev: Device Structure
243 * @common: DMA device structure
244 * @chan: Driver specific DMA channel
245 * @clk_main: Pointer to main clock
246 * @clk_apb: Pointer to apb clock
248 struct zynqmp_dma_device {
250 struct dma_device common;
251 struct zynqmp_dma_chan *chan;
252 struct clk *clk_main;
256 static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg,
259 lo_hi_writeq(value, chan->regs + reg);
263 * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller
264 * @chan: ZynqMP DMA DMA channel pointer
265 * @desc: Transaction descriptor pointer
267 static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan,
268 struct zynqmp_dma_desc_sw *desc)
273 zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, addr);
275 zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, addr);
279 * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor
280 * @chan: ZynqMP DMA channel pointer
281 * @desc: Hw descriptor pointer
283 static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan,
286 struct zynqmp_dma_desc_ll *hw = (struct zynqmp_dma_desc_ll *)desc;
288 hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP;
290 hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP;
294 * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor
295 * @chan: ZynqMP DMA channel pointer
296 * @sdesc: Hw descriptor pointer
297 * @src: Source buffer address
298 * @dst: Destination buffer address
299 * @len: Transfer length
300 * @prev: Previous hw descriptor pointer
302 static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan,
303 struct zynqmp_dma_desc_ll *sdesc,
304 dma_addr_t src, dma_addr_t dst, size_t len,
305 struct zynqmp_dma_desc_ll *prev)
307 struct zynqmp_dma_desc_ll *ddesc = sdesc + 1;
309 sdesc->size = ddesc->size = len;
313 sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256;
314 if (chan->is_dmacoherent) {
315 sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;
316 ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;
320 dma_addr_t addr = chan->desc_pool_p +
321 ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v);
323 prev->nxtdscraddr = addr;
324 ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan);
329 * zynqmp_dma_init - Initialize the channel
330 * @chan: ZynqMP DMA channel pointer
332 static void zynqmp_dma_init(struct zynqmp_dma_chan *chan)
336 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
337 val = readl(chan->regs + ZYNQMP_DMA_ISR);
338 writel(val, chan->regs + ZYNQMP_DMA_ISR);
340 if (chan->is_dmacoherent) {
341 val = ZYNQMP_DMA_AXCOHRNT;
342 val = (val & ~ZYNQMP_DMA_AXCACHE) |
343 (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AXCACHE_OFST);
344 writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR);
347 val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);
348 if (chan->is_dmacoherent) {
349 val = (val & ~ZYNQMP_DMA_ARCACHE) |
350 (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_ARCACHE_OFST);
351 val = (val & ~ZYNQMP_DMA_AWCACHE) |
352 (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AWCACHE_OFST);
354 writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);
356 /* Clearing the interrupt account rgisters */
357 val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
358 val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
364 * zynqmp_dma_tx_submit - Submit DMA transaction
365 * @tx: Async transaction descriptor pointer
367 * Return: cookie value
369 static dma_cookie_t zynqmp_dma_tx_submit(struct dma_async_tx_descriptor *tx)
371 struct zynqmp_dma_chan *chan = to_chan(tx->chan);
372 struct zynqmp_dma_desc_sw *desc, *new;
374 unsigned long irqflags;
376 new = tx_to_desc(tx);
377 spin_lock_irqsave(&chan->lock, irqflags);
378 cookie = dma_cookie_assign(tx);
380 if (!list_empty(&chan->pending_list)) {
381 desc = list_last_entry(&chan->pending_list,
382 struct zynqmp_dma_desc_sw, node);
383 if (!list_empty(&desc->tx_list))
384 desc = list_last_entry(&desc->tx_list,
385 struct zynqmp_dma_desc_sw, node);
386 desc->src_v->nxtdscraddr = new->src_p;
387 desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;
388 desc->dst_v->nxtdscraddr = new->dst_p;
389 desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;
392 list_add_tail(&new->node, &chan->pending_list);
393 spin_unlock_irqrestore(&chan->lock, irqflags);
399 * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool
400 * @chan: ZynqMP DMA channel pointer
402 * Return: The sw descriptor
404 static struct zynqmp_dma_desc_sw *
405 zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan)
407 struct zynqmp_dma_desc_sw *desc;
408 unsigned long irqflags;
410 spin_lock_irqsave(&chan->lock, irqflags);
411 desc = list_first_entry(&chan->free_list,
412 struct zynqmp_dma_desc_sw, node);
413 list_del(&desc->node);
414 spin_unlock_irqrestore(&chan->lock, irqflags);
416 INIT_LIST_HEAD(&desc->tx_list);
417 /* Clear the src and dst descriptor memory */
418 memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));
419 memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));
425 * zynqmp_dma_free_descriptor - Issue pending transactions
426 * @chan: ZynqMP DMA channel pointer
427 * @sdesc: Transaction descriptor pointer
429 static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan,
430 struct zynqmp_dma_desc_sw *sdesc)
432 struct zynqmp_dma_desc_sw *child, *next;
434 chan->desc_free_cnt++;
435 list_add_tail(&sdesc->node, &chan->free_list);
436 list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {
437 chan->desc_free_cnt++;
438 list_move_tail(&child->node, &chan->free_list);
443 * zynqmp_dma_free_desc_list - Free descriptors list
444 * @chan: ZynqMP DMA channel pointer
445 * @list: List to parse and delete the descriptor
447 static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan,
448 struct list_head *list)
450 struct zynqmp_dma_desc_sw *desc, *next;
452 list_for_each_entry_safe(desc, next, list, node)
453 zynqmp_dma_free_descriptor(chan, desc);
457 * zynqmp_dma_alloc_chan_resources - Allocate channel resources
458 * @dchan: DMA channel
460 * Return: Number of descriptors on success and failure value on error
462 static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
464 struct zynqmp_dma_chan *chan = to_chan(dchan);
465 struct zynqmp_dma_desc_sw *desc;
468 ret = pm_runtime_get_sync(chan->dev);
472 chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc),
474 if (!chan->sw_desc_pool)
478 chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;
480 INIT_LIST_HEAD(&chan->free_list);
482 for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) {
483 desc = chan->sw_desc_pool + i;
484 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
485 desc->async_tx.tx_submit = zynqmp_dma_tx_submit;
486 list_add_tail(&desc->node, &chan->free_list);
489 chan->desc_pool_v = dma_alloc_coherent(chan->dev,
490 (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),
491 &chan->desc_pool_p, GFP_KERNEL);
492 if (!chan->desc_pool_v)
495 for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) {
496 desc = chan->sw_desc_pool + i;
497 desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v +
498 (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2));
499 desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1);
500 desc->src_p = chan->desc_pool_p +
501 (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2);
502 desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan);
505 return ZYNQMP_DMA_NUM_DESCS;
509 * zynqmp_dma_start - Start DMA channel
510 * @chan: ZynqMP DMA channel pointer
512 static void zynqmp_dma_start(struct zynqmp_dma_chan *chan)
514 writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);
515 writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
517 writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);
521 * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt
522 * @chan: ZynqMP DMA channel pointer
523 * @status: Interrupt status value
525 static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status)
527 if (status & ZYNQMP_DMA_BYTE_CNT_OVRFL)
528 writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
529 if (status & ZYNQMP_DMA_IRQ_DST_ACCT_ERR)
530 readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
531 if (status & ZYNQMP_DMA_IRQ_SRC_ACCT_ERR)
532 readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
535 static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)
539 val = readl(chan->regs + ZYNQMP_DMA_CTRL0);
540 val |= ZYNQMP_DMA_POINT_TYPE_SG;
541 writel(val, chan->regs + ZYNQMP_DMA_CTRL0);
543 val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);
544 val = (val & ~ZYNQMP_DMA_ARLEN) |
545 (chan->src_burst_len << ZYNQMP_DMA_ARLEN_OFST);
546 val = (val & ~ZYNQMP_DMA_AWLEN) |
547 (chan->dst_burst_len << ZYNQMP_DMA_AWLEN_OFST);
548 writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);
552 * zynqmp_dma_device_config - Zynqmp dma device configuration
553 * @dchan: DMA channel
554 * @config: DMA device config
558 static int zynqmp_dma_device_config(struct dma_chan *dchan,
559 struct dma_slave_config *config)
561 struct zynqmp_dma_chan *chan = to_chan(dchan);
563 chan->src_burst_len = config->src_maxburst;
564 chan->dst_burst_len = config->dst_maxburst;
570 * zynqmp_dma_start_transfer - Initiate the new transfer
571 * @chan: ZynqMP DMA channel pointer
573 static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan)
575 struct zynqmp_dma_desc_sw *desc;
580 zynqmp_dma_config(chan);
582 desc = list_first_entry_or_null(&chan->pending_list,
583 struct zynqmp_dma_desc_sw, node);
587 list_splice_tail_init(&chan->pending_list, &chan->active_list);
588 zynqmp_dma_update_desc_to_ctrlr(chan, desc);
589 zynqmp_dma_start(chan);
594 * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors
595 * @chan: ZynqMP DMA channel
597 static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan)
599 struct zynqmp_dma_desc_sw *desc, *next;
601 list_for_each_entry_safe(desc, next, &chan->done_list, node) {
602 dma_async_tx_callback callback;
603 void *callback_param;
605 list_del(&desc->node);
607 callback = desc->async_tx.callback;
608 callback_param = desc->async_tx.callback_param;
610 spin_unlock(&chan->lock);
611 callback(callback_param);
612 spin_lock(&chan->lock);
615 /* Run any dependencies, then free the descriptor */
616 zynqmp_dma_free_descriptor(chan, desc);
621 * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete
622 * @chan: ZynqMP DMA channel pointer
624 static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan)
626 struct zynqmp_dma_desc_sw *desc;
628 desc = list_first_entry_or_null(&chan->active_list,
629 struct zynqmp_dma_desc_sw, node);
632 list_del(&desc->node);
633 dma_cookie_complete(&desc->async_tx);
634 list_add_tail(&desc->node, &chan->done_list);
638 * zynqmp_dma_issue_pending - Issue pending transactions
639 * @dchan: DMA channel pointer
641 static void zynqmp_dma_issue_pending(struct dma_chan *dchan)
643 struct zynqmp_dma_chan *chan = to_chan(dchan);
644 unsigned long irqflags;
646 spin_lock_irqsave(&chan->lock, irqflags);
647 zynqmp_dma_start_transfer(chan);
648 spin_unlock_irqrestore(&chan->lock, irqflags);
652 * zynqmp_dma_free_descriptors - Free channel descriptors
653 * @chan: ZynqMP DMA channel pointer
655 static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan)
657 zynqmp_dma_free_desc_list(chan, &chan->active_list);
658 zynqmp_dma_free_desc_list(chan, &chan->pending_list);
659 zynqmp_dma_free_desc_list(chan, &chan->done_list);
663 * zynqmp_dma_free_chan_resources - Free channel resources
664 * @dchan: DMA channel pointer
666 static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan)
668 struct zynqmp_dma_chan *chan = to_chan(dchan);
669 unsigned long irqflags;
671 spin_lock_irqsave(&chan->lock, irqflags);
672 zynqmp_dma_free_descriptors(chan);
673 spin_unlock_irqrestore(&chan->lock, irqflags);
674 dma_free_coherent(chan->dev,
675 (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),
676 chan->desc_pool_v, chan->desc_pool_p);
677 kfree(chan->sw_desc_pool);
678 pm_runtime_mark_last_busy(chan->dev);
679 pm_runtime_put_autosuspend(chan->dev);
683 * zynqmp_dma_reset - Reset the channel
684 * @chan: ZynqMP DMA channel pointer
686 static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan)
688 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
690 zynqmp_dma_complete_descriptor(chan);
691 zynqmp_dma_chan_desc_cleanup(chan);
692 zynqmp_dma_free_descriptors(chan);
693 zynqmp_dma_init(chan);
697 * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler
699 * @data: Pointer to the ZynqMP DMA channel structure
701 * Return: IRQ_HANDLED/IRQ_NONE
703 static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data)
705 struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data;
706 u32 isr, imr, status;
707 irqreturn_t ret = IRQ_NONE;
709 isr = readl(chan->regs + ZYNQMP_DMA_ISR);
710 imr = readl(chan->regs + ZYNQMP_DMA_IMR);
713 writel(isr, chan->regs + ZYNQMP_DMA_ISR);
714 if (status & ZYNQMP_DMA_INT_DONE) {
715 tasklet_schedule(&chan->tasklet);
719 if (status & ZYNQMP_DMA_DONE)
722 if (status & ZYNQMP_DMA_INT_ERR) {
724 tasklet_schedule(&chan->tasklet);
725 dev_err(chan->dev, "Channel %p has errors\n", chan);
729 if (status & ZYNQMP_DMA_INT_OVRFL) {
730 zynqmp_dma_handle_ovfl_int(chan, status);
731 dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);
739 * zynqmp_dma_do_tasklet - Schedule completion tasklet
740 * @data: Pointer to the ZynqMP DMA channel structure
742 static void zynqmp_dma_do_tasklet(unsigned long data)
744 struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data;
746 unsigned long irqflags;
748 spin_lock_irqsave(&chan->lock, irqflags);
751 zynqmp_dma_reset(chan);
756 count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
759 zynqmp_dma_complete_descriptor(chan);
760 zynqmp_dma_chan_desc_cleanup(chan);
765 zynqmp_dma_start_transfer(chan);
768 spin_unlock_irqrestore(&chan->lock, irqflags);
772 * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel
773 * @dchan: DMA channel pointer
777 static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan)
779 struct zynqmp_dma_chan *chan = to_chan(dchan);
780 unsigned long irqflags;
782 spin_lock_irqsave(&chan->lock, irqflags);
783 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
784 zynqmp_dma_free_descriptors(chan);
785 spin_unlock_irqrestore(&chan->lock, irqflags);
791 * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction
792 * @dchan: DMA channel
793 * @dma_dst: Destination buffer address
794 * @dma_src: Source buffer address
795 * @len: Transfer length
796 * @flags: transfer ack flags
798 * Return: Async transaction descriptor on success and NULL on failure
800 static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy(
801 struct dma_chan *dchan, dma_addr_t dma_dst,
802 dma_addr_t dma_src, size_t len, ulong flags)
804 struct zynqmp_dma_chan *chan;
805 struct zynqmp_dma_desc_sw *new, *first = NULL;
806 void *desc = NULL, *prev = NULL;
809 unsigned long irqflags;
811 chan = to_chan(dchan);
813 desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN);
815 spin_lock_irqsave(&chan->lock, irqflags);
816 if (desc_cnt > chan->desc_free_cnt) {
817 spin_unlock_irqrestore(&chan->lock, irqflags);
818 dev_dbg(chan->dev, "chan %p descs are not available\n", chan);
821 chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;
822 spin_unlock_irqrestore(&chan->lock, irqflags);
825 /* Allocate and populate the descriptor */
826 new = zynqmp_dma_get_descriptor(chan);
828 copy = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN);
829 desc = (struct zynqmp_dma_desc_ll *)new->src_v;
830 zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src,
831 dma_dst, copy, prev);
839 list_add_tail(&new->node, &first->tx_list);
842 zynqmp_dma_desc_config_eod(chan, desc);
843 async_tx_ack(&first->async_tx);
844 first->async_tx.flags = flags;
845 return &first->async_tx;
849 * zynqmp_dma_chan_remove - Channel remove function
850 * @chan: ZynqMP DMA channel pointer
852 static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)
858 devm_free_irq(chan->zdev->dev, chan->irq, chan);
859 tasklet_kill(&chan->tasklet);
860 list_del(&chan->common.device_node);
864 * zynqmp_dma_chan_probe - Per Channel Probing
865 * @zdev: Driver specific device structure
866 * @pdev: Pointer to the platform_device structure
868 * Return: '0' on success and failure value on error
870 static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
871 struct platform_device *pdev)
873 struct zynqmp_dma_chan *chan;
874 struct resource *res;
875 struct device_node *node = pdev->dev.of_node;
878 chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);
881 chan->dev = zdev->dev;
884 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
885 chan->regs = devm_ioremap_resource(&pdev->dev, res);
886 if (IS_ERR(chan->regs))
887 return PTR_ERR(chan->regs);
889 chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;
890 chan->dst_burst_len = ZYNQMP_DMA_AWLEN_RST_VAL;
891 chan->src_burst_len = ZYNQMP_DMA_ARLEN_RST_VAL;
892 err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);
894 dev_err(&pdev->dev, "missing xlnx,bus-width property\n");
898 if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 &&
899 chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) {
900 dev_err(zdev->dev, "invalid bus-width value");
904 chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent");
906 tasklet_init(&chan->tasklet, zynqmp_dma_do_tasklet, (ulong)chan);
907 spin_lock_init(&chan->lock);
908 INIT_LIST_HEAD(&chan->active_list);
909 INIT_LIST_HEAD(&chan->pending_list);
910 INIT_LIST_HEAD(&chan->done_list);
911 INIT_LIST_HEAD(&chan->free_list);
913 dma_cookie_init(&chan->common);
914 chan->common.device = &zdev->common;
915 list_add_tail(&chan->common.device_node, &zdev->common.channels);
917 zynqmp_dma_init(chan);
918 chan->irq = platform_get_irq(pdev, 0);
921 err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,
926 chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);
932 * of_zynqmp_dma_xlate - Translation function
933 * @dma_spec: Pointer to DMA specifier as found in the device tree
934 * @ofdma: Pointer to DMA controller data
936 * Return: DMA channel pointer on success and NULL on error
938 static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec,
939 struct of_dma *ofdma)
941 struct zynqmp_dma_device *zdev = ofdma->of_dma_data;
943 return dma_get_slave_channel(&zdev->chan->common);
947 * zynqmp_dma_suspend - Suspend method for the driver
948 * @dev: Address of the device structure
950 * Put the driver into low power mode.
951 * Return: 0 on success and failure value on error
953 static int __maybe_unused zynqmp_dma_suspend(struct device *dev)
955 if (!device_may_wakeup(dev))
956 return pm_runtime_force_suspend(dev);
962 * zynqmp_dma_resume - Resume from suspend
963 * @dev: Address of the device structure
965 * Resume operation after suspend.
966 * Return: 0 on success and failure value on error
968 static int __maybe_unused zynqmp_dma_resume(struct device *dev)
970 if (!device_may_wakeup(dev))
971 return pm_runtime_force_resume(dev);
977 * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver
978 * @dev: Address of the device structure
980 * Put the driver into low power mode.
983 static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev)
985 struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
987 clk_disable_unprepare(zdev->clk_main);
988 clk_disable_unprepare(zdev->clk_apb);
994 * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
995 * @dev: Address of the device structure
997 * Put the driver into low power mode.
1000 static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev)
1002 struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
1005 err = clk_prepare_enable(zdev->clk_main);
1007 dev_err(dev, "Unable to enable main clock.\n");
1011 err = clk_prepare_enable(zdev->clk_apb);
1013 dev_err(dev, "Unable to enable apb clock.\n");
1014 clk_disable_unprepare(zdev->clk_main);
1021 static const struct dev_pm_ops zynqmp_dma_dev_pm_ops = {
1022 SET_SYSTEM_SLEEP_PM_OPS(zynqmp_dma_suspend, zynqmp_dma_resume)
1023 SET_RUNTIME_PM_OPS(zynqmp_dma_runtime_suspend,
1024 zynqmp_dma_runtime_resume, NULL)
1028 * zynqmp_dma_probe - Driver probe function
1029 * @pdev: Pointer to the platform_device structure
1031 * Return: '0' on success and failure value on error
1033 static int zynqmp_dma_probe(struct platform_device *pdev)
1035 struct zynqmp_dma_device *zdev;
1036 struct dma_device *p;
1039 zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL);
1043 zdev->dev = &pdev->dev;
1044 INIT_LIST_HEAD(&zdev->common.channels);
1046 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
1047 dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask);
1050 p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;
1051 p->device_terminate_all = zynqmp_dma_device_terminate_all;
1052 p->device_issue_pending = zynqmp_dma_issue_pending;
1053 p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources;
1054 p->device_free_chan_resources = zynqmp_dma_free_chan_resources;
1055 p->device_tx_status = dma_cookie_status;
1056 p->device_config = zynqmp_dma_device_config;
1057 p->dev = &pdev->dev;
1059 zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");
1060 if (IS_ERR(zdev->clk_main)) {
1061 dev_err(&pdev->dev, "main clock not found.\n");
1062 return PTR_ERR(zdev->clk_main);
1065 zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");
1066 if (IS_ERR(zdev->clk_apb)) {
1067 dev_err(&pdev->dev, "apb clock not found.\n");
1068 return PTR_ERR(zdev->clk_apb);
1071 platform_set_drvdata(pdev, zdev);
1072 pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);
1073 pm_runtime_use_autosuspend(zdev->dev);
1074 pm_runtime_enable(zdev->dev);
1075 pm_runtime_get_sync(zdev->dev);
1076 if (!pm_runtime_enabled(zdev->dev)) {
1077 ret = zynqmp_dma_runtime_resume(zdev->dev);
1082 ret = zynqmp_dma_chan_probe(zdev, pdev);
1084 dev_err(&pdev->dev, "Probing channel failed\n");
1085 goto err_disable_pm;
1088 p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);
1089 p->src_addr_widths = BIT(zdev->chan->bus_width / 8);
1091 dma_async_device_register(&zdev->common);
1093 ret = of_dma_controller_register(pdev->dev.of_node,
1094 of_zynqmp_dma_xlate, zdev);
1096 dev_err(&pdev->dev, "Unable to register DMA to DT\n");
1097 dma_async_device_unregister(&zdev->common);
1098 goto free_chan_resources;
1101 pm_runtime_mark_last_busy(zdev->dev);
1102 pm_runtime_put_sync_autosuspend(zdev->dev);
1104 dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");
1108 free_chan_resources:
1109 zynqmp_dma_chan_remove(zdev->chan);
1111 if (!pm_runtime_enabled(zdev->dev))
1112 zynqmp_dma_runtime_suspend(zdev->dev);
1113 pm_runtime_disable(zdev->dev);
1118 * zynqmp_dma_remove - Driver remove function
1119 * @pdev: Pointer to the platform_device structure
1121 * Return: Always '0'
1123 static int zynqmp_dma_remove(struct platform_device *pdev)
1125 struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev);
1127 of_dma_controller_free(pdev->dev.of_node);
1128 dma_async_device_unregister(&zdev->common);
1130 zynqmp_dma_chan_remove(zdev->chan);
1131 pm_runtime_disable(zdev->dev);
1132 if (!pm_runtime_enabled(zdev->dev))
1133 zynqmp_dma_runtime_suspend(zdev->dev);
1138 static const struct of_device_id zynqmp_dma_of_match[] = {
1139 { .compatible = "xlnx,zynqmp-dma-1.0", },
1142 MODULE_DEVICE_TABLE(of, zynqmp_dma_of_match);
1144 static struct platform_driver zynqmp_dma_driver = {
1146 .name = "xilinx-zynqmp-dma",
1147 .of_match_table = zynqmp_dma_of_match,
1148 .pm = &zynqmp_dma_dev_pm_ops,
1150 .probe = zynqmp_dma_probe,
1151 .remove = zynqmp_dma_remove,
1154 module_platform_driver(zynqmp_dma_driver);
1156 MODULE_LICENSE("GPL");
1157 MODULE_AUTHOR("Xilinx, Inc.");
1158 MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver");