1 /* linux/drivers/dma/pl330.c
3 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
4 * Jaswinder Singh <jassi.brar@samsung.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
13 #include <linux/init.h>
14 #include <linux/slab.h>
15 #include <linux/module.h>
16 #include <linux/dmaengine.h>
17 #include <linux/interrupt.h>
18 #include <linux/amba/bus.h>
19 #include <linux/amba/pl330.h>
20 #include <linux/pm_runtime.h>
22 #define NR_DEFAULT_DESC 16
25 /* In the DMAC pool */
28 * Allocted to some channel during prep_xxx
29 * Also may be sitting on the work_list.
33 * Sitting on the work_list and already submitted
34 * to the PL330 core. Not more than two descriptors
35 * of a channel can be BUSY at any time.
39 * Sitting on the channel work_list but xfer done
45 struct dma_pl330_chan {
46 /* Schedule desc completion */
47 struct tasklet_struct task;
49 /* DMA-Engine Channel */
52 /* Last completed cookie */
53 dma_cookie_t completed;
55 /* List of to be xfered descriptors */
56 struct list_head work_list;
58 /* Pointer to the DMAC that manages this channel,
59 * NULL if the channel is available to be acquired.
60 * As the parent, this DMAC also provides descriptors
63 struct dma_pl330_dmac *dmac;
65 /* To protect channel manipulation */
68 /* Token of a hardware channel thread of PL330 DMAC
69 * NULL if the channel is available to be acquired.
74 struct dma_pl330_dmac {
75 struct pl330_info pif;
77 /* DMA-Engine Device */
78 struct dma_device ddma;
80 /* Pool of descriptors available for the DMAC's channels */
81 struct list_head desc_pool;
82 /* To protect desc_pool manipulation */
85 /* Peripheral channels connected to this DMAC */
86 struct dma_pl330_chan *peripherals; /* keep at end */
91 struct dma_pl330_desc {
92 /* To attach to a queue as child */
93 struct list_head node;
95 /* Descriptor for the DMA Engine API */
96 struct dma_async_tx_descriptor txd;
98 /* Xfer for PL330 core */
101 struct pl330_reqcfg rqcfg;
102 struct pl330_req req;
104 enum desc_status status;
106 /* The channel which currently holds this desc */
107 struct dma_pl330_chan *pchan;
110 static inline struct dma_pl330_chan *
111 to_pchan(struct dma_chan *ch)
116 return container_of(ch, struct dma_pl330_chan, chan);
119 static inline struct dma_pl330_desc *
120 to_desc(struct dma_async_tx_descriptor *tx)
122 return container_of(tx, struct dma_pl330_desc, txd);
125 static inline void free_desc_list(struct list_head *list)
127 struct dma_pl330_dmac *pdmac;
128 struct dma_pl330_desc *desc;
129 struct dma_pl330_chan *pch;
132 if (list_empty(list))
135 /* Finish off the work list */
136 list_for_each_entry(desc, list, node) {
137 dma_async_tx_callback callback;
140 /* All desc in a list belong to same channel */
142 callback = desc->txd.callback;
143 param = desc->txd.callback_param;
153 spin_lock_irqsave(&pdmac->pool_lock, flags);
154 list_splice_tail_init(list, &pdmac->desc_pool);
155 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
158 static inline void fill_queue(struct dma_pl330_chan *pch)
160 struct dma_pl330_desc *desc;
163 list_for_each_entry(desc, &pch->work_list, node) {
165 /* If already submitted */
166 if (desc->status == BUSY)
169 ret = pl330_submit_req(pch->pl330_chid,
174 } else if (ret == -EAGAIN) {
175 /* QFull or DMAC Dying */
178 /* Unacceptable request */
180 dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
181 __func__, __LINE__, desc->txd.cookie);
182 tasklet_schedule(&pch->task);
187 static void pl330_tasklet(unsigned long data)
189 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
190 struct dma_pl330_desc *desc, *_dt;
194 spin_lock_irqsave(&pch->lock, flags);
196 /* Pick up ripe tomatoes */
197 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
198 if (desc->status == DONE) {
199 pch->completed = desc->txd.cookie;
200 list_move_tail(&desc->node, &list);
203 /* Try to submit a req imm. next to the last completed cookie */
206 /* Make sure the PL330 Channel thread is active */
207 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
209 spin_unlock_irqrestore(&pch->lock, flags);
211 free_desc_list(&list);
214 static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
216 struct dma_pl330_desc *desc = token;
217 struct dma_pl330_chan *pch = desc->pchan;
220 /* If desc aborted */
224 spin_lock_irqsave(&pch->lock, flags);
228 spin_unlock_irqrestore(&pch->lock, flags);
230 tasklet_schedule(&pch->task);
233 static int pl330_alloc_chan_resources(struct dma_chan *chan)
235 struct dma_pl330_chan *pch = to_pchan(chan);
236 struct dma_pl330_dmac *pdmac = pch->dmac;
239 spin_lock_irqsave(&pch->lock, flags);
241 pch->completed = chan->cookie = 1;
243 pch->pl330_chid = pl330_request_channel(&pdmac->pif);
244 if (!pch->pl330_chid) {
245 spin_unlock_irqrestore(&pch->lock, flags);
249 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
251 spin_unlock_irqrestore(&pch->lock, flags);
256 static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
258 struct dma_pl330_chan *pch = to_pchan(chan);
259 struct dma_pl330_desc *desc;
262 /* Only supports DMA_TERMINATE_ALL */
263 if (cmd != DMA_TERMINATE_ALL)
266 spin_lock_irqsave(&pch->lock, flags);
268 /* FLUSH the PL330 Channel thread */
269 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
271 /* Mark all desc done */
272 list_for_each_entry(desc, &pch->work_list, node)
275 spin_unlock_irqrestore(&pch->lock, flags);
277 pl330_tasklet((unsigned long) pch);
282 static void pl330_free_chan_resources(struct dma_chan *chan)
284 struct dma_pl330_chan *pch = to_pchan(chan);
287 spin_lock_irqsave(&pch->lock, flags);
289 tasklet_kill(&pch->task);
291 pl330_release_channel(pch->pl330_chid);
292 pch->pl330_chid = NULL;
294 spin_unlock_irqrestore(&pch->lock, flags);
297 static enum dma_status
298 pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
299 struct dma_tx_state *txstate)
301 struct dma_pl330_chan *pch = to_pchan(chan);
302 dma_cookie_t last_done, last_used;
305 last_done = pch->completed;
306 last_used = chan->cookie;
308 ret = dma_async_is_complete(cookie, last_done, last_used);
310 dma_set_tx_state(txstate, last_done, last_used, 0);
315 static void pl330_issue_pending(struct dma_chan *chan)
317 pl330_tasklet((unsigned long) to_pchan(chan));
321 * We returned the last one of the circular list of descriptor(s)
322 * from prep_xxx, so the argument to submit corresponds to the last
323 * descriptor of the list.
325 static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
327 struct dma_pl330_desc *desc, *last = to_desc(tx);
328 struct dma_pl330_chan *pch = to_pchan(tx->chan);
332 spin_lock_irqsave(&pch->lock, flags);
334 /* Assign cookies to all nodes */
335 cookie = tx->chan->cookie;
337 while (!list_empty(&last->node)) {
338 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
342 desc->txd.cookie = cookie;
344 list_move_tail(&desc->node, &pch->work_list);
349 last->txd.cookie = cookie;
351 list_add_tail(&last->node, &pch->work_list);
353 tx->chan->cookie = cookie;
355 spin_unlock_irqrestore(&pch->lock, flags);
360 static inline void _init_desc(struct dma_pl330_desc *desc)
363 desc->req.x = &desc->px;
364 desc->req.token = desc;
365 desc->rqcfg.swap = SWAP_NO;
366 desc->rqcfg.privileged = 0;
367 desc->rqcfg.insnaccess = 0;
368 desc->rqcfg.scctl = SCCTRL0;
369 desc->rqcfg.dcctl = DCCTRL0;
370 desc->req.cfg = &desc->rqcfg;
371 desc->req.xfer_cb = dma_pl330_rqcb;
372 desc->txd.tx_submit = pl330_tx_submit;
374 INIT_LIST_HEAD(&desc->node);
377 /* Returns the number of descriptors added to the DMAC pool */
378 int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
380 struct dma_pl330_desc *desc;
387 desc = kmalloc(count * sizeof(*desc), flg);
391 spin_lock_irqsave(&pdmac->pool_lock, flags);
393 for (i = 0; i < count; i++) {
394 _init_desc(&desc[i]);
395 list_add_tail(&desc[i].node, &pdmac->desc_pool);
398 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
403 static struct dma_pl330_desc *
404 pluck_desc(struct dma_pl330_dmac *pdmac)
406 struct dma_pl330_desc *desc = NULL;
412 spin_lock_irqsave(&pdmac->pool_lock, flags);
414 if (!list_empty(&pdmac->desc_pool)) {
415 desc = list_entry(pdmac->desc_pool.next,
416 struct dma_pl330_desc, node);
418 list_del_init(&desc->node);
421 desc->txd.callback = NULL;
424 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
429 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
431 struct dma_pl330_dmac *pdmac = pch->dmac;
432 struct dma_pl330_peri *peri = pch->chan.private;
433 struct dma_pl330_desc *desc;
435 /* Pluck one desc from the pool of DMAC */
436 desc = pluck_desc(pdmac);
438 /* If the DMAC pool is empty, alloc new */
440 if (!add_desc(pdmac, GFP_ATOMIC, 1))
444 desc = pluck_desc(pdmac);
446 dev_err(pch->dmac->pif.dev,
447 "%s:%d ALERT!\n", __func__, __LINE__);
452 /* Initialize the descriptor */
454 desc->txd.cookie = 0;
455 async_tx_ack(&desc->txd);
458 desc->req.rqtype = peri->rqtype;
459 desc->req.peri = peri->peri_id;
461 desc->req.rqtype = MEMTOMEM;
465 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
470 static inline void fill_px(struct pl330_xfer *px,
471 dma_addr_t dst, dma_addr_t src, size_t len)
479 static struct dma_pl330_desc *
480 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
481 dma_addr_t src, size_t len)
483 struct dma_pl330_desc *desc = pl330_get_desc(pch);
486 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
492 * Ideally we should lookout for reqs bigger than
493 * those that can be programmed with 256 bytes of
494 * MC buffer, but considering a req size is seldom
495 * going to be word-unaligned and more than 200MB,
497 * Also, should the limit is reached we'd rather
498 * have the platform increase MC buffer size than
499 * complicating this API driver.
501 fill_px(&desc->px, dst, src, len);
506 /* Call after fixing burst size */
507 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
509 struct dma_pl330_chan *pch = desc->pchan;
510 struct pl330_info *pi = &pch->dmac->pif;
513 burst_len = pi->pcfg.data_bus_width / 8;
514 burst_len *= pi->pcfg.data_buf_dep;
515 burst_len >>= desc->rqcfg.brst_size;
517 /* src/dst_burst_len can't be more than 16 */
521 while (burst_len > 1) {
522 if (!(len % (burst_len << desc->rqcfg.brst_size)))
530 static struct dma_async_tx_descriptor *
531 pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
532 dma_addr_t src, size_t len, unsigned long flags)
534 struct dma_pl330_desc *desc;
535 struct dma_pl330_chan *pch = to_pchan(chan);
536 struct dma_pl330_peri *peri = chan->private;
537 struct pl330_info *pi;
540 if (unlikely(!pch || !len))
543 if (peri && peri->rqtype != MEMTOMEM)
546 pi = &pch->dmac->pif;
548 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
552 desc->rqcfg.src_inc = 1;
553 desc->rqcfg.dst_inc = 1;
555 /* Select max possible burst size */
556 burst = pi->pcfg.data_bus_width / 8;
564 desc->rqcfg.brst_size = 0;
565 while (burst != (1 << desc->rqcfg.brst_size))
566 desc->rqcfg.brst_size++;
568 desc->rqcfg.brst_len = get_burst_len(desc, len);
570 desc->txd.flags = flags;
575 static struct dma_async_tx_descriptor *
576 pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
577 unsigned int sg_len, enum dma_data_direction direction,
580 struct dma_pl330_desc *first, *desc = NULL;
581 struct dma_pl330_chan *pch = to_pchan(chan);
582 struct dma_pl330_peri *peri = chan->private;
583 struct scatterlist *sg;
588 if (unlikely(!pch || !sgl || !sg_len || !peri))
591 /* Make sure the direction is consistent */
592 if ((direction == DMA_TO_DEVICE &&
593 peri->rqtype != MEMTODEV) ||
594 (direction == DMA_FROM_DEVICE &&
595 peri->rqtype != DEVTOMEM)) {
596 dev_err(pch->dmac->pif.dev, "%s:%d Invalid Direction\n",
601 addr = peri->fifo_addr;
602 burst_size = peri->burst_sz;
606 for_each_sg(sgl, sg, sg_len, i) {
608 desc = pl330_get_desc(pch);
610 struct dma_pl330_dmac *pdmac = pch->dmac;
612 dev_err(pch->dmac->pif.dev,
613 "%s:%d Unable to fetch desc\n",
618 spin_lock_irqsave(&pdmac->pool_lock, flags);
620 while (!list_empty(&first->node)) {
621 desc = list_entry(first->node.next,
622 struct dma_pl330_desc, node);
623 list_move_tail(&desc->node, &pdmac->desc_pool);
626 list_move_tail(&first->node, &pdmac->desc_pool);
628 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
636 list_add_tail(&desc->node, &first->node);
638 if (direction == DMA_TO_DEVICE) {
639 desc->rqcfg.src_inc = 1;
640 desc->rqcfg.dst_inc = 0;
642 addr, sg_dma_address(sg), sg_dma_len(sg));
644 desc->rqcfg.src_inc = 0;
645 desc->rqcfg.dst_inc = 1;
647 sg_dma_address(sg), addr, sg_dma_len(sg));
650 desc->rqcfg.brst_size = burst_size;
651 desc->rqcfg.brst_len = 1;
654 /* Return the last desc in the chain */
655 desc->txd.flags = flg;
659 static irqreturn_t pl330_irq_handler(int irq, void *data)
661 if (pl330_update(data))
668 pl330_probe(struct amba_device *adev, const struct amba_id *id)
670 struct dma_pl330_platdata *pdat;
671 struct dma_pl330_dmac *pdmac;
672 struct dma_pl330_chan *pch;
673 struct pl330_info *pi;
674 struct dma_device *pd;
675 struct resource *res;
679 pdat = adev->dev.platform_data;
681 /* Allocate a new DMAC and its Channels */
682 pdmac = kzalloc(sizeof(*pdmac), GFP_KERNEL);
684 dev_err(&adev->dev, "unable to allocate mem\n");
689 pi->dev = &adev->dev;
690 pi->pl330_data = NULL;
691 pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
694 request_mem_region(res->start, resource_size(res), "dma-pl330");
696 pi->base = ioremap(res->start, resource_size(res));
702 pdmac->clk = clk_get(&adev->dev, "dma");
703 if (IS_ERR(pdmac->clk)) {
704 dev_err(&adev->dev, "Cannot get operation clock.\n");
709 amba_set_drvdata(adev, pdmac);
711 #ifdef CONFIG_PM_RUNTIME
712 /* to use the runtime PM helper functions */
713 pm_runtime_enable(&adev->dev);
715 /* enable the power domain */
716 if (pm_runtime_get_sync(&adev->dev)) {
717 dev_err(&adev->dev, "failed to get runtime pm\n");
723 clk_enable(pdmac->clk);
727 ret = request_irq(irq, pl330_irq_handler, 0,
728 dev_name(&adev->dev), pi);
736 INIT_LIST_HEAD(&pdmac->desc_pool);
737 spin_lock_init(&pdmac->pool_lock);
739 /* Create a descriptor pool of default size */
740 if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
741 dev_warn(&adev->dev, "unable to allocate desc\n");
744 INIT_LIST_HEAD(&pd->channels);
746 /* Initialize channel parameters */
747 num_chan = max(pdat ? pdat->nr_valid_peri : 0, (u8)pi->pcfg.num_chan);
748 pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
750 for (i = 0; i < num_chan; i++) {
751 pch = &pdmac->peripherals[i];
753 struct dma_pl330_peri *peri = &pdat->peri[i];
755 switch (peri->rqtype) {
757 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
761 dma_cap_set(DMA_SLAVE, pd->cap_mask);
764 dev_err(&adev->dev, "DEVTODEV Not Supported\n");
767 pch->chan.private = peri;
769 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
770 pch->chan.private = NULL;
773 INIT_LIST_HEAD(&pch->work_list);
774 spin_lock_init(&pch->lock);
775 pch->pl330_chid = NULL;
776 pch->chan.device = pd;
777 pch->chan.chan_id = i;
780 /* Add the channel to the DMAC list */
782 list_add_tail(&pch->chan.device_node, &pd->channels);
785 pd->dev = &adev->dev;
787 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
788 pd->device_free_chan_resources = pl330_free_chan_resources;
789 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
790 pd->device_tx_status = pl330_tx_status;
791 pd->device_prep_slave_sg = pl330_prep_slave_sg;
792 pd->device_control = pl330_control;
793 pd->device_issue_pending = pl330_issue_pending;
795 ret = dma_async_device_register(pd);
797 dev_err(&adev->dev, "unable to register DMAC\n");
802 "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
804 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
805 pi->pcfg.data_buf_dep,
806 pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
807 pi->pcfg.num_peri, pi->pcfg.num_events);
818 release_mem_region(res->start, resource_size(res));
824 static int __devexit pl330_remove(struct amba_device *adev)
826 struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
827 struct dma_pl330_chan *pch, *_p;
828 struct pl330_info *pi;
829 struct resource *res;
835 amba_set_drvdata(adev, NULL);
838 list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
841 /* Remove the channel */
842 list_del(&pch->chan.device_node);
844 /* Flush the channel */
845 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
846 pl330_free_chan_resources(&pch->chan);
859 release_mem_region(res->start, resource_size(res));
861 #ifdef CONFIG_PM_RUNTIME
862 pm_runtime_put(&adev->dev);
863 pm_runtime_disable(&adev->dev);
865 clk_disable(pdmac->clk);
873 static struct amba_id pl330_ids[] = {
881 #ifdef CONFIG_PM_RUNTIME
882 static int pl330_runtime_suspend(struct device *dev)
884 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
887 dev_err(dev, "failed to get dmac\n");
891 clk_disable(pdmac->clk);
896 static int pl330_runtime_resume(struct device *dev)
898 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
901 dev_err(dev, "failed to get dmac\n");
905 clk_enable(pdmac->clk);
910 #define pl330_runtime_suspend NULL
911 #define pl330_runtime_resume NULL
912 #endif /* CONFIG_PM_RUNTIME */
914 static const struct dev_pm_ops pl330_pm_ops = {
915 .runtime_suspend = pl330_runtime_suspend,
916 .runtime_resume = pl330_runtime_resume,
919 static struct amba_driver pl330_driver = {
921 .owner = THIS_MODULE,
925 .id_table = pl330_ids,
926 .probe = pl330_probe,
927 .remove = pl330_remove,
930 static int __init pl330_init(void)
932 return amba_driver_register(&pl330_driver);
934 module_init(pl330_init);
936 static void __exit pl330_exit(void)
938 amba_driver_unregister(&pl330_driver);
941 module_exit(pl330_exit);
943 MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
944 MODULE_DESCRIPTION("API Driver for PL330 DMAC");
945 MODULE_LICENSE("GPL");