2 * OMAP DMAengine support
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 #include <linux/dmaengine.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/interrupt.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/omap-dma.h>
16 #include <linux/platform_device.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
26 struct dma_device ddev;
28 struct tasklet_struct task;
29 struct list_head pending;
33 struct virt_dma_chan vc;
34 struct list_head node;
36 struct dma_slave_config cfg;
42 struct omap_desc *desc;
48 uint32_t en; /* number of elements (24-bit) */
49 uint32_t fn; /* number of frames (16-bit) */
53 struct virt_dma_desc vd;
54 enum dma_transfer_direction dir;
57 int16_t fi; /* for OMAP_DMA_SYNC_PACKET */
58 uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */
59 uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */
60 uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */
61 uint8_t periph_port; /* Peripheral port */
67 static const unsigned es_bytes[] = {
68 [OMAP_DMA_DATA_TYPE_S8] = 1,
69 [OMAP_DMA_DATA_TYPE_S16] = 2,
70 [OMAP_DMA_DATA_TYPE_S32] = 4,
73 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
75 return container_of(d, struct omap_dmadev, ddev);
78 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
80 return container_of(c, struct omap_chan, vc.chan);
83 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
85 return container_of(t, struct omap_desc, vd.tx);
88 static void omap_dma_desc_free(struct virt_dma_desc *vd)
90 kfree(container_of(vd, struct omap_desc, vd));
93 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,
96 struct omap_sg *sg = d->sg + idx;
98 if (d->dir == DMA_DEV_TO_MEM)
99 omap_set_dma_dest_params(c->dma_ch, OMAP_DMA_PORT_EMIFF,
100 OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0);
102 omap_set_dma_src_params(c->dma_ch, OMAP_DMA_PORT_EMIFF,
103 OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0);
105 omap_set_dma_transfer_params(c->dma_ch, d->es, sg->en, sg->fn,
106 d->sync_mode, c->dma_sig, d->sync_type);
108 omap_start_dma(c->dma_ch);
111 static void omap_dma_start_desc(struct omap_chan *c)
113 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
123 c->desc = d = to_omap_dma_desc(&vd->tx);
126 if (d->dir == DMA_DEV_TO_MEM)
127 omap_set_dma_src_params(c->dma_ch, d->periph_port,
128 OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi);
130 omap_set_dma_dest_params(c->dma_ch, d->periph_port,
131 OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi);
133 omap_dma_start_sg(c, d, 0);
136 static void omap_dma_callback(int ch, u16 status, void *data)
138 struct omap_chan *c = data;
142 spin_lock_irqsave(&c->vc.lock, flags);
146 if (++c->sgidx < d->sglen) {
147 omap_dma_start_sg(c, d, c->sgidx);
149 omap_dma_start_desc(c);
150 vchan_cookie_complete(&d->vd);
153 vchan_cyclic_callback(&d->vd);
156 spin_unlock_irqrestore(&c->vc.lock, flags);
160 * This callback schedules all pending channels. We could be more
161 * clever here by postponing allocation of the real DMA channels to
162 * this point, and freeing them when our virtual channel becomes idle.
164 * We would then need to deal with 'all channels in-use'
166 static void omap_dma_sched(unsigned long data)
168 struct omap_dmadev *d = (struct omap_dmadev *)data;
171 spin_lock_irq(&d->lock);
172 list_splice_tail_init(&d->pending, &head);
173 spin_unlock_irq(&d->lock);
175 while (!list_empty(&head)) {
176 struct omap_chan *c = list_first_entry(&head,
177 struct omap_chan, node);
179 spin_lock_irq(&c->vc.lock);
180 list_del_init(&c->node);
181 omap_dma_start_desc(c);
182 spin_unlock_irq(&c->vc.lock);
186 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
188 struct omap_chan *c = to_omap_dma_chan(chan);
190 dev_info(c->vc.chan.device->dev, "allocating channel for %u\n", c->dma_sig);
192 return omap_request_dma(c->dma_sig, "DMA engine",
193 omap_dma_callback, c, &c->dma_ch);
196 static void omap_dma_free_chan_resources(struct dma_chan *chan)
198 struct omap_chan *c = to_omap_dma_chan(chan);
200 vchan_free_chan_resources(&c->vc);
201 omap_free_dma(c->dma_ch);
203 dev_info(c->vc.chan.device->dev, "freeing channel for %u\n", c->dma_sig);
206 static size_t omap_dma_sg_size(struct omap_sg *sg)
208 return sg->en * sg->fn;
211 static size_t omap_dma_desc_size(struct omap_desc *d)
216 for (size = i = 0; i < d->sglen; i++)
217 size += omap_dma_sg_size(&d->sg[i]);
219 return size * es_bytes[d->es];
222 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
225 size_t size, es_size = es_bytes[d->es];
227 for (size = i = 0; i < d->sglen; i++) {
228 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
232 else if (addr >= d->sg[i].addr &&
233 addr < d->sg[i].addr + this_size)
234 size += d->sg[i].addr + this_size - addr;
239 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
240 dma_cookie_t cookie, struct dma_tx_state *txstate)
242 struct omap_chan *c = to_omap_dma_chan(chan);
243 struct virt_dma_desc *vd;
247 ret = dma_cookie_status(chan, cookie, txstate);
248 if (ret == DMA_SUCCESS || !txstate)
251 spin_lock_irqsave(&c->vc.lock, flags);
252 vd = vchan_find_desc(&c->vc, cookie);
254 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
255 } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
256 struct omap_desc *d = c->desc;
259 if (d->dir == DMA_MEM_TO_DEV)
260 pos = omap_get_dma_src_pos(c->dma_ch);
261 else if (d->dir == DMA_DEV_TO_MEM)
262 pos = omap_get_dma_dst_pos(c->dma_ch);
266 txstate->residue = omap_dma_desc_size_pos(d, pos);
268 txstate->residue = 0;
270 spin_unlock_irqrestore(&c->vc.lock, flags);
275 static void omap_dma_issue_pending(struct dma_chan *chan)
277 struct omap_chan *c = to_omap_dma_chan(chan);
280 spin_lock_irqsave(&c->vc.lock, flags);
281 if (vchan_issue_pending(&c->vc) && !c->desc) {
282 struct omap_dmadev *d = to_omap_dma_dev(chan->device);
284 if (list_empty(&c->node))
285 list_add_tail(&c->node, &d->pending);
286 spin_unlock(&d->lock);
287 tasklet_schedule(&d->task);
289 spin_unlock_irqrestore(&c->vc.lock, flags);
292 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
293 struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
294 enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
296 struct omap_chan *c = to_omap_dma_chan(chan);
297 enum dma_slave_buswidth dev_width;
298 struct scatterlist *sgent;
301 unsigned i, j = 0, es, en, frame_bytes, sync_type;
304 if (dir == DMA_DEV_TO_MEM) {
305 dev_addr = c->cfg.src_addr;
306 dev_width = c->cfg.src_addr_width;
307 burst = c->cfg.src_maxburst;
308 sync_type = OMAP_DMA_SRC_SYNC;
309 } else if (dir == DMA_MEM_TO_DEV) {
310 dev_addr = c->cfg.dst_addr;
311 dev_width = c->cfg.dst_addr_width;
312 burst = c->cfg.dst_maxburst;
313 sync_type = OMAP_DMA_DST_SYNC;
315 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
319 /* Bus width translates to the element size (ES) */
321 case DMA_SLAVE_BUSWIDTH_1_BYTE:
322 es = OMAP_DMA_DATA_TYPE_S8;
324 case DMA_SLAVE_BUSWIDTH_2_BYTES:
325 es = OMAP_DMA_DATA_TYPE_S16;
327 case DMA_SLAVE_BUSWIDTH_4_BYTES:
328 es = OMAP_DMA_DATA_TYPE_S32;
330 default: /* not reached */
334 /* Now allocate and setup the descriptor. */
335 d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
340 d->dev_addr = dev_addr;
342 d->sync_mode = OMAP_DMA_SYNC_FRAME;
343 d->sync_type = sync_type;
344 d->periph_port = OMAP_DMA_PORT_TIPB;
347 * Build our scatterlist entries: each contains the address,
348 * the number of elements (EN) in each frame, and the number of
349 * frames (FN). Number of bytes for this entry = ES * EN * FN.
351 * Burst size translates to number of elements with frame sync.
352 * Note: DMA engine defines burst to be the number of dev-width
356 frame_bytes = es_bytes[es] * en;
357 for_each_sg(sgl, sgent, sglen, i) {
358 d->sg[j].addr = sg_dma_address(sgent);
360 d->sg[j].fn = sg_dma_len(sgent) / frame_bytes;
366 return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
369 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
370 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
371 size_t period_len, enum dma_transfer_direction dir, unsigned long flags,
374 struct omap_chan *c = to_omap_dma_chan(chan);
375 enum dma_slave_buswidth dev_width;
378 unsigned es, sync_type;
381 if (dir == DMA_DEV_TO_MEM) {
382 dev_addr = c->cfg.src_addr;
383 dev_width = c->cfg.src_addr_width;
384 burst = c->cfg.src_maxburst;
385 sync_type = OMAP_DMA_SRC_SYNC;
386 } else if (dir == DMA_MEM_TO_DEV) {
387 dev_addr = c->cfg.dst_addr;
388 dev_width = c->cfg.dst_addr_width;
389 burst = c->cfg.dst_maxburst;
390 sync_type = OMAP_DMA_DST_SYNC;
392 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
396 /* Bus width translates to the element size (ES) */
398 case DMA_SLAVE_BUSWIDTH_1_BYTE:
399 es = OMAP_DMA_DATA_TYPE_S8;
401 case DMA_SLAVE_BUSWIDTH_2_BYTES:
402 es = OMAP_DMA_DATA_TYPE_S16;
404 case DMA_SLAVE_BUSWIDTH_4_BYTES:
405 es = OMAP_DMA_DATA_TYPE_S32;
407 default: /* not reached */
411 /* Now allocate and setup the descriptor. */
412 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
417 d->dev_addr = dev_addr;
421 d->sync_mode = OMAP_DMA_SYNC_PACKET;
423 d->sync_mode = OMAP_DMA_SYNC_ELEMENT;
424 d->sync_type = sync_type;
425 d->periph_port = OMAP_DMA_PORT_MPUI;
426 d->sg[0].addr = buf_addr;
427 d->sg[0].en = period_len / es_bytes[es];
428 d->sg[0].fn = buf_len / period_len;
433 omap_dma_link_lch(c->dma_ch, c->dma_ch);
435 if (flags & DMA_PREP_INTERRUPT)
436 omap_enable_dma_irq(c->dma_ch, OMAP_DMA_FRAME_IRQ);
438 omap_disable_dma_irq(c->dma_ch, OMAP_DMA_BLOCK_IRQ);
441 if (!cpu_class_is_omap1()) {
442 omap_set_dma_src_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16);
443 omap_set_dma_dest_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16);
446 return vchan_tx_prep(&c->vc, &d->vd, flags);
449 static int omap_dma_slave_config(struct omap_chan *c, struct dma_slave_config *cfg)
451 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
452 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
455 memcpy(&c->cfg, cfg, sizeof(c->cfg));
460 static int omap_dma_terminate_all(struct omap_chan *c)
462 struct omap_dmadev *d = to_omap_dma_dev(c->vc.chan.device);
466 spin_lock_irqsave(&c->vc.lock, flags);
468 /* Prevent this channel being scheduled */
470 list_del_init(&c->node);
471 spin_unlock(&d->lock);
474 * Stop DMA activity: we assume the callback will not be called
475 * after omap_stop_dma() returns (even if it does, it will see
476 * c->desc is NULL and exit.)
480 /* Avoid stopping the dma twice */
482 omap_stop_dma(c->dma_ch);
488 omap_dma_unlink_lch(c->dma_ch, c->dma_ch);
491 vchan_get_all_descriptors(&c->vc, &head);
492 spin_unlock_irqrestore(&c->vc.lock, flags);
493 vchan_dma_desc_free_list(&c->vc, &head);
498 static int omap_dma_pause(struct omap_chan *c)
500 /* Pause/Resume only allowed with cyclic mode */
505 omap_stop_dma(c->dma_ch);
512 static int omap_dma_resume(struct omap_chan *c)
514 /* Pause/Resume only allowed with cyclic mode */
519 omap_start_dma(c->dma_ch);
526 static int omap_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
529 struct omap_chan *c = to_omap_dma_chan(chan);
533 case DMA_SLAVE_CONFIG:
534 ret = omap_dma_slave_config(c, (struct dma_slave_config *)arg);
537 case DMA_TERMINATE_ALL:
538 ret = omap_dma_terminate_all(c);
542 ret = omap_dma_pause(c);
546 ret = omap_dma_resume(c);
557 static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig)
561 c = kzalloc(sizeof(*c), GFP_KERNEL);
565 c->dma_sig = dma_sig;
566 c->vc.desc_free = omap_dma_desc_free;
567 vchan_init(&c->vc, &od->ddev);
568 INIT_LIST_HEAD(&c->node);
575 static void omap_dma_free(struct omap_dmadev *od)
577 tasklet_kill(&od->task);
578 while (!list_empty(&od->ddev.channels)) {
579 struct omap_chan *c = list_first_entry(&od->ddev.channels,
580 struct omap_chan, vc.chan.device_node);
582 list_del(&c->vc.chan.device_node);
583 tasklet_kill(&c->vc.task);
589 static int omap_dma_probe(struct platform_device *pdev)
591 struct omap_dmadev *od;
594 od = kzalloc(sizeof(*od), GFP_KERNEL);
598 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
599 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
600 od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
601 od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
602 od->ddev.device_tx_status = omap_dma_tx_status;
603 od->ddev.device_issue_pending = omap_dma_issue_pending;
604 od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
605 od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
606 od->ddev.device_control = omap_dma_control;
607 od->ddev.dev = &pdev->dev;
608 INIT_LIST_HEAD(&od->ddev.channels);
609 INIT_LIST_HEAD(&od->pending);
610 spin_lock_init(&od->lock);
612 tasklet_init(&od->task, omap_dma_sched, (unsigned long)od);
614 for (i = 0; i < 127; i++) {
615 rc = omap_dma_chan_init(od, i);
622 rc = dma_async_device_register(&od->ddev);
624 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
628 platform_set_drvdata(pdev, od);
631 dev_info(&pdev->dev, "OMAP DMA engine driver\n");
636 static int omap_dma_remove(struct platform_device *pdev)
638 struct omap_dmadev *od = platform_get_drvdata(pdev);
640 dma_async_device_unregister(&od->ddev);
646 static struct platform_driver omap_dma_driver = {
647 .probe = omap_dma_probe,
648 .remove = omap_dma_remove,
650 .name = "omap-dma-engine",
651 .owner = THIS_MODULE,
655 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
657 if (chan->device->dev->driver == &omap_dma_driver.driver) {
658 struct omap_chan *c = to_omap_dma_chan(chan);
659 unsigned req = *(unsigned *)param;
661 return req == c->dma_sig;
665 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
667 static struct platform_device *pdev;
669 static const struct platform_device_info omap_dma_dev_info = {
670 .name = "omap-dma-engine",
672 .dma_mask = DMA_BIT_MASK(32),
675 static int omap_dma_init(void)
677 int rc = platform_driver_register(&omap_dma_driver);
680 pdev = platform_device_register_full(&omap_dma_dev_info);
682 platform_driver_unregister(&omap_dma_driver);
688 subsys_initcall(omap_dma_init);
690 static void __exit omap_dma_exit(void)
692 platform_device_unregister(pdev);
693 platform_driver_unregister(&omap_dma_driver);
695 module_exit(omap_dma_exit);
697 MODULE_AUTHOR("Russell King");
698 MODULE_LICENSE("GPL");