[linux-2.6-block.git] / drivers / dma / dma-jz4740.c

/*
 *  Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
 *  JZ4740 DMAC support
 *
 *  This program is free software; you can redistribute it and/or modify it
 *  under  the terms of the GNU General	 Public License as published by the
 *  Free Software Foundation;  either version 2 of the License, or (at your
 *  option) any later version.
 *
 */

#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/clk.h>

#include <asm/mach-jz4740/dma.h>

#include "virt-dma.h"

#define JZ_DMA_NR_CHANS 6

#define JZ_REG_DMA_SRC_ADDR(x)		(0x00 + (x) * 0x20)
#define JZ_REG_DMA_DST_ADDR(x)		(0x04 + (x) * 0x20)
#define JZ_REG_DMA_TRANSFER_COUNT(x)	(0x08 + (x) * 0x20)
#define JZ_REG_DMA_REQ_TYPE(x)		(0x0C + (x) * 0x20)
#define JZ_REG_DMA_STATUS_CTRL(x)	(0x10 + (x) * 0x20)
#define JZ_REG_DMA_CMD(x)		(0x14 + (x) * 0x20)
#define JZ_REG_DMA_DESC_ADDR(x)		(0x18 + (x) * 0x20)

#define JZ_REG_DMA_CTRL			0x300
#define JZ_REG_DMA_IRQ			0x304
#define JZ_REG_DMA_DOORBELL		0x308
#define JZ_REG_DMA_DOORBELL_SET		0x30C

#define JZ_DMA_STATUS_CTRL_NO_DESC		BIT(31)
#define JZ_DMA_STATUS_CTRL_DESC_INV		BIT(6)
#define JZ_DMA_STATUS_CTRL_ADDR_ERR		BIT(4)
#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE	BIT(3)
#define JZ_DMA_STATUS_CTRL_HALT			BIT(2)
#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE	BIT(1)
#define JZ_DMA_STATUS_CTRL_ENABLE		BIT(0)

#define JZ_DMA_CMD_SRC_INC			BIT(23)
#define JZ_DMA_CMD_DST_INC			BIT(22)
#define JZ_DMA_CMD_RDIL_MASK			(0xf << 16)
#define JZ_DMA_CMD_SRC_WIDTH_MASK		(0x3 << 14)
#define JZ_DMA_CMD_DST_WIDTH_MASK		(0x3 << 12)
#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK		(0x7 << 8)
#define JZ_DMA_CMD_BLOCK_MODE			BIT(7)
#define JZ_DMA_CMD_DESC_VALID			BIT(4)
#define JZ_DMA_CMD_DESC_VALID_MODE		BIT(3)
#define JZ_DMA_CMD_VALID_IRQ_ENABLE		BIT(2)
#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE		BIT(1)
#define JZ_DMA_CMD_LINK_ENABLE			BIT(0)

#define JZ_DMA_CMD_FLAGS_OFFSET 22
#define JZ_DMA_CMD_RDIL_OFFSET 16
#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
#define JZ_DMA_CMD_MODE_OFFSET 7

#define JZ_DMA_CTRL_PRIORITY_MASK		(0x3 << 8)
#define JZ_DMA_CTRL_HALT			BIT(3)
#define JZ_DMA_CTRL_ADDRESS_ERROR		BIT(2)
#define JZ_DMA_CTRL_ENABLE			BIT(0)

enum jz4740_dma_width {
	JZ4740_DMA_WIDTH_32BIT	= 0,
	JZ4740_DMA_WIDTH_8BIT	= 1,
	JZ4740_DMA_WIDTH_16BIT	= 2,
};

enum jz4740_dma_transfer_size {
	JZ4740_DMA_TRANSFER_SIZE_4BYTE	= 0,
	JZ4740_DMA_TRANSFER_SIZE_1BYTE	= 1,
	JZ4740_DMA_TRANSFER_SIZE_2BYTE	= 2,
	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
};

enum jz4740_dma_flags {
	JZ4740_DMA_SRC_AUTOINC = 0x2,
	JZ4740_DMA_DST_AUTOINC = 0x1,
};

enum jz4740_dma_mode {
	JZ4740_DMA_MODE_SINGLE	= 0,
	JZ4740_DMA_MODE_BLOCK	= 1,
};

struct jz4740_dma_sg {
	dma_addr_t addr;
	unsigned int len;
};

struct jz4740_dma_desc {
	struct virt_dma_desc vdesc;

	enum dma_transfer_direction direction;
	bool cyclic;

	unsigned int num_sgs;
	struct jz4740_dma_sg sg[];
};

struct jz4740_dmaengine_chan {
	struct virt_dma_chan vchan;
	unsigned int id;

	dma_addr_t fifo_addr;
	unsigned int transfer_shift;

	struct jz4740_dma_desc *desc;
	unsigned int next_sg;
};

struct jz4740_dma_dev {
	struct dma_device ddev;
	void __iomem *base;
	struct clk *clk;

	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
};

static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	struct jz4740_dmaengine_chan *chan)
{
	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
		ddev);
}

static struct jz4740_dmaengine_chan *to_jz4740_dma_chan(struct dma_chan *c)
{
	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
}

static struct jz4740_dma_desc *to_jz4740_dma_desc(struct virt_dma_desc *vdesc)
{
	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
}

static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	unsigned int reg)
{
	return readl(dmadev->base + reg);
}

static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	unsigned reg, uint32_t val)
{
	writel(val, dmadev->base + reg);
}

static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	unsigned int reg, uint32_t val, uint32_t mask)
{
	uint32_t tmp;

	tmp = jz4740_dma_read(dmadev, reg);
	tmp &= ~mask;
	tmp |= val;
	jz4740_dma_write(dmadev, reg, tmp);
}

static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
{
	return kzalloc(sizeof(struct jz4740_dma_desc) +
		sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
}

static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
{
	switch (width) {
	case DMA_SLAVE_BUSWIDTH_1_BYTE:
		return JZ4740_DMA_WIDTH_8BIT;
	case DMA_SLAVE_BUSWIDTH_2_BYTES:
		return JZ4740_DMA_WIDTH_16BIT;
	case DMA_SLAVE_BUSWIDTH_4_BYTES:
		return JZ4740_DMA_WIDTH_32BIT;
	default:
		return JZ4740_DMA_WIDTH_32BIT;
	}
}

static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
{
	if (maxburst <= 1)
		return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	else if (maxburst <= 3)
		return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	else if (maxburst <= 15)
		return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	else if (maxburst <= 31)
		return JZ4740_DMA_TRANSFER_SIZE_16BYTE;

	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
}

static int jz4740_dma_slave_config(struct dma_chan *c,
				   struct dma_slave_config *config)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	enum jz4740_dma_width src_width;
	enum jz4740_dma_width dst_width;
	enum jz4740_dma_transfer_size transfer_size;
	enum jz4740_dma_flags flags;
	uint32_t cmd;

	switch (config->direction) {
	case DMA_MEM_TO_DEV:
		flags = JZ4740_DMA_SRC_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
		chan->fifo_addr = config->dst_addr;
		break;
	case DMA_DEV_TO_MEM:
		flags = JZ4740_DMA_DST_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->src_maxburst);
		chan->fifo_addr = config->src_addr;
		break;
	default:
		return -EINVAL;
	}

	src_width = jz4740_dma_width(config->src_addr_width);
	dst_width = jz4740_dma_width(config->dst_addr_width);

	switch (transfer_size) {
	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
		chan->transfer_shift = 1;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
		chan->transfer_shift = 2;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
		chan->transfer_shift = 4;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
		chan->transfer_shift = 5;
		break;
	default:
		chan->transfer_shift = 0;
		break;
	}

	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	cmd |= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	cmd |= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	cmd |= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	cmd |= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	cmd |= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;

	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
		config->slave_id);

	return 0;
}

static int jz4740_dma_terminate_all(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&chan->vchan.lock, flags);
	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);
	chan->desc = NULL;
	vchan_get_all_descriptors(&chan->vchan, &head);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	vchan_dma_desc_free_list(&chan->vchan, &head);

	return 0;
}

static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	dma_addr_t src_addr, dst_addr;
	struct virt_dma_desc *vdesc;
	struct jz4740_dma_sg *sg;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);

	if (!chan->desc) {
		vdesc = vchan_next_desc(&chan->vchan);
		if (!vdesc)
			return 0;
		chan->desc = to_jz4740_dma_desc(vdesc);
		chan->next_sg = 0;
	}

	if (chan->next_sg == chan->desc->num_sgs)
		chan->next_sg = 0;

	sg = &chan->desc->sg[chan->next_sg];

	if (chan->desc->direction == DMA_MEM_TO_DEV) {
		src_addr = sg->addr;
		dst_addr = chan->fifo_addr;
	} else {
		src_addr = chan->fifo_addr;
		dst_addr = sg->addr;
	}
	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
			sg->len >> chan->transfer_shift);

	chan->next_sg++;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
			JZ_DMA_STATUS_CTRL_NO_DESC | JZ_DMA_STATUS_CTRL_ENABLE,
			JZ_DMA_STATUS_CTRL_HALT | JZ_DMA_STATUS_CTRL_NO_DESC |
			JZ_DMA_STATUS_CTRL_ENABLE);

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
			JZ_DMA_CTRL_ENABLE,
			JZ_DMA_CTRL_HALT | JZ_DMA_CTRL_ENABLE);

	return 0;
}

static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
{
	spin_lock(&chan->vchan.lock);
	if (chan->desc) {
		if (chan->desc->cyclic) {
			vchan_cyclic_callback(&chan->desc->vdesc);
		} else {
			if (chan->next_sg == chan->desc->num_sgs) {
				list_del(&chan->desc->vdesc.node);
				vchan_cookie_complete(&chan->desc->vdesc);
				chan->desc = NULL;
			}
		}
	}
	jz4740_dma_start_transfer(chan);
	spin_unlock(&chan->vchan.lock);
}

static irqreturn_t jz4740_dma_irq(int irq, void *devid)
{
	struct jz4740_dma_dev *dmadev = devid;
	uint32_t irq_status;
	unsigned int i;

	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);

	for (i = 0; i < 6; ++i) {
		if (irq_status & (1 << i)) {
			jz4740_dma_write_mask(dmadev,
				JZ_REG_DMA_STATUS_CTRL(i), 0,
				JZ_DMA_STATUS_CTRL_ENABLE |
				JZ_DMA_STATUS_CTRL_TRANSFER_DONE);

			jz4740_dma_chan_irq(&dmadev->chan[i]);
		}
	}

	return IRQ_HANDLED;
}

static void jz4740_dma_issue_pending(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	unsigned long flags;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
		jz4740_dma_start_transfer(chan);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	struct dma_chan *c, struct scatterlist *sgl,
	unsigned int sg_len, enum dma_transfer_direction direction,
	unsigned long flags, void *context)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	struct scatterlist *sg;
	unsigned int i;

	desc = jz4740_dma_alloc_desc(sg_len);
	if (!desc)
		return NULL;

	for_each_sg(sgl, sg, sg_len, i) {
		desc->sg[i].addr = sg_dma_address(sg);
		desc->sg[i].len = sg_dma_len(sg);
	}

	desc->num_sgs = sg_len;
	desc->direction = direction;
	desc->cyclic = false;

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	size_t period_len, enum dma_transfer_direction direction,
	unsigned long flags)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	unsigned int num_periods, i;

	if (buf_len % period_len)
		return NULL;

	num_periods = buf_len / period_len;

	desc = jz4740_dma_alloc_desc(num_periods);
	if (!desc)
		return NULL;

	for (i = 0; i < num_periods; i++) {
		desc->sg[i].addr = buf_addr;
		desc->sg[i].len = period_len;
		buf_addr += period_len;
	}

	desc->num_sgs = num_periods;
	desc->direction = direction;
	desc->cyclic = true;

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	struct jz4740_dma_desc *desc, unsigned int next_sg)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned int residue, count;
	unsigned int i;

	residue = 0;

	for (i = next_sg; i < desc->num_sgs; i++)
		residue += desc->sg[i].len;

	if (next_sg != 0) {
		count = jz4740_dma_read(dmadev,
			JZ_REG_DMA_TRANSFER_COUNT(chan->id));
		residue += count << chan->transfer_shift;
	}

	return residue;
}

static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	dma_cookie_t cookie, struct dma_tx_state *state)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct virt_dma_desc *vdesc;
	enum dma_status status;
	unsigned long flags;

	status = dma_cookie_status(c, cookie, state);
	if (status == DMA_COMPLETE || !state)
		return status;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	vdesc = vchan_find_desc(&chan->vchan, cookie);
	if (cookie == chan->desc->vdesc.tx.cookie) {
		state->residue = jz4740_dma_desc_residue(chan, chan->desc,
				chan->next_sg);
	} else if (vdesc) {
		state->residue = jz4740_dma_desc_residue(chan,
				to_jz4740_dma_desc(vdesc), 0);
	} else {
		state->residue = 0;
	}
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	return status;
}

static void jz4740_dma_free_chan_resources(struct dma_chan *c)
{
	vchan_free_chan_resources(to_virt_chan(c));
}

static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
{
	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
}

#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))

static int jz4740_dma_probe(struct platform_device *pdev)
{
	struct jz4740_dmaengine_chan *chan;
	struct jz4740_dma_dev *dmadev;
	struct dma_device *dd;
	unsigned int i;
	struct resource *res;
	int ret;
	int irq;

	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	if (!dmadev)
		return -EINVAL;

	dd = &dmadev->ddev;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	if (IS_ERR(dmadev->base))
		return PTR_ERR(dmadev->base);

	dmadev->clk = clk_get(&pdev->dev, "dma");
	if (IS_ERR(dmadev->clk))
		return PTR_ERR(dmadev->clk);

	clk_prepare_enable(dmadev->clk);

	dma_cap_set(DMA_SLAVE, dd->cap_mask);
	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	dd->device_tx_status = jz4740_dma_tx_status;
	dd->device_issue_pending = jz4740_dma_issue_pending;
	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
	dd->device_config = jz4740_dma_slave_config;
	dd->device_terminate_all = jz4740_dma_terminate_all;
	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
	dd->dev = &pdev->dev;
	INIT_LIST_HEAD(&dd->channels);

	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
		chan = &dmadev->chan[i];
		chan->id = i;
		chan->vchan.desc_free = jz4740_dma_desc_free;
		vchan_init(&chan->vchan, dd);
	}

	ret = dma_async_device_register(dd);
	if (ret)
		return ret;

	irq = platform_get_irq(pdev, 0);
	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	if (ret)
		goto err_unregister;

	platform_set_drvdata(pdev, dmadev);

	return 0;

err_unregister:
	dma_async_device_unregister(dd);
	return ret;
}

static int jz4740_dma_remove(struct platform_device *pdev)
{
	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
	int irq = platform_get_irq(pdev, 0);

	free_irq(irq, dmadev);
	dma_async_device_unregister(&dmadev->ddev);
	clk_disable_unprepare(dmadev->clk);

	return 0;
}

static struct platform_driver jz4740_dma_driver = {
	.probe = jz4740_dma_probe,
	.remove = jz4740_dma_remove,
	.driver = {
		.name = "jz4740-dma",
	},
};
module_platform_driver(jz4740_dma_driver);

MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
MODULE_DESCRIPTION("JZ4740 DMA driver");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
7c169a42 LPC	1	/*
	2	* Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
	3	* JZ4740 DMAC support
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License as published by the
	7	* Free Software Foundation; either version 2 of the License, or (at your
	8	* option) any later version.
	9	*
7c169a42 LPC	10	*/
	11
	12	#include <linux/dmaengine.h>
	13	#include <linux/dma-mapping.h>
	14	#include <linux/err.h>
	15	#include <linux/init.h>
	16	#include <linux/list.h>
	17	#include <linux/module.h>
	18	#include <linux/platform_device.h>
	19	#include <linux/slab.h>
	20	#include <linux/spinlock.h>
25ce6c35 LPC	21	#include <linux/irq.h>
25ce6c35 LPC	22	#include <linux/clk.h>
7c169a42 LPC	23
	24	#include <asm/mach-jz4740/dma.h>
	25
	26	#include "virt-dma.h"
	27
	28	#define JZ_DMA_NR_CHANS 6
	29
25ce6c35 LPC	30	#define JZ_REG_DMA_SRC_ADDR(x) (0x00 + (x) * 0x20)
	31	#define JZ_REG_DMA_DST_ADDR(x) (0x04 + (x) * 0x20)
	32	#define JZ_REG_DMA_TRANSFER_COUNT(x) (0x08 + (x) * 0x20)
	33	#define JZ_REG_DMA_REQ_TYPE(x) (0x0C + (x) * 0x20)
	34	#define JZ_REG_DMA_STATUS_CTRL(x) (0x10 + (x) * 0x20)
	35	#define JZ_REG_DMA_CMD(x) (0x14 + (x) * 0x20)
	36	#define JZ_REG_DMA_DESC_ADDR(x) (0x18 + (x) * 0x20)
	37
	38	#define JZ_REG_DMA_CTRL 0x300
	39	#define JZ_REG_DMA_IRQ 0x304
	40	#define JZ_REG_DMA_DOORBELL 0x308
	41	#define JZ_REG_DMA_DOORBELL_SET 0x30C
	42
	43	#define JZ_DMA_STATUS_CTRL_NO_DESC BIT(31)
	44	#define JZ_DMA_STATUS_CTRL_DESC_INV BIT(6)
	45	#define JZ_DMA_STATUS_CTRL_ADDR_ERR BIT(4)
	46	#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE BIT(3)
	47	#define JZ_DMA_STATUS_CTRL_HALT BIT(2)
	48	#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE BIT(1)
	49	#define JZ_DMA_STATUS_CTRL_ENABLE BIT(0)
	50
	51	#define JZ_DMA_CMD_SRC_INC BIT(23)
	52	#define JZ_DMA_CMD_DST_INC BIT(22)
	53	#define JZ_DMA_CMD_RDIL_MASK (0xf << 16)
	54	#define JZ_DMA_CMD_SRC_WIDTH_MASK (0x3 << 14)
	55	#define JZ_DMA_CMD_DST_WIDTH_MASK (0x3 << 12)
	56	#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK (0x7 << 8)
	57	#define JZ_DMA_CMD_BLOCK_MODE BIT(7)
	58	#define JZ_DMA_CMD_DESC_VALID BIT(4)
	59	#define JZ_DMA_CMD_DESC_VALID_MODE BIT(3)
	60	#define JZ_DMA_CMD_VALID_IRQ_ENABLE BIT(2)
	61	#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE BIT(1)
	62	#define JZ_DMA_CMD_LINK_ENABLE BIT(0)
	63
	64	#define JZ_DMA_CMD_FLAGS_OFFSET 22
	65	#define JZ_DMA_CMD_RDIL_OFFSET 16
	66	#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
	67	#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
	68	#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
	69	#define JZ_DMA_CMD_MODE_OFFSET 7
	70
	71	#define JZ_DMA_CTRL_PRIORITY_MASK (0x3 << 8)
	72	#define JZ_DMA_CTRL_HALT BIT(3)
	73	#define JZ_DMA_CTRL_ADDRESS_ERROR BIT(2)
	74	#define JZ_DMA_CTRL_ENABLE BIT(0)
	75
	76	enum jz4740_dma_width {
	77	JZ4740_DMA_WIDTH_32BIT = 0,
	78	JZ4740_DMA_WIDTH_8BIT = 1,
	79	JZ4740_DMA_WIDTH_16BIT = 2,
	80	};
	81
	82	enum jz4740_dma_transfer_size {
	83	JZ4740_DMA_TRANSFER_SIZE_4BYTE = 0,
	84	JZ4740_DMA_TRANSFER_SIZE_1BYTE = 1,
	85	JZ4740_DMA_TRANSFER_SIZE_2BYTE = 2,
	86	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	87	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
	88	};
	89
	90	enum jz4740_dma_flags {
	91	JZ4740_DMA_SRC_AUTOINC = 0x2,
	92	JZ4740_DMA_DST_AUTOINC = 0x1,
	93	};
94
95	enum jz4740_dma_mode {
96	JZ4740_DMA_MODE_SINGLE = 0,
97	JZ4740_DMA_MODE_BLOCK = 1,
98	};
99
7c169a42 LPC	100	struct jz4740_dma_sg {
	101	dma_addr_t addr;
	102	unsigned int len;
	103	};
	104
	105	struct jz4740_dma_desc {
	106	struct virt_dma_desc vdesc;
	107
	108	enum dma_transfer_direction direction;
	109	bool cyclic;
	110
	111	unsigned int num_sgs;
	112	struct jz4740_dma_sg sg[];
	113	};
	114
	115	struct jz4740_dmaengine_chan {
	116	struct virt_dma_chan vchan;
25ce6c35	117	unsigned int id;
7c169a42 LPC	118
7c169a42 LPC	119	dma_addr_t fifo_addr;
25ce6c35	120	unsigned int transfer_shift;
7c169a42 LPC	121
	122	struct jz4740_dma_desc *desc;
	123	unsigned int next_sg;
	124	};
	125
	126	struct jz4740_dma_dev {
	127	struct dma_device ddev;
25ce6c35 LPC	128	void __iomem *base;
25ce6c35 LPC	129	struct clk *clk;
7c169a42 LPC	130
	131	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
	132	};
	133
25ce6c35 LPC	134	static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	135	struct jz4740_dmaengine_chan *chan)
	136	{
	137	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
	138	ddev);
	139	}
	140
7c169a42 LPC	141	static struct jz4740_dmaengine_chan to_jz4740_dma_chan(struct dma_chan c)
	142	{
	143	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
	144	}
	145
	146	static struct jz4740_dma_desc to_jz4740_dma_desc(struct virt_dma_desc vdesc)
	147	{
	148	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
	149	}
	150
25ce6c35 LPC	151	static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	152	unsigned int reg)
	153	{
	154	return readl(dmadev->base + reg);
	155	}
	156
	157	static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	158	unsigned reg, uint32_t val)
	159	{
	160	writel(val, dmadev->base + reg);
	161	}
	162
	163	static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	164	unsigned int reg, uint32_t val, uint32_t mask)
	165	{
	166	uint32_t tmp;
	167
	168	tmp = jz4740_dma_read(dmadev, reg);
	169	tmp &= ~mask;
	170	tmp \|= val;
	171	jz4740_dma_write(dmadev, reg, tmp);
	172	}
	173
7c169a42 LPC	174	static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
	175	{
	176	return kzalloc(sizeof(struct jz4740_dma_desc) +
	177	sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
	178	}
	179
	180	static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
	181	{
	182	switch (width) {
	183	case DMA_SLAVE_BUSWIDTH_1_BYTE:
	184	return JZ4740_DMA_WIDTH_8BIT;
	185	case DMA_SLAVE_BUSWIDTH_2_BYTES:
	186	return JZ4740_DMA_WIDTH_16BIT;
	187	case DMA_SLAVE_BUSWIDTH_4_BYTES:
	188	return JZ4740_DMA_WIDTH_32BIT;
	189	default:
	190	return JZ4740_DMA_WIDTH_32BIT;
	191	}
	192	}
	193
	194	static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
	195	{
	196	if (maxburst <= 1)
	197	return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	198	else if (maxburst <= 3)
	199	return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	200	else if (maxburst <= 15)
	201	return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	202	else if (maxburst <= 31)
	203	return JZ4740_DMA_TRANSFER_SIZE_16BYTE;
	204
	205	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
	206	}
	207
	208	static int jz4740_dma_slave_config(struct dma_chan *c,
1d4c0b8c	209	struct dma_slave_config *config)
7c169a42 LPC	210	{
7c169a42 LPC	211	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35 LPC	212	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	213	enum jz4740_dma_width src_width;
	214	enum jz4740_dma_width dst_width;
	215	enum jz4740_dma_transfer_size transfer_size;
	216	enum jz4740_dma_flags flags;
	217	uint32_t cmd;
7c169a42 LPC	218
	219	switch (config->direction) {
	220	case DMA_MEM_TO_DEV:
25ce6c35 LPC	221	flags = JZ4740_DMA_SRC_AUTOINC;
25ce6c35 LPC	222	transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
7c169a42 LPC	223	chan->fifo_addr = config->dst_addr;
	224	break;
	225	case DMA_DEV_TO_MEM:
25ce6c35 LPC	226	flags = JZ4740_DMA_DST_AUTOINC;
25ce6c35 LPC	227	transfer_size = jz4740_dma_maxburst(config->src_maxburst);
7c169a42 LPC	228	chan->fifo_addr = config->src_addr;
	229	break;
	230	default:
	231	return -EINVAL;
	232	}
	233
25ce6c35 LPC	234	src_width = jz4740_dma_width(config->src_addr_width);
	235	dst_width = jz4740_dma_width(config->dst_addr_width);
	236
	237	switch (transfer_size) {
	238	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
	239	chan->transfer_shift = 1;
	240	break;
	241	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
	242	chan->transfer_shift = 2;
	243	break;
	244	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
	245	chan->transfer_shift = 4;
	246	break;
	247	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
	248	chan->transfer_shift = 5;
	249	break;
	250	default:
	251	chan->transfer_shift = 0;
	252	break;
	253	}
7c169a42	254
25ce6c35 LPC	255	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	256	cmd \|= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	257	cmd \|= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	258	cmd \|= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	259	cmd \|= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	260	cmd \|= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;
7c169a42	261
25ce6c35 LPC	262	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	263	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	264	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
	265	config->slave_id);
7c169a42 LPC	266
	267	return 0;
	268	}
	269
	270	static int jz4740_dma_terminate_all(struct dma_chan *c)
	271	{
	272	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35	273	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	274	unsigned long flags;
	275	LIST_HEAD(head);
	276
	277	spin_lock_irqsave(&chan->vchan.lock, flags);
25ce6c35 LPC	278	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	279	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	280	chan->desc = NULL;
	281	vchan_get_all_descriptors(&chan->vchan, &head);
	282	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	283
	284	vchan_dma_desc_free_list(&chan->vchan, &head);
	285
	286	return 0;
	287	}
	288
7c169a42 LPC	289	static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
7c169a42 LPC	290	{
25ce6c35	291	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	292	dma_addr_t src_addr, dst_addr;
	293	struct virt_dma_desc *vdesc;
	294	struct jz4740_dma_sg *sg;
	295
25ce6c35 LPC	296	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	297	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	298
	299	if (!chan->desc) {
	300	vdesc = vchan_next_desc(&chan->vchan);
	301	if (!vdesc)
	302	return 0;
	303	chan->desc = to_jz4740_dma_desc(vdesc);
	304	chan->next_sg = 0;
	305	}
	306
	307	if (chan->next_sg == chan->desc->num_sgs)
	308	chan->next_sg = 0;
	309
	310	sg = &chan->desc->sg[chan->next_sg];
	311
	312	if (chan->desc->direction == DMA_MEM_TO_DEV) {
	313	src_addr = sg->addr;
	314	dst_addr = chan->fifo_addr;
	315	} else {
	316	src_addr = chan->fifo_addr;
	317	dst_addr = sg->addr;
	318	}
25ce6c35 LPC	319	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	320	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	321	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
	322	sg->len >> chan->transfer_shift);
7c169a42 LPC	323
	324	chan->next_sg++;
	325
25ce6c35 LPC	326	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
	327	JZ_DMA_STATUS_CTRL_NO_DESC \| JZ_DMA_STATUS_CTRL_ENABLE,
	328	JZ_DMA_STATUS_CTRL_HALT \| JZ_DMA_STATUS_CTRL_NO_DESC \|
	329	JZ_DMA_STATUS_CTRL_ENABLE);
	330
	331	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
	332	JZ_DMA_CTRL_ENABLE,
	333	JZ_DMA_CTRL_HALT \| JZ_DMA_CTRL_ENABLE);
7c169a42 LPC	334
	335	return 0;
	336	}
	337
25ce6c35	338	static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
7c169a42	339	{
7c169a42 LPC	340	spin_lock(&chan->vchan.lock);
7c169a42 LPC	341	if (chan->desc) {
26f7af37	342	if (chan->desc->cyclic) {
7c169a42 LPC	343	vchan_cyclic_callback(&chan->desc->vdesc);
	344	} else {
	345	if (chan->next_sg == chan->desc->num_sgs) {
f498e064	346	list_del(&chan->desc->vdesc.node);
7c169a42	347	vchan_cookie_complete(&chan->desc->vdesc);
f498e064	348	chan->desc = NULL;
7c169a42 LPC	349	}
	350	}
	351	}
	352	jz4740_dma_start_transfer(chan);
	353	spin_unlock(&chan->vchan.lock);
	354	}
	355
25ce6c35 LPC	356	static irqreturn_t jz4740_dma_irq(int irq, void *devid)
	357	{
	358	struct jz4740_dma_dev *dmadev = devid;
	359	uint32_t irq_status;
	360	unsigned int i;
	361
	362	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);
	363
	364	for (i = 0; i < 6; ++i) {
	365	if (irq_status & (1 << i)) {
	366	jz4740_dma_write_mask(dmadev,
	367	JZ_REG_DMA_STATUS_CTRL(i), 0,
	368	JZ_DMA_STATUS_CTRL_ENABLE \|
	369	JZ_DMA_STATUS_CTRL_TRANSFER_DONE);
	370
	371	jz4740_dma_chan_irq(&dmadev->chan[i]);
	372	}
	373	}
	374
	375	return IRQ_HANDLED;
	376	}
	377
7c169a42 LPC	378	static void jz4740_dma_issue_pending(struct dma_chan *c)
	379	{
	380	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	381	unsigned long flags;
	382
	383	spin_lock_irqsave(&chan->vchan.lock, flags);
	384	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
	385	jz4740_dma_start_transfer(chan);
	386	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	387	}
	388
	389	static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	390	struct dma_chan c, struct scatterlist sgl,
	391	unsigned int sg_len, enum dma_transfer_direction direction,
	392	unsigned long flags, void *context)
	393	{
	394	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	395	struct jz4740_dma_desc *desc;
	396	struct scatterlist *sg;
	397	unsigned int i;
	398
	399	desc = jz4740_dma_alloc_desc(sg_len);
	400	if (!desc)
	401	return NULL;
	402
	403	for_each_sg(sgl, sg, sg_len, i) {
	404	desc->sg[i].addr = sg_dma_address(sg);
	405	desc->sg[i].len = sg_dma_len(sg);
	406	}
	407
	408	desc->num_sgs = sg_len;
	409	desc->direction = direction;
	410	desc->cyclic = false;
	411
	412	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	413	}
	414
	415	static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	416	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	417	size_t period_len, enum dma_transfer_direction direction,
31c1e5a1	418	unsigned long flags)
7c169a42 LPC	419	{
	420	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	421	struct jz4740_dma_desc *desc;
	422	unsigned int num_periods, i;
	423
	424	if (buf_len % period_len)
	425	return NULL;
	426
	427	num_periods = buf_len / period_len;
	428
	429	desc = jz4740_dma_alloc_desc(num_periods);
	430	if (!desc)
	431	return NULL;
	432
	433	for (i = 0; i < num_periods; i++) {
	434	desc->sg[i].addr = buf_addr;
	435	desc->sg[i].len = period_len;
	436	buf_addr += period_len;
	437	}
	438
	439	desc->num_sgs = num_periods;
	440	desc->direction = direction;
	441	desc->cyclic = true;
	442
	443	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	444	}
	445
	446	static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	447	struct jz4740_dma_desc *desc, unsigned int next_sg)
	448	{
25ce6c35 LPC	449	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
25ce6c35 LPC	450	unsigned int residue, count;
7c169a42 LPC	451	unsigned int i;
	452
	453	residue = 0;
	454
	455	for (i = next_sg; i < desc->num_sgs; i++)
	456	residue += desc->sg[i].len;
	457
25ce6c35 LPC	458	if (next_sg != 0) {
	459	count = jz4740_dma_read(dmadev,
	460	JZ_REG_DMA_TRANSFER_COUNT(chan->id));
	461	residue += count << chan->transfer_shift;
	462	}
7c169a42 LPC	463
	464	return residue;
	465	}
	466
	467	static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	468	dma_cookie_t cookie, struct dma_tx_state *state)
	469	{
	470	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	471	struct virt_dma_desc *vdesc;
	472	enum dma_status status;
	473	unsigned long flags;
	474
	475	status = dma_cookie_status(c, cookie, state);
a605c48b	476	if (status == DMA_COMPLETE \|\| !state)
7c169a42 LPC	477	return status;
	478
	479	spin_lock_irqsave(&chan->vchan.lock, flags);
	480	vdesc = vchan_find_desc(&chan->vchan, cookie);
	481	if (cookie == chan->desc->vdesc.tx.cookie) {
	482	state->residue = jz4740_dma_desc_residue(chan, chan->desc,
	483	chan->next_sg);
	484	} else if (vdesc) {
	485	state->residue = jz4740_dma_desc_residue(chan,
	486	to_jz4740_dma_desc(vdesc), 0);
	487	} else {
	488	state->residue = 0;
	489	}
	490	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	491
	492	return status;
	493	}
	494
7c169a42 LPC	495	static void jz4740_dma_free_chan_resources(struct dma_chan *c)
7c169a42 LPC	496	{
25ce6c35	497	vchan_free_chan_resources(to_virt_chan(c));
7c169a42 LPC	498	}
	499
	500	static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
	501	{
	502	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
	503	}
	504
ca766839 LPC	505	#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) \| \
	506	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) \| BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
	507
7c169a42 LPC	508	static int jz4740_dma_probe(struct platform_device *pdev)
	509	{
	510	struct jz4740_dmaengine_chan *chan;
	511	struct jz4740_dma_dev *dmadev;
	512	struct dma_device *dd;
	513	unsigned int i;
25ce6c35	514	struct resource *res;
7c169a42	515	int ret;
25ce6c35	516	int irq;
7c169a42 LPC	517
	518	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	519	if (!dmadev)
	520	return -EINVAL;
	521
	522	dd = &dmadev->ddev;
	523
25ce6c35 LPC	524	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	525	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	526	if (IS_ERR(dmadev->base))
	527	return PTR_ERR(dmadev->base);
	528
	529	dmadev->clk = clk_get(&pdev->dev, "dma");
	530	if (IS_ERR(dmadev->clk))
	531	return PTR_ERR(dmadev->clk);
	532
	533	clk_prepare_enable(dmadev->clk);
	534
7c169a42 LPC	535	dma_cap_set(DMA_SLAVE, dd->cap_mask);
7c169a42 LPC	536	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
7c169a42 LPC	537	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	538	dd->device_tx_status = jz4740_dma_tx_status;
	539	dd->device_issue_pending = jz4740_dma_issue_pending;
	540	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	541	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
1d4c0b8c MR	542	dd->device_config = jz4740_dma_slave_config;
1d4c0b8c MR	543	dd->device_terminate_all = jz4740_dma_terminate_all;
ca766839 LPC	544	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	545	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	546	dd->directions = BIT(DMA_DEV_TO_MEM) \| BIT(DMA_MEM_TO_DEV);
	547	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
7c169a42	548	dd->dev = &pdev->dev;
7c169a42 LPC	549	INIT_LIST_HEAD(&dd->channels);
7c169a42 LPC	550
b2c100e0	551	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
7c169a42	552	chan = &dmadev->chan[i];
25ce6c35	553	chan->id = i;
7c169a42 LPC	554	chan->vchan.desc_free = jz4740_dma_desc_free;
	555	vchan_init(&chan->vchan, dd);
	556	}
	557
	558	ret = dma_async_device_register(dd);
	559	if (ret)
	560	return ret;
	561
25ce6c35 LPC	562	irq = platform_get_irq(pdev, 0);
	563	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	564	if (ret)
	565	goto err_unregister;
	566
7c169a42 LPC	567	platform_set_drvdata(pdev, dmadev);
	568
	569	return 0;
25ce6c35 LPC	570
	571	err_unregister:
	572	dma_async_device_unregister(dd);
	573	return ret;
7c169a42 LPC	574	}
	575
	576	static int jz4740_dma_remove(struct platform_device *pdev)
	577	{
	578	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
25ce6c35	579	int irq = platform_get_irq(pdev, 0);
7c169a42	580
25ce6c35	581	free_irq(irq, dmadev);
7c169a42	582	dma_async_device_unregister(&dmadev->ddev);
25ce6c35	583	clk_disable_unprepare(dmadev->clk);
7c169a42 LPC	584
	585	return 0;
	586	}
	587
	588	static struct platform_driver jz4740_dma_driver = {
	589	.probe = jz4740_dma_probe,
	590	.remove = jz4740_dma_remove,
	591	.driver = {
	592	.name = "jz4740-dma",
7c169a42 LPC	593	},
	594	};
	595	module_platform_driver(jz4740_dma_driver);
	596
	597	MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
	598	MODULE_DESCRIPTION("JZ4740 DMA driver");
e2f9922a	599	MODULE_LICENSE("GPL v2");