[linux-2.6-block.git] / drivers / dma / dma-jz4740.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
 *  JZ4740 DMAC support
 */

#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/clk.h>

#include "virt-dma.h"

#define JZ_DMA_NR_CHANS 6

#define JZ_REG_DMA_SRC_ADDR(x)		(0x00 + (x) * 0x20)
#define JZ_REG_DMA_DST_ADDR(x)		(0x04 + (x) * 0x20)
#define JZ_REG_DMA_TRANSFER_COUNT(x)	(0x08 + (x) * 0x20)
#define JZ_REG_DMA_REQ_TYPE(x)		(0x0C + (x) * 0x20)
#define JZ_REG_DMA_STATUS_CTRL(x)	(0x10 + (x) * 0x20)
#define JZ_REG_DMA_CMD(x)		(0x14 + (x) * 0x20)
#define JZ_REG_DMA_DESC_ADDR(x)		(0x18 + (x) * 0x20)

#define JZ_REG_DMA_CTRL			0x300
#define JZ_REG_DMA_IRQ			0x304
#define JZ_REG_DMA_DOORBELL		0x308
#define JZ_REG_DMA_DOORBELL_SET		0x30C

#define JZ_DMA_STATUS_CTRL_NO_DESC		BIT(31)
#define JZ_DMA_STATUS_CTRL_DESC_INV		BIT(6)
#define JZ_DMA_STATUS_CTRL_ADDR_ERR		BIT(4)
#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE	BIT(3)
#define JZ_DMA_STATUS_CTRL_HALT			BIT(2)
#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE	BIT(1)
#define JZ_DMA_STATUS_CTRL_ENABLE		BIT(0)

#define JZ_DMA_CMD_SRC_INC			BIT(23)
#define JZ_DMA_CMD_DST_INC			BIT(22)
#define JZ_DMA_CMD_RDIL_MASK			(0xf << 16)
#define JZ_DMA_CMD_SRC_WIDTH_MASK		(0x3 << 14)
#define JZ_DMA_CMD_DST_WIDTH_MASK		(0x3 << 12)
#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK		(0x7 << 8)
#define JZ_DMA_CMD_BLOCK_MODE			BIT(7)
#define JZ_DMA_CMD_DESC_VALID			BIT(4)
#define JZ_DMA_CMD_DESC_VALID_MODE		BIT(3)
#define JZ_DMA_CMD_VALID_IRQ_ENABLE		BIT(2)
#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE		BIT(1)
#define JZ_DMA_CMD_LINK_ENABLE			BIT(0)

#define JZ_DMA_CMD_FLAGS_OFFSET 22
#define JZ_DMA_CMD_RDIL_OFFSET 16
#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
#define JZ_DMA_CMD_MODE_OFFSET 7

#define JZ_DMA_CTRL_PRIORITY_MASK		(0x3 << 8)
#define JZ_DMA_CTRL_HALT			BIT(3)
#define JZ_DMA_CTRL_ADDRESS_ERROR		BIT(2)
#define JZ_DMA_CTRL_ENABLE			BIT(0)

enum jz4740_dma_width {
	JZ4740_DMA_WIDTH_32BIT	= 0,
	JZ4740_DMA_WIDTH_8BIT	= 1,
	JZ4740_DMA_WIDTH_16BIT	= 2,
};

enum jz4740_dma_transfer_size {
	JZ4740_DMA_TRANSFER_SIZE_4BYTE	= 0,
	JZ4740_DMA_TRANSFER_SIZE_1BYTE	= 1,
	JZ4740_DMA_TRANSFER_SIZE_2BYTE	= 2,
	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
};

enum jz4740_dma_flags {
	JZ4740_DMA_SRC_AUTOINC = 0x2,
	JZ4740_DMA_DST_AUTOINC = 0x1,
};

enum jz4740_dma_mode {
	JZ4740_DMA_MODE_SINGLE	= 0,
	JZ4740_DMA_MODE_BLOCK	= 1,
};

struct jz4740_dma_sg {
	dma_addr_t addr;
	unsigned int len;
};

struct jz4740_dma_desc {
	struct virt_dma_desc vdesc;

	enum dma_transfer_direction direction;
	bool cyclic;

	unsigned int num_sgs;
	struct jz4740_dma_sg sg[];
};

struct jz4740_dmaengine_chan {
	struct virt_dma_chan vchan;
	unsigned int id;
	struct dma_slave_config config;

	dma_addr_t fifo_addr;
	unsigned int transfer_shift;

	struct jz4740_dma_desc *desc;
	unsigned int next_sg;
};

struct jz4740_dma_dev {
	struct dma_device ddev;
	void __iomem *base;
	struct clk *clk;

	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
};

static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	struct jz4740_dmaengine_chan *chan)
{
	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
		ddev);
}

static struct jz4740_dmaengine_chan *to_jz4740_dma_chan(struct dma_chan *c)
{
	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
}

static struct jz4740_dma_desc *to_jz4740_dma_desc(struct virt_dma_desc *vdesc)
{
	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
}

static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	unsigned int reg)
{
	return readl(dmadev->base + reg);
}

static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	unsigned reg, uint32_t val)
{
	writel(val, dmadev->base + reg);
}

static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	unsigned int reg, uint32_t val, uint32_t mask)
{
	uint32_t tmp;

	tmp = jz4740_dma_read(dmadev, reg);
	tmp &= ~mask;
	tmp |= val;
	jz4740_dma_write(dmadev, reg, tmp);
}

static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
{
	return kzalloc(sizeof(struct jz4740_dma_desc) +
		sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
}

static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
{
	switch (width) {
	case DMA_SLAVE_BUSWIDTH_1_BYTE:
		return JZ4740_DMA_WIDTH_8BIT;
	case DMA_SLAVE_BUSWIDTH_2_BYTES:
		return JZ4740_DMA_WIDTH_16BIT;
	case DMA_SLAVE_BUSWIDTH_4_BYTES:
		return JZ4740_DMA_WIDTH_32BIT;
	default:
		return JZ4740_DMA_WIDTH_32BIT;
	}
}

static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
{
	if (maxburst <= 1)
		return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	else if (maxburst <= 3)
		return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	else if (maxburst <= 15)
		return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	else if (maxburst <= 31)
		return JZ4740_DMA_TRANSFER_SIZE_16BYTE;

	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
}

static int jz4740_dma_slave_config_write(struct dma_chan *c,
				   struct dma_slave_config *config,
				   enum dma_transfer_direction direction)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	enum jz4740_dma_width src_width;
	enum jz4740_dma_width dst_width;
	enum jz4740_dma_transfer_size transfer_size;
	enum jz4740_dma_flags flags;
	uint32_t cmd;

	switch (direction) {
	case DMA_MEM_TO_DEV:
		flags = JZ4740_DMA_SRC_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
		chan->fifo_addr = config->dst_addr;
		break;
	case DMA_DEV_TO_MEM:
		flags = JZ4740_DMA_DST_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->src_maxburst);
		chan->fifo_addr = config->src_addr;
		break;
	default:
		return -EINVAL;
	}

	src_width = jz4740_dma_width(config->src_addr_width);
	dst_width = jz4740_dma_width(config->dst_addr_width);

	switch (transfer_size) {
	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
		chan->transfer_shift = 1;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
		chan->transfer_shift = 2;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
		chan->transfer_shift = 4;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
		chan->transfer_shift = 5;
		break;
	default:
		chan->transfer_shift = 0;
		break;
	}

	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	cmd |= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	cmd |= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	cmd |= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	cmd |= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	cmd |= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;

	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
		config->slave_id);

	return 0;
}

static int jz4740_dma_slave_config(struct dma_chan *c,
				   struct dma_slave_config *config)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);

	memcpy(&chan->config, config, sizeof(*config));
	return 0;
}

static int jz4740_dma_terminate_all(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&chan->vchan.lock, flags);
	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);
	chan->desc = NULL;
	vchan_get_all_descriptors(&chan->vchan, &head);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	vchan_dma_desc_free_list(&chan->vchan, &head);

	return 0;
}

static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	dma_addr_t src_addr, dst_addr;
	struct virt_dma_desc *vdesc;
	struct jz4740_dma_sg *sg;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);

	if (!chan->desc) {
		vdesc = vchan_next_desc(&chan->vchan);
		if (!vdesc)
			return 0;
		chan->desc = to_jz4740_dma_desc(vdesc);
		chan->next_sg = 0;
	}

	if (chan->next_sg == chan->desc->num_sgs)
		chan->next_sg = 0;

	sg = &chan->desc->sg[chan->next_sg];

	if (chan->desc->direction == DMA_MEM_TO_DEV) {
		src_addr = sg->addr;
		dst_addr = chan->fifo_addr;
	} else {
		src_addr = chan->fifo_addr;
		dst_addr = sg->addr;
	}
	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
			sg->len >> chan->transfer_shift);

	chan->next_sg++;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
			JZ_DMA_STATUS_CTRL_NO_DESC | JZ_DMA_STATUS_CTRL_ENABLE,
			JZ_DMA_STATUS_CTRL_HALT | JZ_DMA_STATUS_CTRL_NO_DESC |
			JZ_DMA_STATUS_CTRL_ENABLE);

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
			JZ_DMA_CTRL_ENABLE,
			JZ_DMA_CTRL_HALT | JZ_DMA_CTRL_ENABLE);

	return 0;
}

static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
{
	spin_lock(&chan->vchan.lock);
	if (chan->desc) {
		if (chan->desc->cyclic) {
			vchan_cyclic_callback(&chan->desc->vdesc);
		} else {
			if (chan->next_sg == chan->desc->num_sgs) {
				list_del(&chan->desc->vdesc.node);
				vchan_cookie_complete(&chan->desc->vdesc);
				chan->desc = NULL;
			}
		}
	}
	jz4740_dma_start_transfer(chan);
	spin_unlock(&chan->vchan.lock);
}

static irqreturn_t jz4740_dma_irq(int irq, void *devid)
{
	struct jz4740_dma_dev *dmadev = devid;
	uint32_t irq_status;
	unsigned int i;

	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);

	for (i = 0; i < 6; ++i) {
		if (irq_status & (1 << i)) {
			jz4740_dma_write_mask(dmadev,
				JZ_REG_DMA_STATUS_CTRL(i), 0,
				JZ_DMA_STATUS_CTRL_ENABLE |
				JZ_DMA_STATUS_CTRL_TRANSFER_DONE);

			jz4740_dma_chan_irq(&dmadev->chan[i]);
		}
	}

	return IRQ_HANDLED;
}

static void jz4740_dma_issue_pending(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	unsigned long flags;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
		jz4740_dma_start_transfer(chan);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	struct dma_chan *c, struct scatterlist *sgl,
	unsigned int sg_len, enum dma_transfer_direction direction,
	unsigned long flags, void *context)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	struct scatterlist *sg;
	unsigned int i;

	desc = jz4740_dma_alloc_desc(sg_len);
	if (!desc)
		return NULL;

	for_each_sg(sgl, sg, sg_len, i) {
		desc->sg[i].addr = sg_dma_address(sg);
		desc->sg[i].len = sg_dma_len(sg);
	}

	desc->num_sgs = sg_len;
	desc->direction = direction;
	desc->cyclic = false;

	jz4740_dma_slave_config_write(c, &chan->config, direction);

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	size_t period_len, enum dma_transfer_direction direction,
	unsigned long flags)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	unsigned int num_periods, i;

	if (buf_len % period_len)
		return NULL;

	num_periods = buf_len / period_len;

	desc = jz4740_dma_alloc_desc(num_periods);
	if (!desc)
		return NULL;

	for (i = 0; i < num_periods; i++) {
		desc->sg[i].addr = buf_addr;
		desc->sg[i].len = period_len;
		buf_addr += period_len;
	}

	desc->num_sgs = num_periods;
	desc->direction = direction;
	desc->cyclic = true;

	jz4740_dma_slave_config_write(c, &chan->config, direction);

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	struct jz4740_dma_desc *desc, unsigned int next_sg)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned int residue, count;
	unsigned int i;

	residue = 0;

	for (i = next_sg; i < desc->num_sgs; i++)
		residue += desc->sg[i].len;

	if (next_sg != 0) {
		count = jz4740_dma_read(dmadev,
			JZ_REG_DMA_TRANSFER_COUNT(chan->id));
		residue += count << chan->transfer_shift;
	}

	return residue;
}

static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	dma_cookie_t cookie, struct dma_tx_state *state)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct virt_dma_desc *vdesc;
	enum dma_status status;
	unsigned long flags;

	status = dma_cookie_status(c, cookie, state);
	if (status == DMA_COMPLETE || !state)
		return status;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	vdesc = vchan_find_desc(&chan->vchan, cookie);
	if (cookie == chan->desc->vdesc.tx.cookie) {
		state->residue = jz4740_dma_desc_residue(chan, chan->desc,
				chan->next_sg);
	} else if (vdesc) {
		state->residue = jz4740_dma_desc_residue(chan,
				to_jz4740_dma_desc(vdesc), 0);
	} else {
		state->residue = 0;
	}
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	return status;
}

static void jz4740_dma_free_chan_resources(struct dma_chan *c)
{
	vchan_free_chan_resources(to_virt_chan(c));
}

static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
{
	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
}

#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))

static int jz4740_dma_probe(struct platform_device *pdev)
{
	struct jz4740_dmaengine_chan *chan;
	struct jz4740_dma_dev *dmadev;
	struct dma_device *dd;
	unsigned int i;
	struct resource *res;
	int ret;
	int irq;

	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	if (!dmadev)
		return -EINVAL;

	dd = &dmadev->ddev;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	if (IS_ERR(dmadev->base))
		return PTR_ERR(dmadev->base);

	dmadev->clk = clk_get(&pdev->dev, "dma");
	if (IS_ERR(dmadev->clk))
		return PTR_ERR(dmadev->clk);

	clk_prepare_enable(dmadev->clk);

	dma_cap_set(DMA_SLAVE, dd->cap_mask);
	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	dd->device_tx_status = jz4740_dma_tx_status;
	dd->device_issue_pending = jz4740_dma_issue_pending;
	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
	dd->device_config = jz4740_dma_slave_config;
	dd->device_terminate_all = jz4740_dma_terminate_all;
	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
	dd->dev = &pdev->dev;
	INIT_LIST_HEAD(&dd->channels);

	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
		chan = &dmadev->chan[i];
		chan->id = i;
		chan->vchan.desc_free = jz4740_dma_desc_free;
		vchan_init(&chan->vchan, dd);
	}

	ret = dma_async_device_register(dd);
	if (ret)
		goto err_clk;

	irq = platform_get_irq(pdev, 0);
	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	if (ret)
		goto err_unregister;

	platform_set_drvdata(pdev, dmadev);

	return 0;

err_unregister:
	dma_async_device_unregister(dd);
err_clk:
	clk_disable_unprepare(dmadev->clk);
	return ret;
}

static void jz4740_cleanup_vchan(struct dma_device *dmadev)
{
	struct jz4740_dmaengine_chan *chan, *_chan;

	list_for_each_entry_safe(chan, _chan,
				&dmadev->channels, vchan.chan.device_node) {
		list_del(&chan->vchan.chan.device_node);
		tasklet_kill(&chan->vchan.task);
	}
}


static int jz4740_dma_remove(struct platform_device *pdev)
{
	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
	int irq = platform_get_irq(pdev, 0);

	free_irq(irq, dmadev);

	jz4740_cleanup_vchan(&dmadev->ddev);
	dma_async_device_unregister(&dmadev->ddev);
	clk_disable_unprepare(dmadev->clk);

	return 0;
}

static struct platform_driver jz4740_dma_driver = {
	.probe = jz4740_dma_probe,
	.remove = jz4740_dma_remove,
	.driver = {
		.name = "jz4740-dma",
	},
};
module_platform_driver(jz4740_dma_driver);

MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
MODULE_DESCRIPTION("JZ4740 DMA driver");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
7c169a42 LPC	2	/*
	3	* Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
	4	* JZ4740 DMAC support
7c169a42 LPC	5	*/
	6
	7	#include <linux/dmaengine.h>
	8	#include <linux/dma-mapping.h>
	9	#include <linux/err.h>
	10	#include <linux/init.h>
	11	#include <linux/list.h>
	12	#include <linux/module.h>
	13	#include <linux/platform_device.h>
	14	#include <linux/slab.h>
	15	#include <linux/spinlock.h>
25ce6c35 LPC	16	#include <linux/irq.h>
25ce6c35 LPC	17	#include <linux/clk.h>
7c169a42	18
7c169a42 LPC	19	#include "virt-dma.h"
	20
	21	#define JZ_DMA_NR_CHANS 6
	22
25ce6c35 LPC	23	#define JZ_REG_DMA_SRC_ADDR(x) (0x00 + (x) * 0x20)
	24	#define JZ_REG_DMA_DST_ADDR(x) (0x04 + (x) * 0x20)
	25	#define JZ_REG_DMA_TRANSFER_COUNT(x) (0x08 + (x) * 0x20)
	26	#define JZ_REG_DMA_REQ_TYPE(x) (0x0C + (x) * 0x20)
	27	#define JZ_REG_DMA_STATUS_CTRL(x) (0x10 + (x) * 0x20)
	28	#define JZ_REG_DMA_CMD(x) (0x14 + (x) * 0x20)
	29	#define JZ_REG_DMA_DESC_ADDR(x) (0x18 + (x) * 0x20)
	30
	31	#define JZ_REG_DMA_CTRL 0x300
	32	#define JZ_REG_DMA_IRQ 0x304
	33	#define JZ_REG_DMA_DOORBELL 0x308
	34	#define JZ_REG_DMA_DOORBELL_SET 0x30C
	35
	36	#define JZ_DMA_STATUS_CTRL_NO_DESC BIT(31)
	37	#define JZ_DMA_STATUS_CTRL_DESC_INV BIT(6)
	38	#define JZ_DMA_STATUS_CTRL_ADDR_ERR BIT(4)
	39	#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE BIT(3)
	40	#define JZ_DMA_STATUS_CTRL_HALT BIT(2)
	41	#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE BIT(1)
	42	#define JZ_DMA_STATUS_CTRL_ENABLE BIT(0)
	43
	44	#define JZ_DMA_CMD_SRC_INC BIT(23)
	45	#define JZ_DMA_CMD_DST_INC BIT(22)
	46	#define JZ_DMA_CMD_RDIL_MASK (0xf << 16)
	47	#define JZ_DMA_CMD_SRC_WIDTH_MASK (0x3 << 14)
	48	#define JZ_DMA_CMD_DST_WIDTH_MASK (0x3 << 12)
	49	#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK (0x7 << 8)
	50	#define JZ_DMA_CMD_BLOCK_MODE BIT(7)
	51	#define JZ_DMA_CMD_DESC_VALID BIT(4)
	52	#define JZ_DMA_CMD_DESC_VALID_MODE BIT(3)
	53	#define JZ_DMA_CMD_VALID_IRQ_ENABLE BIT(2)
	54	#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE BIT(1)
	55	#define JZ_DMA_CMD_LINK_ENABLE BIT(0)
	56
	57	#define JZ_DMA_CMD_FLAGS_OFFSET 22
	58	#define JZ_DMA_CMD_RDIL_OFFSET 16
	59	#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
	60	#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
	61	#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
	62	#define JZ_DMA_CMD_MODE_OFFSET 7
	63
	64	#define JZ_DMA_CTRL_PRIORITY_MASK (0x3 << 8)
	65	#define JZ_DMA_CTRL_HALT BIT(3)
	66	#define JZ_DMA_CTRL_ADDRESS_ERROR BIT(2)
	67	#define JZ_DMA_CTRL_ENABLE BIT(0)
	68
	69	enum jz4740_dma_width {
	70	JZ4740_DMA_WIDTH_32BIT = 0,
	71	JZ4740_DMA_WIDTH_8BIT = 1,
	72	JZ4740_DMA_WIDTH_16BIT = 2,
	73	};
	74
	75	enum jz4740_dma_transfer_size {
	76	JZ4740_DMA_TRANSFER_SIZE_4BYTE = 0,
	77	JZ4740_DMA_TRANSFER_SIZE_1BYTE = 1,
	78	JZ4740_DMA_TRANSFER_SIZE_2BYTE = 2,
	79	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	80	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
	81	};
	82
	83	enum jz4740_dma_flags {
	84	JZ4740_DMA_SRC_AUTOINC = 0x2,
	85	JZ4740_DMA_DST_AUTOINC = 0x1,
	86	};
87
88	enum jz4740_dma_mode {
89	JZ4740_DMA_MODE_SINGLE = 0,
90	JZ4740_DMA_MODE_BLOCK = 1,
91	};
92
7c169a42 LPC	93	struct jz4740_dma_sg {
	94	dma_addr_t addr;
	95	unsigned int len;
	96	};
	97
	98	struct jz4740_dma_desc {
	99	struct virt_dma_desc vdesc;
	100
	101	enum dma_transfer_direction direction;
	102	bool cyclic;
	103
	104	unsigned int num_sgs;
	105	struct jz4740_dma_sg sg[];
	106	};
	107
	108	struct jz4740_dmaengine_chan {
	109	struct virt_dma_chan vchan;
25ce6c35	110	unsigned int id;
09347e3f	111	struct dma_slave_config config;
7c169a42 LPC	112
7c169a42 LPC	113	dma_addr_t fifo_addr;
25ce6c35	114	unsigned int transfer_shift;
7c169a42 LPC	115
	116	struct jz4740_dma_desc *desc;
	117	unsigned int next_sg;
	118	};
	119
	120	struct jz4740_dma_dev {
	121	struct dma_device ddev;
25ce6c35 LPC	122	void __iomem *base;
25ce6c35 LPC	123	struct clk *clk;
7c169a42 LPC	124
	125	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
	126	};
	127
25ce6c35 LPC	128	static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	129	struct jz4740_dmaengine_chan *chan)
	130	{
	131	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
	132	ddev);
	133	}
	134
7c169a42 LPC	135	static struct jz4740_dmaengine_chan to_jz4740_dma_chan(struct dma_chan c)
	136	{
	137	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
	138	}
	139
	140	static struct jz4740_dma_desc to_jz4740_dma_desc(struct virt_dma_desc vdesc)
	141	{
	142	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
	143	}
	144
25ce6c35 LPC	145	static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	146	unsigned int reg)
	147	{
	148	return readl(dmadev->base + reg);
	149	}
	150
	151	static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	152	unsigned reg, uint32_t val)
	153	{
	154	writel(val, dmadev->base + reg);
	155	}
	156
	157	static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	158	unsigned int reg, uint32_t val, uint32_t mask)
	159	{
	160	uint32_t tmp;
	161
	162	tmp = jz4740_dma_read(dmadev, reg);
	163	tmp &= ~mask;
	164	tmp \|= val;
	165	jz4740_dma_write(dmadev, reg, tmp);
	166	}
	167
7c169a42 LPC	168	static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
	169	{
	170	return kzalloc(sizeof(struct jz4740_dma_desc) +
	171	sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
	172	}
	173
	174	static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
	175	{
	176	switch (width) {
	177	case DMA_SLAVE_BUSWIDTH_1_BYTE:
	178	return JZ4740_DMA_WIDTH_8BIT;
	179	case DMA_SLAVE_BUSWIDTH_2_BYTES:
	180	return JZ4740_DMA_WIDTH_16BIT;
	181	case DMA_SLAVE_BUSWIDTH_4_BYTES:
	182	return JZ4740_DMA_WIDTH_32BIT;
	183	default:
	184	return JZ4740_DMA_WIDTH_32BIT;
	185	}
	186	}
	187
	188	static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
	189	{
	190	if (maxburst <= 1)
	191	return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	192	else if (maxburst <= 3)
	193	return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	194	else if (maxburst <= 15)
	195	return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	196	else if (maxburst <= 31)
	197	return JZ4740_DMA_TRANSFER_SIZE_16BYTE;
	198
	199	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
	200	}
	201
09347e3f VK	202	static int jz4740_dma_slave_config_write(struct dma_chan *c,
	203	struct dma_slave_config *config,
	204	enum dma_transfer_direction direction)
7c169a42 LPC	205	{
7c169a42 LPC	206	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35 LPC	207	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	208	enum jz4740_dma_width src_width;
	209	enum jz4740_dma_width dst_width;
	210	enum jz4740_dma_transfer_size transfer_size;
	211	enum jz4740_dma_flags flags;
	212	uint32_t cmd;
7c169a42	213
09347e3f	214	switch (direction) {
7c169a42	215	case DMA_MEM_TO_DEV:
25ce6c35 LPC	216	flags = JZ4740_DMA_SRC_AUTOINC;
25ce6c35 LPC	217	transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
7c169a42 LPC	218	chan->fifo_addr = config->dst_addr;
	219	break;
	220	case DMA_DEV_TO_MEM:
25ce6c35 LPC	221	flags = JZ4740_DMA_DST_AUTOINC;
25ce6c35 LPC	222	transfer_size = jz4740_dma_maxburst(config->src_maxburst);
7c169a42 LPC	223	chan->fifo_addr = config->src_addr;
	224	break;
	225	default:
	226	return -EINVAL;
	227	}
	228
25ce6c35 LPC	229	src_width = jz4740_dma_width(config->src_addr_width);
	230	dst_width = jz4740_dma_width(config->dst_addr_width);
	231
	232	switch (transfer_size) {
	233	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
	234	chan->transfer_shift = 1;
	235	break;
	236	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
	237	chan->transfer_shift = 2;
	238	break;
	239	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
	240	chan->transfer_shift = 4;
	241	break;
	242	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
	243	chan->transfer_shift = 5;
	244	break;
	245	default:
	246	chan->transfer_shift = 0;
	247	break;
	248	}
7c169a42	249
25ce6c35 LPC	250	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	251	cmd \|= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	252	cmd \|= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	253	cmd \|= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	254	cmd \|= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	255	cmd \|= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;
7c169a42	256
25ce6c35 LPC	257	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	258	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	259	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
	260	config->slave_id);
7c169a42 LPC	261
	262	return 0;
	263	}
	264
09347e3f VK	265	static int jz4740_dma_slave_config(struct dma_chan *c,
	266	struct dma_slave_config *config)
	267	{
	268	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	269
	270	memcpy(&chan->config, config, sizeof(*config));
	271	return 0;
	272	}
	273
7c169a42 LPC	274	static int jz4740_dma_terminate_all(struct dma_chan *c)
	275	{
	276	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35	277	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	278	unsigned long flags;
	279	LIST_HEAD(head);
	280
	281	spin_lock_irqsave(&chan->vchan.lock, flags);
25ce6c35 LPC	282	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	283	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	284	chan->desc = NULL;
	285	vchan_get_all_descriptors(&chan->vchan, &head);
	286	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	287
	288	vchan_dma_desc_free_list(&chan->vchan, &head);
	289
	290	return 0;
	291	}
	292
7c169a42 LPC	293	static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
7c169a42 LPC	294	{
25ce6c35	295	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	296	dma_addr_t src_addr, dst_addr;
	297	struct virt_dma_desc *vdesc;
	298	struct jz4740_dma_sg *sg;
	299
25ce6c35 LPC	300	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	301	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	302
	303	if (!chan->desc) {
	304	vdesc = vchan_next_desc(&chan->vchan);
	305	if (!vdesc)
	306	return 0;
	307	chan->desc = to_jz4740_dma_desc(vdesc);
	308	chan->next_sg = 0;
	309	}
	310
	311	if (chan->next_sg == chan->desc->num_sgs)
	312	chan->next_sg = 0;
	313
	314	sg = &chan->desc->sg[chan->next_sg];
	315
	316	if (chan->desc->direction == DMA_MEM_TO_DEV) {
	317	src_addr = sg->addr;
	318	dst_addr = chan->fifo_addr;
	319	} else {
	320	src_addr = chan->fifo_addr;
	321	dst_addr = sg->addr;
	322	}
25ce6c35 LPC	323	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	324	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	325	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
	326	sg->len >> chan->transfer_shift);
7c169a42 LPC	327
	328	chan->next_sg++;
	329
25ce6c35 LPC	330	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
	331	JZ_DMA_STATUS_CTRL_NO_DESC \| JZ_DMA_STATUS_CTRL_ENABLE,
	332	JZ_DMA_STATUS_CTRL_HALT \| JZ_DMA_STATUS_CTRL_NO_DESC \|
	333	JZ_DMA_STATUS_CTRL_ENABLE);
	334
	335	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
	336	JZ_DMA_CTRL_ENABLE,
	337	JZ_DMA_CTRL_HALT \| JZ_DMA_CTRL_ENABLE);
7c169a42 LPC	338
	339	return 0;
	340	}
	341
25ce6c35	342	static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
7c169a42	343	{
7c169a42 LPC	344	spin_lock(&chan->vchan.lock);
7c169a42 LPC	345	if (chan->desc) {
26f7af37	346	if (chan->desc->cyclic) {
7c169a42 LPC	347	vchan_cyclic_callback(&chan->desc->vdesc);
	348	} else {
	349	if (chan->next_sg == chan->desc->num_sgs) {
f498e064	350	list_del(&chan->desc->vdesc.node);
7c169a42	351	vchan_cookie_complete(&chan->desc->vdesc);
f498e064	352	chan->desc = NULL;
7c169a42 LPC	353	}
	354	}
	355	}
	356	jz4740_dma_start_transfer(chan);
	357	spin_unlock(&chan->vchan.lock);
	358	}
	359
25ce6c35 LPC	360	static irqreturn_t jz4740_dma_irq(int irq, void *devid)
	361	{
	362	struct jz4740_dma_dev *dmadev = devid;
	363	uint32_t irq_status;
	364	unsigned int i;
	365
	366	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);
	367
	368	for (i = 0; i < 6; ++i) {
	369	if (irq_status & (1 << i)) {
	370	jz4740_dma_write_mask(dmadev,
	371	JZ_REG_DMA_STATUS_CTRL(i), 0,
	372	JZ_DMA_STATUS_CTRL_ENABLE \|
	373	JZ_DMA_STATUS_CTRL_TRANSFER_DONE);
	374
	375	jz4740_dma_chan_irq(&dmadev->chan[i]);
	376	}
	377	}
	378
	379	return IRQ_HANDLED;
	380	}
	381
7c169a42 LPC	382	static void jz4740_dma_issue_pending(struct dma_chan *c)
	383	{
	384	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	385	unsigned long flags;
	386
	387	spin_lock_irqsave(&chan->vchan.lock, flags);
	388	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
	389	jz4740_dma_start_transfer(chan);
	390	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	391	}
	392
	393	static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	394	struct dma_chan c, struct scatterlist sgl,
	395	unsigned int sg_len, enum dma_transfer_direction direction,
	396	unsigned long flags, void *context)
	397	{
	398	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	399	struct jz4740_dma_desc *desc;
	400	struct scatterlist *sg;
	401	unsigned int i;
	402
	403	desc = jz4740_dma_alloc_desc(sg_len);
	404	if (!desc)
	405	return NULL;
	406
	407	for_each_sg(sgl, sg, sg_len, i) {
	408	desc->sg[i].addr = sg_dma_address(sg);
	409	desc->sg[i].len = sg_dma_len(sg);
	410	}
	411
	412	desc->num_sgs = sg_len;
	413	desc->direction = direction;
	414	desc->cyclic = false;
	415
09347e3f VK	416	jz4740_dma_slave_config_write(c, &chan->config, direction);
09347e3f VK	417
7c169a42 LPC	418	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	419	}
	420
	421	static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	422	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	423	size_t period_len, enum dma_transfer_direction direction,
31c1e5a1	424	unsigned long flags)
7c169a42 LPC	425	{
	426	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	427	struct jz4740_dma_desc *desc;
	428	unsigned int num_periods, i;
	429
	430	if (buf_len % period_len)
	431	return NULL;
	432
	433	num_periods = buf_len / period_len;
	434
	435	desc = jz4740_dma_alloc_desc(num_periods);
	436	if (!desc)
	437	return NULL;
	438
	439	for (i = 0; i < num_periods; i++) {
	440	desc->sg[i].addr = buf_addr;
	441	desc->sg[i].len = period_len;
	442	buf_addr += period_len;
	443	}
	444
	445	desc->num_sgs = num_periods;
	446	desc->direction = direction;
	447	desc->cyclic = true;
	448
09347e3f VK	449	jz4740_dma_slave_config_write(c, &chan->config, direction);
09347e3f VK	450
7c169a42 LPC	451	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	452	}
	453
	454	static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	455	struct jz4740_dma_desc *desc, unsigned int next_sg)
	456	{
25ce6c35 LPC	457	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
25ce6c35 LPC	458	unsigned int residue, count;
7c169a42 LPC	459	unsigned int i;
	460
	461	residue = 0;
	462
	463	for (i = next_sg; i < desc->num_sgs; i++)
	464	residue += desc->sg[i].len;
	465
25ce6c35 LPC	466	if (next_sg != 0) {
	467	count = jz4740_dma_read(dmadev,
	468	JZ_REG_DMA_TRANSFER_COUNT(chan->id));
	469	residue += count << chan->transfer_shift;
	470	}
7c169a42 LPC	471
	472	return residue;
	473	}
	474
	475	static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	476	dma_cookie_t cookie, struct dma_tx_state *state)
	477	{
	478	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	479	struct virt_dma_desc *vdesc;
	480	enum dma_status status;
	481	unsigned long flags;
	482
	483	status = dma_cookie_status(c, cookie, state);
a605c48b	484	if (status == DMA_COMPLETE \|\| !state)
7c169a42 LPC	485	return status;
	486
	487	spin_lock_irqsave(&chan->vchan.lock, flags);
	488	vdesc = vchan_find_desc(&chan->vchan, cookie);
	489	if (cookie == chan->desc->vdesc.tx.cookie) {
	490	state->residue = jz4740_dma_desc_residue(chan, chan->desc,
	491	chan->next_sg);
	492	} else if (vdesc) {
	493	state->residue = jz4740_dma_desc_residue(chan,
	494	to_jz4740_dma_desc(vdesc), 0);
	495	} else {
	496	state->residue = 0;
	497	}
	498	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	499
	500	return status;
	501	}
	502
7c169a42 LPC	503	static void jz4740_dma_free_chan_resources(struct dma_chan *c)
7c169a42 LPC	504	{
25ce6c35	505	vchan_free_chan_resources(to_virt_chan(c));
7c169a42 LPC	506	}
	507
	508	static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
	509	{
	510	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
	511	}
	512
ca766839 LPC	513	#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) \| \
	514	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) \| BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
	515
7c169a42 LPC	516	static int jz4740_dma_probe(struct platform_device *pdev)
	517	{
	518	struct jz4740_dmaengine_chan *chan;
	519	struct jz4740_dma_dev *dmadev;
	520	struct dma_device *dd;
	521	unsigned int i;
25ce6c35	522	struct resource *res;
7c169a42	523	int ret;
25ce6c35	524	int irq;
7c169a42 LPC	525
	526	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	527	if (!dmadev)
	528	return -EINVAL;
	529
	530	dd = &dmadev->ddev;
	531
25ce6c35 LPC	532	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	533	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	534	if (IS_ERR(dmadev->base))
	535	return PTR_ERR(dmadev->base);
	536
	537	dmadev->clk = clk_get(&pdev->dev, "dma");
	538	if (IS_ERR(dmadev->clk))
	539	return PTR_ERR(dmadev->clk);
	540
	541	clk_prepare_enable(dmadev->clk);
	542
7c169a42 LPC	543	dma_cap_set(DMA_SLAVE, dd->cap_mask);
7c169a42 LPC	544	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
7c169a42 LPC	545	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	546	dd->device_tx_status = jz4740_dma_tx_status;
	547	dd->device_issue_pending = jz4740_dma_issue_pending;
	548	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	549	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
1d4c0b8c MR	550	dd->device_config = jz4740_dma_slave_config;
1d4c0b8c MR	551	dd->device_terminate_all = jz4740_dma_terminate_all;
ca766839 LPC	552	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	553	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	554	dd->directions = BIT(DMA_DEV_TO_MEM) \| BIT(DMA_MEM_TO_DEV);
	555	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
7c169a42	556	dd->dev = &pdev->dev;
7c169a42 LPC	557	INIT_LIST_HEAD(&dd->channels);
7c169a42 LPC	558
b2c100e0	559	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
7c169a42	560	chan = &dmadev->chan[i];
25ce6c35	561	chan->id = i;
7c169a42 LPC	562	chan->vchan.desc_free = jz4740_dma_desc_free;
	563	vchan_init(&chan->vchan, dd);
	564	}
	565
	566	ret = dma_async_device_register(dd);
	567	if (ret)
eb943696	568	goto err_clk;
7c169a42	569
25ce6c35 LPC	570	irq = platform_get_irq(pdev, 0);
	571	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	572	if (ret)
	573	goto err_unregister;
	574
7c169a42 LPC	575	platform_set_drvdata(pdev, dmadev);
	576
	577	return 0;
25ce6c35 LPC	578
	579	err_unregister:
	580	dma_async_device_unregister(dd);
eb943696 TJ	581	err_clk:
eb943696 TJ	582	clk_disable_unprepare(dmadev->clk);
25ce6c35	583	return ret;
7c169a42 LPC	584	}
7c169a42 LPC	585
cec9cfa8 VK	586	static void jz4740_cleanup_vchan(struct dma_device *dmadev)
	587	{
	588	struct jz4740_dmaengine_chan chan, _chan;
	589
	590	list_for_each_entry_safe(chan, _chan,
	591	&dmadev->channels, vchan.chan.device_node) {
	592	list_del(&chan->vchan.chan.device_node);
	593	tasklet_kill(&chan->vchan.task);
	594	}
	595	}
	596
	597
7c169a42 LPC	598	static int jz4740_dma_remove(struct platform_device *pdev)
	599	{
	600	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
25ce6c35	601	int irq = platform_get_irq(pdev, 0);
7c169a42	602
25ce6c35	603	free_irq(irq, dmadev);
cec9cfa8 VK	604
cec9cfa8 VK	605	jz4740_cleanup_vchan(&dmadev->ddev);
7c169a42	606	dma_async_device_unregister(&dmadev->ddev);
25ce6c35	607	clk_disable_unprepare(dmadev->clk);
7c169a42 LPC	608
	609	return 0;
	610	}
	611
	612	static struct platform_driver jz4740_dma_driver = {
	613	.probe = jz4740_dma_probe,
	614	.remove = jz4740_dma_remove,
	615	.driver = {
	616	.name = "jz4740-dma",
7c169a42 LPC	617	},
	618	};
	619	module_platform_driver(jz4740_dma_driver);
	620
	621	MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
	622	MODULE_DESCRIPTION("JZ4740 DMA driver");
e2f9922a	623	MODULE_LICENSE("GPL v2");