[linux-2.6-block.git] / drivers / dma / dma-jz4740.c

/*
 *  Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
 *  JZ4740 DMAC support
 *
 *  This program is free software; you can redistribute it and/or modify it
 *  under  the terms of the GNU General	 Public License as published by the
 *  Free Software Foundation;  either version 2 of the License, or (at your
 *  option) any later version.
 *
 */

#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/clk.h>

#include "virt-dma.h"

#define JZ_DMA_NR_CHANS 6

#define JZ_REG_DMA_SRC_ADDR(x)		(0x00 + (x) * 0x20)
#define JZ_REG_DMA_DST_ADDR(x)		(0x04 + (x) * 0x20)
#define JZ_REG_DMA_TRANSFER_COUNT(x)	(0x08 + (x) * 0x20)
#define JZ_REG_DMA_REQ_TYPE(x)		(0x0C + (x) * 0x20)
#define JZ_REG_DMA_STATUS_CTRL(x)	(0x10 + (x) * 0x20)
#define JZ_REG_DMA_CMD(x)		(0x14 + (x) * 0x20)
#define JZ_REG_DMA_DESC_ADDR(x)		(0x18 + (x) * 0x20)

#define JZ_REG_DMA_CTRL			0x300
#define JZ_REG_DMA_IRQ			0x304
#define JZ_REG_DMA_DOORBELL		0x308
#define JZ_REG_DMA_DOORBELL_SET		0x30C

#define JZ_DMA_STATUS_CTRL_NO_DESC		BIT(31)
#define JZ_DMA_STATUS_CTRL_DESC_INV		BIT(6)
#define JZ_DMA_STATUS_CTRL_ADDR_ERR		BIT(4)
#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE	BIT(3)
#define JZ_DMA_STATUS_CTRL_HALT			BIT(2)
#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE	BIT(1)
#define JZ_DMA_STATUS_CTRL_ENABLE		BIT(0)

#define JZ_DMA_CMD_SRC_INC			BIT(23)
#define JZ_DMA_CMD_DST_INC			BIT(22)
#define JZ_DMA_CMD_RDIL_MASK			(0xf << 16)
#define JZ_DMA_CMD_SRC_WIDTH_MASK		(0x3 << 14)
#define JZ_DMA_CMD_DST_WIDTH_MASK		(0x3 << 12)
#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK		(0x7 << 8)
#define JZ_DMA_CMD_BLOCK_MODE			BIT(7)
#define JZ_DMA_CMD_DESC_VALID			BIT(4)
#define JZ_DMA_CMD_DESC_VALID_MODE		BIT(3)
#define JZ_DMA_CMD_VALID_IRQ_ENABLE		BIT(2)
#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE		BIT(1)
#define JZ_DMA_CMD_LINK_ENABLE			BIT(0)

#define JZ_DMA_CMD_FLAGS_OFFSET 22
#define JZ_DMA_CMD_RDIL_OFFSET 16
#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
#define JZ_DMA_CMD_MODE_OFFSET 7

#define JZ_DMA_CTRL_PRIORITY_MASK		(0x3 << 8)
#define JZ_DMA_CTRL_HALT			BIT(3)
#define JZ_DMA_CTRL_ADDRESS_ERROR		BIT(2)
#define JZ_DMA_CTRL_ENABLE			BIT(0)

enum jz4740_dma_width {
	JZ4740_DMA_WIDTH_32BIT	= 0,
	JZ4740_DMA_WIDTH_8BIT	= 1,
	JZ4740_DMA_WIDTH_16BIT	= 2,
};

enum jz4740_dma_transfer_size {
	JZ4740_DMA_TRANSFER_SIZE_4BYTE	= 0,
	JZ4740_DMA_TRANSFER_SIZE_1BYTE	= 1,
	JZ4740_DMA_TRANSFER_SIZE_2BYTE	= 2,
	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
};

enum jz4740_dma_flags {
	JZ4740_DMA_SRC_AUTOINC = 0x2,
	JZ4740_DMA_DST_AUTOINC = 0x1,
};

enum jz4740_dma_mode {
	JZ4740_DMA_MODE_SINGLE	= 0,
	JZ4740_DMA_MODE_BLOCK	= 1,
};

struct jz4740_dma_sg {
	dma_addr_t addr;
	unsigned int len;
};

struct jz4740_dma_desc {
	struct virt_dma_desc vdesc;

	enum dma_transfer_direction direction;
	bool cyclic;

	unsigned int num_sgs;
	struct jz4740_dma_sg sg[];
};

struct jz4740_dmaengine_chan {
	struct virt_dma_chan vchan;
	unsigned int id;
	struct dma_slave_config config;

	dma_addr_t fifo_addr;
	unsigned int transfer_shift;

	struct jz4740_dma_desc *desc;
	unsigned int next_sg;
};

struct jz4740_dma_dev {
	struct dma_device ddev;
	void __iomem *base;
	struct clk *clk;

	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
};

static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	struct jz4740_dmaengine_chan *chan)
{
	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
		ddev);
}

static struct jz4740_dmaengine_chan *to_jz4740_dma_chan(struct dma_chan *c)
{
	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
}

static struct jz4740_dma_desc *to_jz4740_dma_desc(struct virt_dma_desc *vdesc)
{
	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
}

static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	unsigned int reg)
{
	return readl(dmadev->base + reg);
}

static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	unsigned reg, uint32_t val)
{
	writel(val, dmadev->base + reg);
}

static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	unsigned int reg, uint32_t val, uint32_t mask)
{
	uint32_t tmp;

	tmp = jz4740_dma_read(dmadev, reg);
	tmp &= ~mask;
	tmp |= val;
	jz4740_dma_write(dmadev, reg, tmp);
}

static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
{
	return kzalloc(sizeof(struct jz4740_dma_desc) +
		sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
}

static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
{
	switch (width) {
	case DMA_SLAVE_BUSWIDTH_1_BYTE:
		return JZ4740_DMA_WIDTH_8BIT;
	case DMA_SLAVE_BUSWIDTH_2_BYTES:
		return JZ4740_DMA_WIDTH_16BIT;
	case DMA_SLAVE_BUSWIDTH_4_BYTES:
		return JZ4740_DMA_WIDTH_32BIT;
	default:
		return JZ4740_DMA_WIDTH_32BIT;
	}
}

static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
{
	if (maxburst <= 1)
		return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	else if (maxburst <= 3)
		return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	else if (maxburst <= 15)
		return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	else if (maxburst <= 31)
		return JZ4740_DMA_TRANSFER_SIZE_16BYTE;

	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
}

static int jz4740_dma_slave_config_write(struct dma_chan *c,
				   struct dma_slave_config *config,
				   enum dma_transfer_direction direction)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	enum jz4740_dma_width src_width;
	enum jz4740_dma_width dst_width;
	enum jz4740_dma_transfer_size transfer_size;
	enum jz4740_dma_flags flags;
	uint32_t cmd;

	switch (direction) {
	case DMA_MEM_TO_DEV:
		flags = JZ4740_DMA_SRC_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
		chan->fifo_addr = config->dst_addr;
		break;
	case DMA_DEV_TO_MEM:
		flags = JZ4740_DMA_DST_AUTOINC;
		transfer_size = jz4740_dma_maxburst(config->src_maxburst);
		chan->fifo_addr = config->src_addr;
		break;
	default:
		return -EINVAL;
	}

	src_width = jz4740_dma_width(config->src_addr_width);
	dst_width = jz4740_dma_width(config->dst_addr_width);

	switch (transfer_size) {
	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
		chan->transfer_shift = 1;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
		chan->transfer_shift = 2;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
		chan->transfer_shift = 4;
		break;
	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
		chan->transfer_shift = 5;
		break;
	default:
		chan->transfer_shift = 0;
		break;
	}

	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	cmd |= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	cmd |= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	cmd |= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	cmd |= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	cmd |= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;

	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
		config->slave_id);

	return 0;
}

static int jz4740_dma_slave_config(struct dma_chan *c,
				   struct dma_slave_config *config)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);

	memcpy(&chan->config, config, sizeof(*config));
	return 0;
}

static int jz4740_dma_terminate_all(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&chan->vchan.lock, flags);
	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);
	chan->desc = NULL;
	vchan_get_all_descriptors(&chan->vchan, &head);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	vchan_dma_desc_free_list(&chan->vchan, &head);

	return 0;
}

static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	dma_addr_t src_addr, dst_addr;
	struct virt_dma_desc *vdesc;
	struct jz4740_dma_sg *sg;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
			JZ_DMA_STATUS_CTRL_ENABLE);

	if (!chan->desc) {
		vdesc = vchan_next_desc(&chan->vchan);
		if (!vdesc)
			return 0;
		chan->desc = to_jz4740_dma_desc(vdesc);
		chan->next_sg = 0;
	}

	if (chan->next_sg == chan->desc->num_sgs)
		chan->next_sg = 0;

	sg = &chan->desc->sg[chan->next_sg];

	if (chan->desc->direction == DMA_MEM_TO_DEV) {
		src_addr = sg->addr;
		dst_addr = chan->fifo_addr;
	} else {
		src_addr = chan->fifo_addr;
		dst_addr = sg->addr;
	}
	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
			sg->len >> chan->transfer_shift);

	chan->next_sg++;

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
			JZ_DMA_STATUS_CTRL_NO_DESC | JZ_DMA_STATUS_CTRL_ENABLE,
			JZ_DMA_STATUS_CTRL_HALT | JZ_DMA_STATUS_CTRL_NO_DESC |
			JZ_DMA_STATUS_CTRL_ENABLE);

	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
			JZ_DMA_CTRL_ENABLE,
			JZ_DMA_CTRL_HALT | JZ_DMA_CTRL_ENABLE);

	return 0;
}

static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
{
	spin_lock(&chan->vchan.lock);
	if (chan->desc) {
		if (chan->desc->cyclic) {
			vchan_cyclic_callback(&chan->desc->vdesc);
		} else {
			if (chan->next_sg == chan->desc->num_sgs) {
				list_del(&chan->desc->vdesc.node);
				vchan_cookie_complete(&chan->desc->vdesc);
				chan->desc = NULL;
			}
		}
	}
	jz4740_dma_start_transfer(chan);
	spin_unlock(&chan->vchan.lock);
}

static irqreturn_t jz4740_dma_irq(int irq, void *devid)
{
	struct jz4740_dma_dev *dmadev = devid;
	uint32_t irq_status;
	unsigned int i;

	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);

	for (i = 0; i < 6; ++i) {
		if (irq_status & (1 << i)) {
			jz4740_dma_write_mask(dmadev,
				JZ_REG_DMA_STATUS_CTRL(i), 0,
				JZ_DMA_STATUS_CTRL_ENABLE |
				JZ_DMA_STATUS_CTRL_TRANSFER_DONE);

			jz4740_dma_chan_irq(&dmadev->chan[i]);
		}
	}

	return IRQ_HANDLED;
}

static void jz4740_dma_issue_pending(struct dma_chan *c)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	unsigned long flags;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
		jz4740_dma_start_transfer(chan);
	spin_unlock_irqrestore(&chan->vchan.lock, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	struct dma_chan *c, struct scatterlist *sgl,
	unsigned int sg_len, enum dma_transfer_direction direction,
	unsigned long flags, void *context)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	struct scatterlist *sg;
	unsigned int i;

	desc = jz4740_dma_alloc_desc(sg_len);
	if (!desc)
		return NULL;

	for_each_sg(sgl, sg, sg_len, i) {
		desc->sg[i].addr = sg_dma_address(sg);
		desc->sg[i].len = sg_dma_len(sg);
	}

	desc->num_sgs = sg_len;
	desc->direction = direction;
	desc->cyclic = false;

	jz4740_dma_slave_config_write(c, &chan->config, direction);

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	size_t period_len, enum dma_transfer_direction direction,
	unsigned long flags)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct jz4740_dma_desc *desc;
	unsigned int num_periods, i;

	if (buf_len % period_len)
		return NULL;

	num_periods = buf_len / period_len;

	desc = jz4740_dma_alloc_desc(num_periods);
	if (!desc)
		return NULL;

	for (i = 0; i < num_periods; i++) {
		desc->sg[i].addr = buf_addr;
		desc->sg[i].len = period_len;
		buf_addr += period_len;
	}

	desc->num_sgs = num_periods;
	desc->direction = direction;
	desc->cyclic = true;

	jz4740_dma_slave_config_write(c, &chan->config, direction);

	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
}

static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	struct jz4740_dma_desc *desc, unsigned int next_sg)
{
	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	unsigned int residue, count;
	unsigned int i;

	residue = 0;

	for (i = next_sg; i < desc->num_sgs; i++)
		residue += desc->sg[i].len;

	if (next_sg != 0) {
		count = jz4740_dma_read(dmadev,
			JZ_REG_DMA_TRANSFER_COUNT(chan->id));
		residue += count << chan->transfer_shift;
	}

	return residue;
}

static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	dma_cookie_t cookie, struct dma_tx_state *state)
{
	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	struct virt_dma_desc *vdesc;
	enum dma_status status;
	unsigned long flags;

	status = dma_cookie_status(c, cookie, state);
	if (status == DMA_COMPLETE || !state)
		return status;

	spin_lock_irqsave(&chan->vchan.lock, flags);
	vdesc = vchan_find_desc(&chan->vchan, cookie);
	if (cookie == chan->desc->vdesc.tx.cookie) {
		state->residue = jz4740_dma_desc_residue(chan, chan->desc,
				chan->next_sg);
	} else if (vdesc) {
		state->residue = jz4740_dma_desc_residue(chan,
				to_jz4740_dma_desc(vdesc), 0);
	} else {
		state->residue = 0;
	}
	spin_unlock_irqrestore(&chan->vchan.lock, flags);

	return status;
}

static void jz4740_dma_free_chan_resources(struct dma_chan *c)
{
	vchan_free_chan_resources(to_virt_chan(c));
}

static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
{
	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
}

#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))

static int jz4740_dma_probe(struct platform_device *pdev)
{
	struct jz4740_dmaengine_chan *chan;
	struct jz4740_dma_dev *dmadev;
	struct dma_device *dd;
	unsigned int i;
	struct resource *res;
	int ret;
	int irq;

	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	if (!dmadev)
		return -EINVAL;

	dd = &dmadev->ddev;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	if (IS_ERR(dmadev->base))
		return PTR_ERR(dmadev->base);

	dmadev->clk = clk_get(&pdev->dev, "dma");
	if (IS_ERR(dmadev->clk))
		return PTR_ERR(dmadev->clk);

	clk_prepare_enable(dmadev->clk);

	dma_cap_set(DMA_SLAVE, dd->cap_mask);
	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	dd->device_tx_status = jz4740_dma_tx_status;
	dd->device_issue_pending = jz4740_dma_issue_pending;
	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
	dd->device_config = jz4740_dma_slave_config;
	dd->device_terminate_all = jz4740_dma_terminate_all;
	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
	dd->dev = &pdev->dev;
	INIT_LIST_HEAD(&dd->channels);

	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
		chan = &dmadev->chan[i];
		chan->id = i;
		chan->vchan.desc_free = jz4740_dma_desc_free;
		vchan_init(&chan->vchan, dd);
	}

	ret = dma_async_device_register(dd);
	if (ret)
		goto err_clk;

	irq = platform_get_irq(pdev, 0);
	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	if (ret)
		goto err_unregister;

	platform_set_drvdata(pdev, dmadev);

	return 0;

err_unregister:
	dma_async_device_unregister(dd);
err_clk:
	clk_disable_unprepare(dmadev->clk);
	return ret;
}

static void jz4740_cleanup_vchan(struct dma_device *dmadev)
{
	struct jz4740_dmaengine_chan *chan, *_chan;

	list_for_each_entry_safe(chan, _chan,
				&dmadev->channels, vchan.chan.device_node) {
		list_del(&chan->vchan.chan.device_node);
		tasklet_kill(&chan->vchan.task);
	}
}


static int jz4740_dma_remove(struct platform_device *pdev)
{
	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
	int irq = platform_get_irq(pdev, 0);

	free_irq(irq, dmadev);

	jz4740_cleanup_vchan(&dmadev->ddev);
	dma_async_device_unregister(&dmadev->ddev);
	clk_disable_unprepare(dmadev->clk);

	return 0;
}

static struct platform_driver jz4740_dma_driver = {
	.probe = jz4740_dma_probe,
	.remove = jz4740_dma_remove,
	.driver = {
		.name = "jz4740-dma",
	},
};
module_platform_driver(jz4740_dma_driver);

MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
MODULE_DESCRIPTION("JZ4740 DMA driver");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
7c169a42 LPC	1	/*
	2	* Copyright (C) 2013, Lars-Peter Clausen <lars@metafoo.de>
	3	* JZ4740 DMAC support
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License as published by the
	7	* Free Software Foundation; either version 2 of the License, or (at your
	8	* option) any later version.
	9	*
7c169a42 LPC	10	*/
	11
	12	#include <linux/dmaengine.h>
	13	#include <linux/dma-mapping.h>
	14	#include <linux/err.h>
	15	#include <linux/init.h>
	16	#include <linux/list.h>
	17	#include <linux/module.h>
	18	#include <linux/platform_device.h>
	19	#include <linux/slab.h>
	20	#include <linux/spinlock.h>
25ce6c35 LPC	21	#include <linux/irq.h>
25ce6c35 LPC	22	#include <linux/clk.h>
7c169a42	23
7c169a42 LPC	24	#include "virt-dma.h"
	25
	26	#define JZ_DMA_NR_CHANS 6
	27
25ce6c35 LPC	28	#define JZ_REG_DMA_SRC_ADDR(x) (0x00 + (x) * 0x20)
	29	#define JZ_REG_DMA_DST_ADDR(x) (0x04 + (x) * 0x20)
	30	#define JZ_REG_DMA_TRANSFER_COUNT(x) (0x08 + (x) * 0x20)
	31	#define JZ_REG_DMA_REQ_TYPE(x) (0x0C + (x) * 0x20)
	32	#define JZ_REG_DMA_STATUS_CTRL(x) (0x10 + (x) * 0x20)
	33	#define JZ_REG_DMA_CMD(x) (0x14 + (x) * 0x20)
	34	#define JZ_REG_DMA_DESC_ADDR(x) (0x18 + (x) * 0x20)
	35
	36	#define JZ_REG_DMA_CTRL 0x300
	37	#define JZ_REG_DMA_IRQ 0x304
	38	#define JZ_REG_DMA_DOORBELL 0x308
	39	#define JZ_REG_DMA_DOORBELL_SET 0x30C
	40
	41	#define JZ_DMA_STATUS_CTRL_NO_DESC BIT(31)
	42	#define JZ_DMA_STATUS_CTRL_DESC_INV BIT(6)
	43	#define JZ_DMA_STATUS_CTRL_ADDR_ERR BIT(4)
	44	#define JZ_DMA_STATUS_CTRL_TRANSFER_DONE BIT(3)
	45	#define JZ_DMA_STATUS_CTRL_HALT BIT(2)
	46	#define JZ_DMA_STATUS_CTRL_COUNT_TERMINATE BIT(1)
	47	#define JZ_DMA_STATUS_CTRL_ENABLE BIT(0)
	48
	49	#define JZ_DMA_CMD_SRC_INC BIT(23)
	50	#define JZ_DMA_CMD_DST_INC BIT(22)
	51	#define JZ_DMA_CMD_RDIL_MASK (0xf << 16)
	52	#define JZ_DMA_CMD_SRC_WIDTH_MASK (0x3 << 14)
	53	#define JZ_DMA_CMD_DST_WIDTH_MASK (0x3 << 12)
	54	#define JZ_DMA_CMD_INTERVAL_LENGTH_MASK (0x7 << 8)
	55	#define JZ_DMA_CMD_BLOCK_MODE BIT(7)
	56	#define JZ_DMA_CMD_DESC_VALID BIT(4)
	57	#define JZ_DMA_CMD_DESC_VALID_MODE BIT(3)
	58	#define JZ_DMA_CMD_VALID_IRQ_ENABLE BIT(2)
	59	#define JZ_DMA_CMD_TRANSFER_IRQ_ENABLE BIT(1)
	60	#define JZ_DMA_CMD_LINK_ENABLE BIT(0)
	61
	62	#define JZ_DMA_CMD_FLAGS_OFFSET 22
	63	#define JZ_DMA_CMD_RDIL_OFFSET 16
	64	#define JZ_DMA_CMD_SRC_WIDTH_OFFSET 14
	65	#define JZ_DMA_CMD_DST_WIDTH_OFFSET 12
	66	#define JZ_DMA_CMD_TRANSFER_SIZE_OFFSET 8
	67	#define JZ_DMA_CMD_MODE_OFFSET 7
	68
	69	#define JZ_DMA_CTRL_PRIORITY_MASK (0x3 << 8)
	70	#define JZ_DMA_CTRL_HALT BIT(3)
	71	#define JZ_DMA_CTRL_ADDRESS_ERROR BIT(2)
	72	#define JZ_DMA_CTRL_ENABLE BIT(0)
	73
	74	enum jz4740_dma_width {
	75	JZ4740_DMA_WIDTH_32BIT = 0,
	76	JZ4740_DMA_WIDTH_8BIT = 1,
	77	JZ4740_DMA_WIDTH_16BIT = 2,
	78	};
	79
	80	enum jz4740_dma_transfer_size {
	81	JZ4740_DMA_TRANSFER_SIZE_4BYTE = 0,
	82	JZ4740_DMA_TRANSFER_SIZE_1BYTE = 1,
	83	JZ4740_DMA_TRANSFER_SIZE_2BYTE = 2,
	84	JZ4740_DMA_TRANSFER_SIZE_16BYTE = 3,
	85	JZ4740_DMA_TRANSFER_SIZE_32BYTE = 4,
	86	};
	87
	88	enum jz4740_dma_flags {
	89	JZ4740_DMA_SRC_AUTOINC = 0x2,
	90	JZ4740_DMA_DST_AUTOINC = 0x1,
	91	};
92
93	enum jz4740_dma_mode {
94	JZ4740_DMA_MODE_SINGLE = 0,
95	JZ4740_DMA_MODE_BLOCK = 1,
96	};
97
7c169a42 LPC	98	struct jz4740_dma_sg {
	99	dma_addr_t addr;
	100	unsigned int len;
	101	};
	102
	103	struct jz4740_dma_desc {
	104	struct virt_dma_desc vdesc;
	105
	106	enum dma_transfer_direction direction;
	107	bool cyclic;
	108
	109	unsigned int num_sgs;
	110	struct jz4740_dma_sg sg[];
	111	};
	112
	113	struct jz4740_dmaengine_chan {
	114	struct virt_dma_chan vchan;
25ce6c35	115	unsigned int id;
09347e3f	116	struct dma_slave_config config;
7c169a42 LPC	117
7c169a42 LPC	118	dma_addr_t fifo_addr;
25ce6c35	119	unsigned int transfer_shift;
7c169a42 LPC	120
	121	struct jz4740_dma_desc *desc;
	122	unsigned int next_sg;
	123	};
	124
	125	struct jz4740_dma_dev {
	126	struct dma_device ddev;
25ce6c35 LPC	127	void __iomem *base;
25ce6c35 LPC	128	struct clk *clk;
7c169a42 LPC	129
	130	struct jz4740_dmaengine_chan chan[JZ_DMA_NR_CHANS];
	131	};
	132
25ce6c35 LPC	133	static struct jz4740_dma_dev *jz4740_dma_chan_get_dev(
	134	struct jz4740_dmaengine_chan *chan)
	135	{
	136	return container_of(chan->vchan.chan.device, struct jz4740_dma_dev,
	137	ddev);
	138	}
	139
7c169a42 LPC	140	static struct jz4740_dmaengine_chan to_jz4740_dma_chan(struct dma_chan c)
	141	{
	142	return container_of(c, struct jz4740_dmaengine_chan, vchan.chan);
	143	}
	144
	145	static struct jz4740_dma_desc to_jz4740_dma_desc(struct virt_dma_desc vdesc)
	146	{
	147	return container_of(vdesc, struct jz4740_dma_desc, vdesc);
	148	}
	149
25ce6c35 LPC	150	static inline uint32_t jz4740_dma_read(struct jz4740_dma_dev *dmadev,
	151	unsigned int reg)
	152	{
	153	return readl(dmadev->base + reg);
	154	}
	155
	156	static inline void jz4740_dma_write(struct jz4740_dma_dev *dmadev,
	157	unsigned reg, uint32_t val)
	158	{
	159	writel(val, dmadev->base + reg);
	160	}
	161
	162	static inline void jz4740_dma_write_mask(struct jz4740_dma_dev *dmadev,
	163	unsigned int reg, uint32_t val, uint32_t mask)
	164	{
	165	uint32_t tmp;
	166
	167	tmp = jz4740_dma_read(dmadev, reg);
	168	tmp &= ~mask;
	169	tmp \|= val;
	170	jz4740_dma_write(dmadev, reg, tmp);
	171	}
	172
7c169a42 LPC	173	static struct jz4740_dma_desc *jz4740_dma_alloc_desc(unsigned int num_sgs)
	174	{
	175	return kzalloc(sizeof(struct jz4740_dma_desc) +
	176	sizeof(struct jz4740_dma_sg) * num_sgs, GFP_ATOMIC);
	177	}
	178
	179	static enum jz4740_dma_width jz4740_dma_width(enum dma_slave_buswidth width)
	180	{
	181	switch (width) {
	182	case DMA_SLAVE_BUSWIDTH_1_BYTE:
	183	return JZ4740_DMA_WIDTH_8BIT;
	184	case DMA_SLAVE_BUSWIDTH_2_BYTES:
	185	return JZ4740_DMA_WIDTH_16BIT;
	186	case DMA_SLAVE_BUSWIDTH_4_BYTES:
	187	return JZ4740_DMA_WIDTH_32BIT;
	188	default:
	189	return JZ4740_DMA_WIDTH_32BIT;
	190	}
	191	}
	192
	193	static enum jz4740_dma_transfer_size jz4740_dma_maxburst(u32 maxburst)
	194	{
	195	if (maxburst <= 1)
	196	return JZ4740_DMA_TRANSFER_SIZE_1BYTE;
	197	else if (maxburst <= 3)
	198	return JZ4740_DMA_TRANSFER_SIZE_2BYTE;
	199	else if (maxburst <= 15)
	200	return JZ4740_DMA_TRANSFER_SIZE_4BYTE;
	201	else if (maxburst <= 31)
	202	return JZ4740_DMA_TRANSFER_SIZE_16BYTE;
	203
	204	return JZ4740_DMA_TRANSFER_SIZE_32BYTE;
	205	}
	206
09347e3f VK	207	static int jz4740_dma_slave_config_write(struct dma_chan *c,
	208	struct dma_slave_config *config,
	209	enum dma_transfer_direction direction)
7c169a42 LPC	210	{
7c169a42 LPC	211	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35 LPC	212	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
	213	enum jz4740_dma_width src_width;
	214	enum jz4740_dma_width dst_width;
	215	enum jz4740_dma_transfer_size transfer_size;
	216	enum jz4740_dma_flags flags;
	217	uint32_t cmd;
7c169a42	218
09347e3f	219	switch (direction) {
7c169a42	220	case DMA_MEM_TO_DEV:
25ce6c35 LPC	221	flags = JZ4740_DMA_SRC_AUTOINC;
25ce6c35 LPC	222	transfer_size = jz4740_dma_maxburst(config->dst_maxburst);
7c169a42 LPC	223	chan->fifo_addr = config->dst_addr;
	224	break;
	225	case DMA_DEV_TO_MEM:
25ce6c35 LPC	226	flags = JZ4740_DMA_DST_AUTOINC;
25ce6c35 LPC	227	transfer_size = jz4740_dma_maxburst(config->src_maxburst);
7c169a42 LPC	228	chan->fifo_addr = config->src_addr;
	229	break;
	230	default:
	231	return -EINVAL;
	232	}
	233
25ce6c35 LPC	234	src_width = jz4740_dma_width(config->src_addr_width);
	235	dst_width = jz4740_dma_width(config->dst_addr_width);
	236
	237	switch (transfer_size) {
	238	case JZ4740_DMA_TRANSFER_SIZE_2BYTE:
	239	chan->transfer_shift = 1;
	240	break;
	241	case JZ4740_DMA_TRANSFER_SIZE_4BYTE:
	242	chan->transfer_shift = 2;
	243	break;
	244	case JZ4740_DMA_TRANSFER_SIZE_16BYTE:
	245	chan->transfer_shift = 4;
	246	break;
	247	case JZ4740_DMA_TRANSFER_SIZE_32BYTE:
	248	chan->transfer_shift = 5;
	249	break;
	250	default:
	251	chan->transfer_shift = 0;
	252	break;
	253	}
7c169a42	254
25ce6c35 LPC	255	cmd = flags << JZ_DMA_CMD_FLAGS_OFFSET;
	256	cmd \|= src_width << JZ_DMA_CMD_SRC_WIDTH_OFFSET;
	257	cmd \|= dst_width << JZ_DMA_CMD_DST_WIDTH_OFFSET;
	258	cmd \|= transfer_size << JZ_DMA_CMD_TRANSFER_SIZE_OFFSET;
	259	cmd \|= JZ4740_DMA_MODE_SINGLE << JZ_DMA_CMD_MODE_OFFSET;
	260	cmd \|= JZ_DMA_CMD_TRANSFER_IRQ_ENABLE;
7c169a42	261
25ce6c35 LPC	262	jz4740_dma_write(dmadev, JZ_REG_DMA_CMD(chan->id), cmd);
	263	jz4740_dma_write(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0);
	264	jz4740_dma_write(dmadev, JZ_REG_DMA_REQ_TYPE(chan->id),
	265	config->slave_id);
7c169a42 LPC	266
	267	return 0;
	268	}
	269
09347e3f VK	270	static int jz4740_dma_slave_config(struct dma_chan *c,
	271	struct dma_slave_config *config)
	272	{
	273	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	274
	275	memcpy(&chan->config, config, sizeof(*config));
	276	return 0;
	277	}
	278
7c169a42 LPC	279	static int jz4740_dma_terminate_all(struct dma_chan *c)
	280	{
	281	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
25ce6c35	282	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	283	unsigned long flags;
	284	LIST_HEAD(head);
	285
	286	spin_lock_irqsave(&chan->vchan.lock, flags);
25ce6c35 LPC	287	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	288	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	289	chan->desc = NULL;
	290	vchan_get_all_descriptors(&chan->vchan, &head);
	291	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	292
	293	vchan_dma_desc_free_list(&chan->vchan, &head);
	294
	295	return 0;
	296	}
	297
7c169a42 LPC	298	static int jz4740_dma_start_transfer(struct jz4740_dmaengine_chan *chan)
7c169a42 LPC	299	{
25ce6c35	300	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
7c169a42 LPC	301	dma_addr_t src_addr, dst_addr;
	302	struct virt_dma_desc *vdesc;
	303	struct jz4740_dma_sg *sg;
	304
25ce6c35 LPC	305	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id), 0,
25ce6c35 LPC	306	JZ_DMA_STATUS_CTRL_ENABLE);
7c169a42 LPC	307
	308	if (!chan->desc) {
	309	vdesc = vchan_next_desc(&chan->vchan);
	310	if (!vdesc)
	311	return 0;
	312	chan->desc = to_jz4740_dma_desc(vdesc);
	313	chan->next_sg = 0;
	314	}
	315
	316	if (chan->next_sg == chan->desc->num_sgs)
	317	chan->next_sg = 0;
	318
	319	sg = &chan->desc->sg[chan->next_sg];
	320
	321	if (chan->desc->direction == DMA_MEM_TO_DEV) {
	322	src_addr = sg->addr;
	323	dst_addr = chan->fifo_addr;
	324	} else {
	325	src_addr = chan->fifo_addr;
	326	dst_addr = sg->addr;
	327	}
25ce6c35 LPC	328	jz4740_dma_write(dmadev, JZ_REG_DMA_SRC_ADDR(chan->id), src_addr);
	329	jz4740_dma_write(dmadev, JZ_REG_DMA_DST_ADDR(chan->id), dst_addr);
	330	jz4740_dma_write(dmadev, JZ_REG_DMA_TRANSFER_COUNT(chan->id),
	331	sg->len >> chan->transfer_shift);
7c169a42 LPC	332
	333	chan->next_sg++;
	334
25ce6c35 LPC	335	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_STATUS_CTRL(chan->id),
	336	JZ_DMA_STATUS_CTRL_NO_DESC \| JZ_DMA_STATUS_CTRL_ENABLE,
	337	JZ_DMA_STATUS_CTRL_HALT \| JZ_DMA_STATUS_CTRL_NO_DESC \|
	338	JZ_DMA_STATUS_CTRL_ENABLE);
	339
	340	jz4740_dma_write_mask(dmadev, JZ_REG_DMA_CTRL,
	341	JZ_DMA_CTRL_ENABLE,
	342	JZ_DMA_CTRL_HALT \| JZ_DMA_CTRL_ENABLE);
7c169a42 LPC	343
	344	return 0;
	345	}
	346
25ce6c35	347	static void jz4740_dma_chan_irq(struct jz4740_dmaengine_chan *chan)
7c169a42	348	{
7c169a42 LPC	349	spin_lock(&chan->vchan.lock);
7c169a42 LPC	350	if (chan->desc) {
26f7af37	351	if (chan->desc->cyclic) {
7c169a42 LPC	352	vchan_cyclic_callback(&chan->desc->vdesc);
	353	} else {
	354	if (chan->next_sg == chan->desc->num_sgs) {
f498e064	355	list_del(&chan->desc->vdesc.node);
7c169a42	356	vchan_cookie_complete(&chan->desc->vdesc);
f498e064	357	chan->desc = NULL;
7c169a42 LPC	358	}
	359	}
	360	}
	361	jz4740_dma_start_transfer(chan);
	362	spin_unlock(&chan->vchan.lock);
	363	}
	364
25ce6c35 LPC	365	static irqreturn_t jz4740_dma_irq(int irq, void *devid)
	366	{
	367	struct jz4740_dma_dev *dmadev = devid;
	368	uint32_t irq_status;
	369	unsigned int i;
	370
	371	irq_status = readl(dmadev->base + JZ_REG_DMA_IRQ);
	372
	373	for (i = 0; i < 6; ++i) {
	374	if (irq_status & (1 << i)) {
	375	jz4740_dma_write_mask(dmadev,
	376	JZ_REG_DMA_STATUS_CTRL(i), 0,
	377	JZ_DMA_STATUS_CTRL_ENABLE \|
	378	JZ_DMA_STATUS_CTRL_TRANSFER_DONE);
	379
	380	jz4740_dma_chan_irq(&dmadev->chan[i]);
	381	}
	382	}
	383
	384	return IRQ_HANDLED;
	385	}
	386
7c169a42 LPC	387	static void jz4740_dma_issue_pending(struct dma_chan *c)
	388	{
	389	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	390	unsigned long flags;
	391
	392	spin_lock_irqsave(&chan->vchan.lock, flags);
	393	if (vchan_issue_pending(&chan->vchan) && !chan->desc)
	394	jz4740_dma_start_transfer(chan);
	395	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	396	}
	397
	398	static struct dma_async_tx_descriptor *jz4740_dma_prep_slave_sg(
	399	struct dma_chan c, struct scatterlist sgl,
	400	unsigned int sg_len, enum dma_transfer_direction direction,
	401	unsigned long flags, void *context)
	402	{
	403	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	404	struct jz4740_dma_desc *desc;
	405	struct scatterlist *sg;
	406	unsigned int i;
	407
	408	desc = jz4740_dma_alloc_desc(sg_len);
	409	if (!desc)
	410	return NULL;
	411
	412	for_each_sg(sgl, sg, sg_len, i) {
	413	desc->sg[i].addr = sg_dma_address(sg);
	414	desc->sg[i].len = sg_dma_len(sg);
	415	}
	416
	417	desc->num_sgs = sg_len;
	418	desc->direction = direction;
	419	desc->cyclic = false;
	420
09347e3f VK	421	jz4740_dma_slave_config_write(c, &chan->config, direction);
09347e3f VK	422
7c169a42 LPC	423	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	424	}
	425
	426	static struct dma_async_tx_descriptor *jz4740_dma_prep_dma_cyclic(
	427	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
	428	size_t period_len, enum dma_transfer_direction direction,
31c1e5a1	429	unsigned long flags)
7c169a42 LPC	430	{
	431	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	432	struct jz4740_dma_desc *desc;
	433	unsigned int num_periods, i;
	434
	435	if (buf_len % period_len)
	436	return NULL;
	437
	438	num_periods = buf_len / period_len;
	439
	440	desc = jz4740_dma_alloc_desc(num_periods);
	441	if (!desc)
	442	return NULL;
	443
	444	for (i = 0; i < num_periods; i++) {
	445	desc->sg[i].addr = buf_addr;
	446	desc->sg[i].len = period_len;
	447	buf_addr += period_len;
	448	}
	449
	450	desc->num_sgs = num_periods;
	451	desc->direction = direction;
	452	desc->cyclic = true;
	453
09347e3f VK	454	jz4740_dma_slave_config_write(c, &chan->config, direction);
09347e3f VK	455
7c169a42 LPC	456	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
	457	}
	458
	459	static size_t jz4740_dma_desc_residue(struct jz4740_dmaengine_chan *chan,
	460	struct jz4740_dma_desc *desc, unsigned int next_sg)
	461	{
25ce6c35 LPC	462	struct jz4740_dma_dev *dmadev = jz4740_dma_chan_get_dev(chan);
25ce6c35 LPC	463	unsigned int residue, count;
7c169a42 LPC	464	unsigned int i;
	465
	466	residue = 0;
	467
	468	for (i = next_sg; i < desc->num_sgs; i++)
	469	residue += desc->sg[i].len;
	470
25ce6c35 LPC	471	if (next_sg != 0) {
	472	count = jz4740_dma_read(dmadev,
	473	JZ_REG_DMA_TRANSFER_COUNT(chan->id));
	474	residue += count << chan->transfer_shift;
	475	}
7c169a42 LPC	476
	477	return residue;
	478	}
	479
	480	static enum dma_status jz4740_dma_tx_status(struct dma_chan *c,
	481	dma_cookie_t cookie, struct dma_tx_state *state)
	482	{
	483	struct jz4740_dmaengine_chan *chan = to_jz4740_dma_chan(c);
	484	struct virt_dma_desc *vdesc;
	485	enum dma_status status;
	486	unsigned long flags;
	487
	488	status = dma_cookie_status(c, cookie, state);
a605c48b	489	if (status == DMA_COMPLETE \|\| !state)
7c169a42 LPC	490	return status;
	491
	492	spin_lock_irqsave(&chan->vchan.lock, flags);
	493	vdesc = vchan_find_desc(&chan->vchan, cookie);
	494	if (cookie == chan->desc->vdesc.tx.cookie) {
	495	state->residue = jz4740_dma_desc_residue(chan, chan->desc,
	496	chan->next_sg);
	497	} else if (vdesc) {
	498	state->residue = jz4740_dma_desc_residue(chan,
	499	to_jz4740_dma_desc(vdesc), 0);
	500	} else {
	501	state->residue = 0;
	502	}
	503	spin_unlock_irqrestore(&chan->vchan.lock, flags);
	504
	505	return status;
	506	}
	507
7c169a42 LPC	508	static void jz4740_dma_free_chan_resources(struct dma_chan *c)
7c169a42 LPC	509	{
25ce6c35	510	vchan_free_chan_resources(to_virt_chan(c));
7c169a42 LPC	511	}
	512
	513	static void jz4740_dma_desc_free(struct virt_dma_desc *vdesc)
	514	{
	515	kfree(container_of(vdesc, struct jz4740_dma_desc, vdesc));
	516	}
	517
ca766839 LPC	518	#define JZ4740_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) \| \
	519	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) \| BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
	520
7c169a42 LPC	521	static int jz4740_dma_probe(struct platform_device *pdev)
	522	{
	523	struct jz4740_dmaengine_chan *chan;
	524	struct jz4740_dma_dev *dmadev;
	525	struct dma_device *dd;
	526	unsigned int i;
25ce6c35	527	struct resource *res;
7c169a42	528	int ret;
25ce6c35	529	int irq;
7c169a42 LPC	530
	531	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
	532	if (!dmadev)
	533	return -EINVAL;
	534
	535	dd = &dmadev->ddev;
	536
25ce6c35 LPC	537	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	538	dmadev->base = devm_ioremap_resource(&pdev->dev, res);
	539	if (IS_ERR(dmadev->base))
	540	return PTR_ERR(dmadev->base);
	541
	542	dmadev->clk = clk_get(&pdev->dev, "dma");
	543	if (IS_ERR(dmadev->clk))
	544	return PTR_ERR(dmadev->clk);
	545
	546	clk_prepare_enable(dmadev->clk);
	547
7c169a42 LPC	548	dma_cap_set(DMA_SLAVE, dd->cap_mask);
7c169a42 LPC	549	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
7c169a42 LPC	550	dd->device_free_chan_resources = jz4740_dma_free_chan_resources;
	551	dd->device_tx_status = jz4740_dma_tx_status;
	552	dd->device_issue_pending = jz4740_dma_issue_pending;
	553	dd->device_prep_slave_sg = jz4740_dma_prep_slave_sg;
	554	dd->device_prep_dma_cyclic = jz4740_dma_prep_dma_cyclic;
1d4c0b8c MR	555	dd->device_config = jz4740_dma_slave_config;
1d4c0b8c MR	556	dd->device_terminate_all = jz4740_dma_terminate_all;
ca766839 LPC	557	dd->src_addr_widths = JZ4740_DMA_BUSWIDTHS;
	558	dd->dst_addr_widths = JZ4740_DMA_BUSWIDTHS;
	559	dd->directions = BIT(DMA_DEV_TO_MEM) \| BIT(DMA_MEM_TO_DEV);
	560	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
7c169a42	561	dd->dev = &pdev->dev;
7c169a42 LPC	562	INIT_LIST_HEAD(&dd->channels);
7c169a42 LPC	563
b2c100e0	564	for (i = 0; i < JZ_DMA_NR_CHANS; i++) {
7c169a42	565	chan = &dmadev->chan[i];
25ce6c35	566	chan->id = i;
7c169a42 LPC	567	chan->vchan.desc_free = jz4740_dma_desc_free;
	568	vchan_init(&chan->vchan, dd);
	569	}
	570
	571	ret = dma_async_device_register(dd);
	572	if (ret)
eb943696	573	goto err_clk;
7c169a42	574
25ce6c35 LPC	575	irq = platform_get_irq(pdev, 0);
	576	ret = request_irq(irq, jz4740_dma_irq, 0, dev_name(&pdev->dev), dmadev);
	577	if (ret)
	578	goto err_unregister;
	579
7c169a42 LPC	580	platform_set_drvdata(pdev, dmadev);
	581
	582	return 0;
25ce6c35 LPC	583
	584	err_unregister:
	585	dma_async_device_unregister(dd);
eb943696 TJ	586	err_clk:
eb943696 TJ	587	clk_disable_unprepare(dmadev->clk);
25ce6c35	588	return ret;
7c169a42 LPC	589	}
7c169a42 LPC	590
cec9cfa8 VK	591	static void jz4740_cleanup_vchan(struct dma_device *dmadev)
	592	{
	593	struct jz4740_dmaengine_chan chan, _chan;
	594
	595	list_for_each_entry_safe(chan, _chan,
	596	&dmadev->channels, vchan.chan.device_node) {
	597	list_del(&chan->vchan.chan.device_node);
	598	tasklet_kill(&chan->vchan.task);
	599	}
	600	}
	601
	602
7c169a42 LPC	603	static int jz4740_dma_remove(struct platform_device *pdev)
	604	{
	605	struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev);
25ce6c35	606	int irq = platform_get_irq(pdev, 0);
7c169a42	607
25ce6c35	608	free_irq(irq, dmadev);
cec9cfa8 VK	609
cec9cfa8 VK	610	jz4740_cleanup_vchan(&dmadev->ddev);
7c169a42	611	dma_async_device_unregister(&dmadev->ddev);
25ce6c35	612	clk_disable_unprepare(dmadev->clk);
7c169a42 LPC	613
	614	return 0;
	615	}
	616
	617	static struct platform_driver jz4740_dma_driver = {
	618	.probe = jz4740_dma_probe,
	619	.remove = jz4740_dma_remove,
	620	.driver = {
	621	.name = "jz4740-dma",
7c169a42 LPC	622	},
	623	};
	624	module_platform_driver(jz4740_dma_driver);
	625
	626	MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
	627	MODULE_DESCRIPTION("JZ4740 DMA driver");
e2f9922a	628	MODULE_LICENSE("GPL v2");