[linux-block.git] / drivers / dma / hisi_dma.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2019 HiSilicon Limited. */
#include <linux/bitfield.h>
#include <linux/dmaengine.h>
#include <linux/init.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include "virt-dma.h"

#define HISI_DMA_SQ_BASE_L		0x0
#define HISI_DMA_SQ_BASE_H		0x4
#define HISI_DMA_SQ_DEPTH		0x8
#define HISI_DMA_SQ_TAIL_PTR		0xc
#define HISI_DMA_CQ_BASE_L		0x10
#define HISI_DMA_CQ_BASE_H		0x14
#define HISI_DMA_CQ_DEPTH		0x18
#define HISI_DMA_CQ_HEAD_PTR		0x1c
#define HISI_DMA_CTRL0			0x20
#define HISI_DMA_CTRL0_QUEUE_EN_S	0
#define HISI_DMA_CTRL0_QUEUE_PAUSE_S	4
#define HISI_DMA_CTRL1			0x24
#define HISI_DMA_CTRL1_QUEUE_RESET_S	0
#define HISI_DMA_Q_FSM_STS		0x30
#define HISI_DMA_FSM_STS_MASK		GENMASK(3, 0)
#define HISI_DMA_INT_STS		0x40
#define HISI_DMA_INT_STS_MASK		GENMASK(12, 0)
#define HISI_DMA_INT_MSK		0x44
#define HISI_DMA_MODE			0x217c
#define HISI_DMA_OFFSET			0x100

#define HISI_DMA_MSI_NUM		30
#define HISI_DMA_CHAN_NUM		30
#define HISI_DMA_Q_DEPTH_VAL		1024

#define PCI_BAR_2			2

enum hisi_dma_mode {
	EP = 0,
	RC,
};

enum hisi_dma_chan_status {
	DISABLE = -1,
	IDLE = 0,
	RUN,
	CPL,
	PAUSE,
	HALT,
	ABORT,
	WAIT,
	BUFFCLR,
};

struct hisi_dma_sqe {
	__le32 dw0;
#define OPCODE_MASK			GENMASK(3, 0)
#define OPCODE_SMALL_PACKAGE		0x1
#define OPCODE_M2M			0x4
#define LOCAL_IRQ_EN			BIT(8)
#define ATTR_SRC_MASK			GENMASK(14, 12)
	__le32 dw1;
	__le32 dw2;
#define ATTR_DST_MASK			GENMASK(26, 24)
	__le32 length;
	__le64 src_addr;
	__le64 dst_addr;
};

struct hisi_dma_cqe {
	__le32 rsv0;
	__le32 rsv1;
	__le16 sq_head;
	__le16 rsv2;
	__le16 rsv3;
	__le16 w0;
#define STATUS_MASK			GENMASK(15, 1)
#define STATUS_SUCC			0x0
#define VALID_BIT			BIT(0)
};

struct hisi_dma_desc {
	struct virt_dma_desc vd;
	struct hisi_dma_sqe sqe;
};

struct hisi_dma_chan {
	struct virt_dma_chan vc;
	struct hisi_dma_dev *hdma_dev;
	struct hisi_dma_sqe *sq;
	struct hisi_dma_cqe *cq;
	dma_addr_t sq_dma;
	dma_addr_t cq_dma;
	u32 sq_tail;
	u32 cq_head;
	u32 qp_num;
	enum hisi_dma_chan_status status;
	struct hisi_dma_desc *desc;
};

struct hisi_dma_dev {
	struct pci_dev *pdev;
	void __iomem *base;
	struct dma_device dma_dev;
	u32 chan_num;
	u32 chan_depth;
	struct hisi_dma_chan chan[];
};

static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
{
	return container_of(c, struct hisi_dma_chan, vc.chan);
}

static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
{
	return container_of(vd, struct hisi_dma_desc, vd);
}

static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
				       u32 val)
{
	writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
}

static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
{
	u32 tmp;

	tmp = readl_relaxed(addr);
	tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
	writel_relaxed(tmp, addr);
}

static void hisi_dma_free_irq_vectors(void *data)
{
	pci_free_irq_vectors(data);
}

static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
			       bool pause)
{
	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
			     HISI_DMA_OFFSET;

	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
}

static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
				bool enable)
{
	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
			     HISI_DMA_OFFSET;

	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
}

static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
			    HISI_DMA_INT_STS_MASK);
}

static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
	void __iomem *base = hdma_dev->base;

	hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
			    HISI_DMA_INT_STS_MASK);
	hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
}

static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
{
	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
			     HISI_DMA_OFFSET;

	hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
}

static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
{
	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
}

static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
{
	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
	u32 index = chan->qp_num, tmp;
	int ret;

	hisi_dma_pause_dma(hdma_dev, index, true);
	hisi_dma_enable_dma(hdma_dev, index, false);
	hisi_dma_mask_irq(hdma_dev, index);

	ret = readl_relaxed_poll_timeout(hdma_dev->base +
		HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
		FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
	if (ret) {
		dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
		WARN_ON(1);
	}

	hisi_dma_do_reset(hdma_dev, index);
	hisi_dma_reset_qp_point(hdma_dev, index);
	hisi_dma_pause_dma(hdma_dev, index, false);
	hisi_dma_enable_dma(hdma_dev, index, true);
	hisi_dma_unmask_irq(hdma_dev, index);

	ret = readl_relaxed_poll_timeout(hdma_dev->base +
		HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
		FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
	if (ret) {
		dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
		WARN_ON(1);
	}
}

static void hisi_dma_free_chan_resources(struct dma_chan *c)
{
	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;

	hisi_dma_reset_hw_chan(chan);
	vchan_free_chan_resources(&chan->vc);

	memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
	memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
	chan->sq_tail = 0;
	chan->cq_head = 0;
	chan->status = DISABLE;
}

static void hisi_dma_desc_free(struct virt_dma_desc *vd)
{
	kfree(to_hisi_dma_desc(vd));
}

static struct dma_async_tx_descriptor *
hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
			 size_t len, unsigned long flags)
{
	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
	struct hisi_dma_desc *desc;

	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
	if (!desc)
		return NULL;

	desc->sqe.length = cpu_to_le32(len);
	desc->sqe.src_addr = cpu_to_le64(src);
	desc->sqe.dst_addr = cpu_to_le64(dst);

	return vchan_tx_prep(&chan->vc, &desc->vd, flags);
}

static enum dma_status
hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
		   struct dma_tx_state *txstate)
{
	return dma_cookie_status(c, cookie, txstate);
}

static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
{
	struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
	struct hisi_dma_desc *desc;
	struct virt_dma_desc *vd;

	vd = vchan_next_desc(&chan->vc);
	if (!vd) {
		dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
		chan->desc = NULL;
		return;
	}
	list_del(&vd->node);
	desc = to_hisi_dma_desc(vd);
	chan->desc = desc;

	memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));

	/* update other field in sqe */
	sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
	sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);

	/* make sure data has been updated in sqe */
	wmb();

	/* update sq tail, point to new sqe position */
	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;

	/* update sq_tail to trigger a new task */
	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
			    chan->sq_tail);
}

static void hisi_dma_issue_pending(struct dma_chan *c)
{
	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
	unsigned long flags;

	spin_lock_irqsave(&chan->vc.lock, flags);

	if (vchan_issue_pending(&chan->vc))
		hisi_dma_start_transfer(chan);

	spin_unlock_irqrestore(&chan->vc.lock, flags);
}

static int hisi_dma_terminate_all(struct dma_chan *c)
{
	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
	unsigned long flags;
	LIST_HEAD(head);

	spin_lock_irqsave(&chan->vc.lock, flags);

	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
	if (chan->desc) {
		vchan_terminate_vdesc(&chan->desc->vd);
		chan->desc = NULL;
	}

	vchan_get_all_descriptors(&chan->vc, &head);

	spin_unlock_irqrestore(&chan->vc.lock, flags);

	vchan_dma_desc_free_list(&chan->vc, &head);
	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);

	return 0;
}

static void hisi_dma_synchronize(struct dma_chan *c)
{
	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);

	vchan_synchronize(&chan->vc);
}

static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
{
	size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
	size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
	struct device *dev = &hdma_dev->pdev->dev;
	struct hisi_dma_chan *chan;
	int i;

	for (i = 0; i < hdma_dev->chan_num; i++) {
		chan = &hdma_dev->chan[i];
		chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
					       GFP_KERNEL);
		if (!chan->sq)
			return -ENOMEM;

		chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
					       GFP_KERNEL);
		if (!chan->cq)
			return -ENOMEM;
	}

	return 0;
}

static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
{
	struct hisi_dma_chan *chan = &hdma_dev->chan[index];
	u32 hw_depth = hdma_dev->chan_depth - 1;
	void __iomem *base = hdma_dev->base;

	/* set sq, cq base */
	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
			    lower_32_bits(chan->sq_dma));
	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
			    upper_32_bits(chan->sq_dma));
	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
			    lower_32_bits(chan->cq_dma));
	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
			    upper_32_bits(chan->cq_dma));

	/* set sq, cq depth */
	hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
	hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);

	/* init sq tail and cq head */
	hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
	hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
}

static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
	hisi_dma_init_hw_qp(hdma_dev, qp_index);
	hisi_dma_unmask_irq(hdma_dev, qp_index);
	hisi_dma_enable_dma(hdma_dev, qp_index, true);
}

static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
{
	hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
}

static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
{
	int i;

	for (i = 0; i < hdma_dev->chan_num; i++) {
		hdma_dev->chan[i].qp_num = i;
		hdma_dev->chan[i].hdma_dev = hdma_dev;
		hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
		vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
		hisi_dma_enable_qp(hdma_dev, i);
	}
}

static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
{
	int i;

	for (i = 0; i < hdma_dev->chan_num; i++) {
		hisi_dma_disable_qp(hdma_dev, i);
		tasklet_kill(&hdma_dev->chan[i].vc.task);
	}
}

static irqreturn_t hisi_dma_irq(int irq, void *data)
{
	struct hisi_dma_chan *chan = data;
	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
	struct hisi_dma_desc *desc;
	struct hisi_dma_cqe *cqe;

	spin_lock(&chan->vc.lock);

	desc = chan->desc;
	cqe = chan->cq + chan->cq_head;
	if (desc) {
		if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
			chan->cq_head = (chan->cq_head + 1) %
					hdma_dev->chan_depth;
			hisi_dma_chan_write(hdma_dev->base,
					    HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
					    chan->cq_head);
			vchan_cookie_complete(&desc->vd);
		} else {
			dev_err(&hdma_dev->pdev->dev, "task error!\n");
		}

		chan->desc = NULL;
	}

	spin_unlock(&chan->vc.lock);

	return IRQ_HANDLED;
}

static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
{
	struct pci_dev *pdev = hdma_dev->pdev;
	int i, ret;

	for (i = 0; i < hdma_dev->chan_num; i++) {
		ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
				       hisi_dma_irq, IRQF_SHARED, "hisi_dma",
				       &hdma_dev->chan[i]);
		if (ret)
			return ret;
	}

	return 0;
}

/* This function enables all hw channels in a device */
static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
{
	int ret;

	ret = hisi_dma_alloc_qps_mem(hdma_dev);
	if (ret) {
		dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
		return ret;
	}

	ret = hisi_dma_request_qps_irq(hdma_dev);
	if (ret) {
		dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
		return ret;
	}

	hisi_dma_enable_qps(hdma_dev);

	return 0;
}

static void hisi_dma_disable_hw_channels(void *data)
{
	hisi_dma_disable_qps(data);
}

static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
			      enum hisi_dma_mode mode)
{
	writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
}

static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct device *dev = &pdev->dev;
	struct hisi_dma_dev *hdma_dev;
	struct dma_device *dma_dev;
	int ret;

	ret = pcim_enable_device(pdev);
	if (ret) {
		dev_err(dev, "failed to enable device mem!\n");
		return ret;
	}

	ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
	if (ret) {
		dev_err(dev, "failed to remap I/O region!\n");
		return ret;
	}

	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
	if (ret)
		return ret;

	ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	if (ret)
		return ret;

	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
	if (!hdma_dev)
		return -EINVAL;

	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
	hdma_dev->pdev = pdev;
	hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;

	pci_set_drvdata(pdev, hdma_dev);
	pci_set_master(pdev);

	ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
				    PCI_IRQ_MSI);
	if (ret < 0) {
		dev_err(dev, "Failed to allocate MSI vectors!\n");
		return ret;
	}

	ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
	if (ret)
		return ret;

	dma_dev = &hdma_dev->dma_dev;
	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
	dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
	dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
	dma_dev->device_tx_status = hisi_dma_tx_status;
	dma_dev->device_issue_pending = hisi_dma_issue_pending;
	dma_dev->device_terminate_all = hisi_dma_terminate_all;
	dma_dev->device_synchronize = hisi_dma_synchronize;
	dma_dev->directions = BIT(DMA_MEM_TO_MEM);
	dma_dev->dev = dev;
	INIT_LIST_HEAD(&dma_dev->channels);

	hisi_dma_set_mode(hdma_dev, RC);

	ret = hisi_dma_enable_hw_channels(hdma_dev);
	if (ret < 0) {
		dev_err(dev, "failed to enable hw channel!\n");
		return ret;
	}

	ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
				       hdma_dev);
	if (ret)
		return ret;

	ret = dmaenginem_async_device_register(dma_dev);
	if (ret < 0)
		dev_err(dev, "failed to register device!\n");

	return ret;
}

static const struct pci_device_id hisi_dma_pci_tbl[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
	{ 0, }
};

static struct pci_driver hisi_dma_pci_driver = {
	.name		= "hisi_dma",
	.id_table	= hisi_dma_pci_tbl,
	.probe		= hisi_dma_probe,
};

module_pci_driver(hisi_dma_pci_driver);

MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
MODULE_LICENSE("GPL v2");
MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
Commit	Line	Data
e9f08b65 ZW	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2019 HiSilicon Limited. */
	3	#include <linux/bitfield.h>
	4	#include <linux/dmaengine.h>
	5	#include <linux/init.h>
	6	#include <linux/iopoll.h>
	7	#include <linux/module.h>
	8	#include <linux/pci.h>
	9	#include <linux/spinlock.h>
	10	#include "virt-dma.h"
	11
	12	#define HISI_DMA_SQ_BASE_L 0x0
	13	#define HISI_DMA_SQ_BASE_H 0x4
	14	#define HISI_DMA_SQ_DEPTH 0x8
	15	#define HISI_DMA_SQ_TAIL_PTR 0xc
	16	#define HISI_DMA_CQ_BASE_L 0x10
	17	#define HISI_DMA_CQ_BASE_H 0x14
	18	#define HISI_DMA_CQ_DEPTH 0x18
	19	#define HISI_DMA_CQ_HEAD_PTR 0x1c
	20	#define HISI_DMA_CTRL0 0x20
	21	#define HISI_DMA_CTRL0_QUEUE_EN_S 0
	22	#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
	23	#define HISI_DMA_CTRL1 0x24
	24	#define HISI_DMA_CTRL1_QUEUE_RESET_S 0
	25	#define HISI_DMA_Q_FSM_STS 0x30
	26	#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
	27	#define HISI_DMA_INT_STS 0x40
	28	#define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
	29	#define HISI_DMA_INT_MSK 0x44
	30	#define HISI_DMA_MODE 0x217c
	31	#define HISI_DMA_OFFSET 0x100
	32
	33	#define HISI_DMA_MSI_NUM 30
	34	#define HISI_DMA_CHAN_NUM 30
	35	#define HISI_DMA_Q_DEPTH_VAL 1024
	36
	37	#define PCI_BAR_2 2
	38
	39	enum hisi_dma_mode {
	40	EP = 0,
	41	RC,
	42	};
	43
	44	enum hisi_dma_chan_status {
	45	DISABLE = -1,
	46	IDLE = 0,
	47	RUN,
	48	CPL,
	49	PAUSE,
	50	HALT,
	51	ABORT,
	52	WAIT,
	53	BUFFCLR,
	54	};
	55
	56	struct hisi_dma_sqe {
	57	__le32 dw0;
	58	#define OPCODE_MASK GENMASK(3, 0)
	59	#define OPCODE_SMALL_PACKAGE 0x1
	60	#define OPCODE_M2M 0x4
	61	#define LOCAL_IRQ_EN BIT(8)
	62	#define ATTR_SRC_MASK GENMASK(14, 12)
	63	__le32 dw1;
	64	__le32 dw2;
65	#define ATTR_DST_MASK GENMASK(26, 24)
66	__le32 length;
67	__le64 src_addr;
68	__le64 dst_addr;
69	};
70
71	struct hisi_dma_cqe {
72	__le32 rsv0;
73	__le32 rsv1;
74	__le16 sq_head;
75	__le16 rsv2;
76	__le16 rsv3;
77	__le16 w0;
78	#define STATUS_MASK GENMASK(15, 1)
79	#define STATUS_SUCC 0x0
80	#define VALID_BIT BIT(0)
81	};
82
83	struct hisi_dma_desc {
84	struct virt_dma_desc vd;
85	struct hisi_dma_sqe sqe;
86	};
87
88	struct hisi_dma_chan {
89	struct virt_dma_chan vc;
90	struct hisi_dma_dev *hdma_dev;
91	struct hisi_dma_sqe *sq;
92	struct hisi_dma_cqe *cq;
93	dma_addr_t sq_dma;
94	dma_addr_t cq_dma;
95	u32 sq_tail;
96	u32 cq_head;
97	u32 qp_num;
98	enum hisi_dma_chan_status status;
99	struct hisi_dma_desc *desc;
100	};
101
102	struct hisi_dma_dev {
103	struct pci_dev *pdev;
104	void __iomem *base;
105	struct dma_device dma_dev;
106	u32 chan_num;
107	u32 chan_depth;
108	struct hisi_dma_chan chan[];
109	};
110
111	static inline struct hisi_dma_chan to_hisi_dma_chan(struct dma_chan c)
112	{
113	return container_of(c, struct hisi_dma_chan, vc.chan);
114	}
115
116	static inline struct hisi_dma_desc to_hisi_dma_desc(struct virt_dma_desc vd)
117	{
118	return container_of(vd, struct hisi_dma_desc, vd);
119	}
120
121	static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
122	u32 val)
123	{
124	writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
125	}
126
127	static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
128	{
129	u32 tmp;
130
131	tmp = readl_relaxed(addr);
132	tmp = val ? tmp \| BIT(pos) : tmp & ~BIT(pos);
133	writel_relaxed(tmp, addr);
134	}
135
136	static void hisi_dma_free_irq_vectors(void *data)
137	{
138	pci_free_irq_vectors(data);
139	}
140
141	static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
142	bool pause)
143	{
144	void __iomem addr = hdma_dev->base + HISI_DMA_CTRL0 + index
145	HISI_DMA_OFFSET;
146
147	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
148	}
149
150	static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
151	bool enable)
152	{
153	void __iomem addr = hdma_dev->base + HISI_DMA_CTRL0 + index
154	HISI_DMA_OFFSET;
155
156	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
157	}
158
159	static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
160	{
161	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
162	HISI_DMA_INT_STS_MASK);
163	}
164
165	static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
166	{
167	void __iomem *base = hdma_dev->base;
168
169	hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
170	HISI_DMA_INT_STS_MASK);
171	hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
172	}
173
174	static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
175	{
176	void __iomem addr = hdma_dev->base + HISI_DMA_CTRL1 + index
177	HISI_DMA_OFFSET;
178
179	hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
180	}
181
182	static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
183	{
184	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
185	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
186	}
187
188	static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
189	{
190	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
191	u32 index = chan->qp_num, tmp;
192	int ret;
193
194	hisi_dma_pause_dma(hdma_dev, index, true);
195	hisi_dma_enable_dma(hdma_dev, index, false);
196	hisi_dma_mask_irq(hdma_dev, index);
197
198	ret = readl_relaxed_poll_timeout(hdma_dev->base +
199	HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
200	FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
201	if (ret) {
202	dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
203	WARN_ON(1);
204	}
205
206	hisi_dma_do_reset(hdma_dev, index);
207	hisi_dma_reset_qp_point(hdma_dev, index);
208	hisi_dma_pause_dma(hdma_dev, index, false);
209	hisi_dma_enable_dma(hdma_dev, index, true);
210	hisi_dma_unmask_irq(hdma_dev, index);
211
212	ret = readl_relaxed_poll_timeout(hdma_dev->base +
213	HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
214	FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
215	if (ret) {
216	dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
217	WARN_ON(1);
218	}
219	}
220
221	static void hisi_dma_free_chan_resources(struct dma_chan *c)
222	{
223	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
224	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
225
226	hisi_dma_reset_hw_chan(chan);
227	vchan_free_chan_resources(&chan->vc);
228
229	memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
230	memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
231	chan->sq_tail = 0;
232	chan->cq_head = 0;
233	chan->status = DISABLE;
234	}
235
236	static void hisi_dma_desc_free(struct virt_dma_desc *vd)
237	{
238	kfree(to_hisi_dma_desc(vd));
239	}
240
241	static struct dma_async_tx_descriptor *
242	hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
243	size_t len, unsigned long flags)
244	{
245	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
246	struct hisi_dma_desc *desc;
247
248	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
249	if (!desc)
250	return NULL;
251
252	desc->sqe.length = cpu_to_le32(len);
253	desc->sqe.src_addr = cpu_to_le64(src);
254	desc->sqe.dst_addr = cpu_to_le64(dst);
255
256	return vchan_tx_prep(&chan->vc, &desc->vd, flags);
257	}
258
259	static enum dma_status
260	hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
261	struct dma_tx_state *txstate)
262	{
263	return dma_cookie_status(c, cookie, txstate);
264	}
265
266	static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
267	{
268	struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
269	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
270	struct hisi_dma_desc *desc;
271	struct virt_dma_desc *vd;
272
273	vd = vchan_next_desc(&chan->vc);
274	if (!vd) {
275	dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
276	chan->desc = NULL;
277	return;
278	}
279	list_del(&vd->node);
280	desc = to_hisi_dma_desc(vd);
281	chan->desc = desc;
282
283	memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
284
285	/* update other field in sqe */
286	sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
287	sqe->dw0 \|= cpu_to_le32(LOCAL_IRQ_EN);
288
289	/* make sure data has been updated in sqe */
290	wmb();
291
292	/* update sq tail, point to new sqe position */
293	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
294
295	/* update sq_tail to trigger a new task */
296	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
297	chan->sq_tail);
298	}
299
300	static void hisi_dma_issue_pending(struct dma_chan *c)
301	{
302	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
303	unsigned long flags;
304
305	spin_lock_irqsave(&chan->vc.lock, flags);
306
307	if (vchan_issue_pending(&chan->vc))
308	hisi_dma_start_transfer(chan);
309
310	spin_unlock_irqrestore(&chan->vc.lock, flags);
311	}
312
313	static int hisi_dma_terminate_all(struct dma_chan *c)
314	{
315	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
316	unsigned long flags;
317	LIST_HEAD(head);
318
319	spin_lock_irqsave(&chan->vc.lock, flags);
320
321	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
322	if (chan->desc) {
323	vchan_terminate_vdesc(&chan->desc->vd);
324	chan->desc = NULL;
325	}
326
327	vchan_get_all_descriptors(&chan->vc, &head);
328
329	spin_unlock_irqrestore(&chan->vc.lock, flags);
330
331	vchan_dma_desc_free_list(&chan->vc, &head);
332	hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
333
334	return 0;
335	}
336
337	static void hisi_dma_synchronize(struct dma_chan *c)
338	{
339	struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
340
341	vchan_synchronize(&chan->vc);
342	}
343
344	static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
345	{
346	size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
347	size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
348	struct device *dev = &hdma_dev->pdev->dev;
349	struct hisi_dma_chan *chan;
350	int i;
351
352	for (i = 0; i < hdma_dev->chan_num; i++) {
353	chan = &hdma_dev->chan[i];
354	chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
355	GFP_KERNEL);
356	if (!chan->sq)
357	return -ENOMEM;
358
359	chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
360	GFP_KERNEL);
361	if (!chan->cq)
362	return -ENOMEM;
363	}
364
365	return 0;
366	}
367
368	static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
369	{
370	struct hisi_dma_chan *chan = &hdma_dev->chan[index];
371	u32 hw_depth = hdma_dev->chan_depth - 1;
372	void __iomem *base = hdma_dev->base;
373
374	/* set sq, cq base */
375	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
376	lower_32_bits(chan->sq_dma));
377	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
378	upper_32_bits(chan->sq_dma));
379	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
380	lower_32_bits(chan->cq_dma));
381	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
382	upper_32_bits(chan->cq_dma));
383
384	/* set sq, cq depth */
385	hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
386	hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
387
388	/* init sq tail and cq head */
389	hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
390	hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
391	}
392
393	static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
394	{
395	hisi_dma_init_hw_qp(hdma_dev, qp_index);
396	hisi_dma_unmask_irq(hdma_dev, qp_index);
397	hisi_dma_enable_dma(hdma_dev, qp_index, true);
398	}
399
400	static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
401	{
402	hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
403	}
404
405	static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
406	{
407	int i;
408
409	for (i = 0; i < hdma_dev->chan_num; i++) {
410	hdma_dev->chan[i].qp_num = i;
411	hdma_dev->chan[i].hdma_dev = hdma_dev;
412	hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
413	vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
414	hisi_dma_enable_qp(hdma_dev, i);
415	}
416	}
417
418	static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
419	{
420	int i;
421
422	for (i = 0; i < hdma_dev->chan_num; i++) {
423	hisi_dma_disable_qp(hdma_dev, i);
424	tasklet_kill(&hdma_dev->chan[i].vc.task);
425	}
426	}
427
428	static irqreturn_t hisi_dma_irq(int irq, void *data)
429	{
430	struct hisi_dma_chan *chan = data;
431	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
432	struct hisi_dma_desc *desc;
433	struct hisi_dma_cqe *cqe;
e9f08b65	434
d9c8d4b2	435	spin_lock(&chan->vc.lock);
e9f08b65 ZW	436
	437	desc = chan->desc;
	438	cqe = chan->cq + chan->cq_head;
	439	if (desc) {
	440	if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
	441	chan->cq_head = (chan->cq_head + 1) %
	442	hdma_dev->chan_depth;
	443	hisi_dma_chan_write(hdma_dev->base,
	444	HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
	445	chan->cq_head);
	446	vchan_cookie_complete(&desc->vd);
	447	} else {
	448	dev_err(&hdma_dev->pdev->dev, "task error!\n");
	449	}
	450
	451	chan->desc = NULL;
	452	}
	453
d9c8d4b2	454	spin_unlock(&chan->vc.lock);
e9f08b65 ZW	455
	456	return IRQ_HANDLED;
	457	}
	458
	459	static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
	460	{
	461	struct pci_dev *pdev = hdma_dev->pdev;
	462	int i, ret;
	463
	464	for (i = 0; i < hdma_dev->chan_num; i++) {
	465	ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
	466	hisi_dma_irq, IRQF_SHARED, "hisi_dma",
	467	&hdma_dev->chan[i]);
	468	if (ret)
	469	return ret;
	470	}
	471
	472	return 0;
	473	}
	474
	475	/* This function enables all hw channels in a device */
	476	static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
	477	{
	478	int ret;
	479
	480	ret = hisi_dma_alloc_qps_mem(hdma_dev);
	481	if (ret) {
	482	dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
	483	return ret;
	484	}
	485
	486	ret = hisi_dma_request_qps_irq(hdma_dev);
	487	if (ret) {
	488	dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
	489	return ret;
	490	}
	491
	492	hisi_dma_enable_qps(hdma_dev);
	493
	494	return 0;
	495	}
	496
	497	static void hisi_dma_disable_hw_channels(void *data)
	498	{
	499	hisi_dma_disable_qps(data);
	500	}
	501
	502	static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
	503	enum hisi_dma_mode mode)
	504	{
	505	writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
	506	}
	507
	508	static int hisi_dma_probe(struct pci_dev pdev, const struct pci_device_id id)
	509	{
	510	struct device *dev = &pdev->dev;
	511	struct hisi_dma_dev *hdma_dev;
	512	struct dma_device *dma_dev;
e9f08b65 ZW	513	int ret;
	514
	515	ret = pcim_enable_device(pdev);
	516	if (ret) {
	517	dev_err(dev, "failed to enable device mem!\n");
	518	return ret;
	519	}
	520
	521	ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
	522	if (ret) {
	523	dev_err(dev, "failed to remap I/O region!\n");
	524	return ret;
	525	}
	526
	527	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
	528	if (ret)
	529	return ret;
	530
	531	ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
	532	if (ret)
	533	return ret;
	534
999a32ef	535	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
e9f08b65 ZW	536	if (!hdma_dev)
	537	return -EINVAL;
	538
	539	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
	540	hdma_dev->pdev = pdev;
	541	hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
	542	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
	543
	544	pci_set_drvdata(pdev, hdma_dev);
	545	pci_set_master(pdev);
	546
	547	ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
	548	PCI_IRQ_MSI);
	549	if (ret < 0) {
	550	dev_err(dev, "Failed to allocate MSI vectors!\n");
	551	return ret;
	552	}
	553
	554	ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
	555	if (ret)
	556	return ret;
	557
	558	dma_dev = &hdma_dev->dma_dev;
	559	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
	560	dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
	561	dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
	562	dma_dev->device_tx_status = hisi_dma_tx_status;
	563	dma_dev->device_issue_pending = hisi_dma_issue_pending;
	564	dma_dev->device_terminate_all = hisi_dma_terminate_all;
	565	dma_dev->device_synchronize = hisi_dma_synchronize;
	566	dma_dev->directions = BIT(DMA_MEM_TO_MEM);
	567	dma_dev->dev = dev;
	568	INIT_LIST_HEAD(&dma_dev->channels);
	569
	570	hisi_dma_set_mode(hdma_dev, RC);
	571
	572	ret = hisi_dma_enable_hw_channels(hdma_dev);
	573	if (ret < 0) {
	574	dev_err(dev, "failed to enable hw channel!\n");
	575	return ret;
	576	}
	577
	578	ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
	579	hdma_dev);
	580	if (ret)
	581	return ret;
	582
	583	ret = dmaenginem_async_device_register(dma_dev);
	584	if (ret < 0)
	585	dev_err(dev, "failed to register device!\n");
	586
	587	return ret;
	588	}
	589
	590	static const struct pci_device_id hisi_dma_pci_tbl[] = {
	591	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
	592	{ 0, }
	593	};
	594
	595	static struct pci_driver hisi_dma_pci_driver = {
	596	.name = "hisi_dma",
	597	.id_table = hisi_dma_pci_tbl,
	598	.probe = hisi_dma_probe,
	599	};
600
601	module_pci_driver(hisi_dma_pci_driver);
602
603	MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
604	MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
605	MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
606	MODULE_LICENSE("GPL v2");
607	MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);