[linux-2.6-block.git] / drivers / spi / spi-qcom-qspi.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2017-2018, The Linux foundation. All rights reserved.

#include <linux/clk.h>
#include <linux/interconnect.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/pm_runtime.h>
#include <linux/pm_opp.h>
#include <linux/spi/spi.h>
#include <linux/spi/spi-mem.h>


#define QSPI_NUM_CS		2
#define QSPI_BYTES_PER_WORD	4

#define MSTR_CONFIG		0x0000
#define FULL_CYCLE_MODE		BIT(3)
#define FB_CLK_EN		BIT(4)
#define PIN_HOLDN		BIT(6)
#define PIN_WPN			BIT(7)
#define DMA_ENABLE		BIT(8)
#define BIG_ENDIAN_MODE		BIT(9)
#define SPI_MODE_MSK		0xc00
#define SPI_MODE_SHFT		10
#define CHIP_SELECT_NUM		BIT(12)
#define SBL_EN			BIT(13)
#define LPA_BASE_MSK		0x3c000
#define LPA_BASE_SHFT		14
#define TX_DATA_DELAY_MSK	0xc0000
#define TX_DATA_DELAY_SHFT	18
#define TX_CLK_DELAY_MSK	0x300000
#define TX_CLK_DELAY_SHFT	20
#define TX_CS_N_DELAY_MSK	0xc00000
#define TX_CS_N_DELAY_SHFT	22
#define TX_DATA_OE_DELAY_MSK	0x3000000
#define TX_DATA_OE_DELAY_SHFT	24

#define AHB_MASTER_CFG				0x0004
#define HMEM_TYPE_START_MID_TRANS_MSK		0x7
#define HMEM_TYPE_START_MID_TRANS_SHFT		0
#define HMEM_TYPE_LAST_TRANS_MSK		0x38
#define HMEM_TYPE_LAST_TRANS_SHFT		3
#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK	0xc0
#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT	6
#define HMEMTYPE_READ_TRANS_MSK			0x700
#define HMEMTYPE_READ_TRANS_SHFT		8
#define HSHARED					BIT(11)
#define HINNERSHARED				BIT(12)

#define MSTR_INT_EN		0x000C
#define MSTR_INT_STATUS		0x0010
#define RESP_FIFO_UNDERRUN	BIT(0)
#define RESP_FIFO_NOT_EMPTY	BIT(1)
#define RESP_FIFO_RDY		BIT(2)
#define HRESP_FROM_NOC_ERR	BIT(3)
#define WR_FIFO_EMPTY		BIT(9)
#define WR_FIFO_FULL		BIT(10)
#define WR_FIFO_OVERRUN		BIT(11)
#define TRANSACTION_DONE	BIT(16)
#define QSPI_ERR_IRQS		(RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
				 WR_FIFO_OVERRUN)
#define QSPI_ALL_IRQS		(QSPI_ERR_IRQS | RESP_FIFO_RDY | \
				 WR_FIFO_EMPTY | WR_FIFO_FULL | \
				 TRANSACTION_DONE)

#define PIO_XFER_CTRL		0x0014
#define REQUEST_COUNT_MSK	0xffff

#define PIO_XFER_CFG		0x0018
#define TRANSFER_DIRECTION	BIT(0)
#define MULTI_IO_MODE_MSK	0xe
#define MULTI_IO_MODE_SHFT	1
#define TRANSFER_FRAGMENT	BIT(8)
#define SDR_1BIT		1
#define SDR_2BIT		2
#define SDR_4BIT		3
#define DDR_1BIT		5
#define DDR_2BIT		6
#define DDR_4BIT		7
#define DMA_DESC_SINGLE_SPI	1
#define DMA_DESC_DUAL_SPI	2
#define DMA_DESC_QUAD_SPI	3

#define PIO_XFER_STATUS		0x001c
#define WR_FIFO_BYTES_MSK	0xffff0000
#define WR_FIFO_BYTES_SHFT	16

#define PIO_DATAOUT_1B		0x0020
#define PIO_DATAOUT_4B		0x0024

#define RD_FIFO_CFG		0x0028
#define CONTINUOUS_MODE		BIT(0)

#define RD_FIFO_STATUS	0x002c
#define FIFO_EMPTY	BIT(11)
#define WR_CNTS_MSK	0x7f0
#define WR_CNTS_SHFT	4
#define RDY_64BYTE	BIT(3)
#define RDY_32BYTE	BIT(2)
#define RDY_16BYTE	BIT(1)
#define FIFO_RDY	BIT(0)

#define RD_FIFO_RESET		0x0030
#define RESET_FIFO		BIT(0)

#define CUR_MEM_ADDR		0x0048
#define HW_VERSION		0x004c
#define RD_FIFO			0x0050
#define SAMPLING_CLK_CFG	0x0090
#define SAMPLING_CLK_STATUS	0x0094


enum qspi_dir {
	QSPI_READ,
	QSPI_WRITE,
};

struct qspi_xfer {
	union {
		const void *tx_buf;
		void *rx_buf;
	};
	unsigned int rem_bytes;
	unsigned int buswidth;
	enum qspi_dir dir;
	bool is_last;
};

enum qspi_clocks {
	QSPI_CLK_CORE,
	QSPI_CLK_IFACE,
	QSPI_NUM_CLKS
};

struct qcom_qspi {
	void __iomem *base;
	struct device *dev;
	struct clk_bulk_data *clks;
	struct qspi_xfer xfer;
	struct icc_path *icc_path_cpu_to_qspi;
	struct opp_table *opp_table;
	unsigned long last_speed;
	/* Lock to protect data accessed by IRQs */
	spinlock_t lock;
};

static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
				   unsigned int buswidth)
{
	switch (buswidth) {
	case 1:
		return SDR_1BIT << MULTI_IO_MODE_SHFT;
	case 2:
		return SDR_2BIT << MULTI_IO_MODE_SHFT;
	case 4:
		return SDR_4BIT << MULTI_IO_MODE_SHFT;
	default:
		dev_warn_once(ctrl->dev,
				"Unexpected bus width: %u\n", buswidth);
		return SDR_1BIT << MULTI_IO_MODE_SHFT;
	}
}

static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
{
	u32 pio_xfer_cfg;
	const struct qspi_xfer *xfer;

	xfer = &ctrl->xfer;
	pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
	pio_xfer_cfg &= ~TRANSFER_DIRECTION;
	pio_xfer_cfg |= xfer->dir;
	if (xfer->is_last)
		pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
	else
		pio_xfer_cfg |= TRANSFER_FRAGMENT;
	pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
	pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);

	writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
}

static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
{
	u32 pio_xfer_ctrl;

	pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
	pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
	pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
	writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
}

static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
{
	u32 ints;

	qcom_qspi_pio_xfer_cfg(ctrl);

	/* Ack any previous interrupts that might be hanging around */
	writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);

	/* Setup new interrupts */
	if (ctrl->xfer.dir == QSPI_WRITE)
		ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
	else
		ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
	writel(ints, ctrl->base + MSTR_INT_EN);

	/* Kick off the transfer */
	qcom_qspi_pio_xfer_ctrl(ctrl);
}

static void qcom_qspi_handle_err(struct spi_master *master,
				 struct spi_message *msg)
{
	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
	unsigned long flags;

	spin_lock_irqsave(&ctrl->lock, flags);
	writel(0, ctrl->base + MSTR_INT_EN);
	ctrl->xfer.rem_bytes = 0;
	spin_unlock_irqrestore(&ctrl->lock, flags);
}

static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
{
	int ret;
	unsigned int avg_bw_cpu;

	if (speed_hz == ctrl->last_speed)
		return 0;

	/* In regular operation (SBL_EN=1) core must be 4x transfer clock */
	ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
	if (ret) {
		dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
		return ret;
	}

	/*
	 * Set BW quota for CPU as driver supports FIFO mode only.
	 * We don't have explicit peak requirement so keep it equal to avg_bw.
	 */
	avg_bw_cpu = Bps_to_icc(speed_hz);
	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
	if (ret) {
		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
			__func__, ret);
		return ret;
	}

	ctrl->last_speed = speed_hz;

	return 0;
}

static int qcom_qspi_transfer_one(struct spi_master *master,
				  struct spi_device *slv,
				  struct spi_transfer *xfer)
{
	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
	int ret;
	unsigned long speed_hz;
	unsigned long flags;

	speed_hz = slv->max_speed_hz;
	if (xfer->speed_hz)
		speed_hz = xfer->speed_hz;

	ret = qcom_qspi_set_speed(ctrl, speed_hz);
	if (ret)
		return ret;

	spin_lock_irqsave(&ctrl->lock, flags);

	/* We are half duplex, so either rx or tx will be set */
	if (xfer->rx_buf) {
		ctrl->xfer.dir = QSPI_READ;
		ctrl->xfer.buswidth = xfer->rx_nbits;
		ctrl->xfer.rx_buf = xfer->rx_buf;
	} else {
		ctrl->xfer.dir = QSPI_WRITE;
		ctrl->xfer.buswidth = xfer->tx_nbits;
		ctrl->xfer.tx_buf = xfer->tx_buf;
	}
	ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
					  &master->cur_msg->transfers);
	ctrl->xfer.rem_bytes = xfer->len;
	qcom_qspi_pio_xfer(ctrl);

	spin_unlock_irqrestore(&ctrl->lock, flags);

	/* We'll call spi_finalize_current_transfer() when done */
	return 1;
}

static int qcom_qspi_prepare_message(struct spi_master *master,
				     struct spi_message *message)
{
	u32 mstr_cfg;
	struct qcom_qspi *ctrl;
	int tx_data_oe_delay = 1;
	int tx_data_delay = 1;
	unsigned long flags;

	ctrl = spi_master_get_devdata(master);
	spin_lock_irqsave(&ctrl->lock, flags);

	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
	mstr_cfg &= ~CHIP_SELECT_NUM;
	if (message->spi->chip_select)
		mstr_cfg |= CHIP_SELECT_NUM;

	mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
	mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
	mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
	mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
	mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
	mstr_cfg &= ~DMA_ENABLE;

	writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
	spin_unlock_irqrestore(&ctrl->lock, flags);

	return 0;
}

static irqreturn_t pio_read(struct qcom_qspi *ctrl)
{
	u32 rd_fifo_status;
	u32 rd_fifo;
	unsigned int wr_cnts;
	unsigned int bytes_to_read;
	unsigned int words_to_read;
	u32 *word_buf;
	u8 *byte_buf;
	int i;

	rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);

	if (!(rd_fifo_status & FIFO_RDY)) {
		dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
		return IRQ_NONE;
	}

	wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
	wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);

	words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
	bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;

	if (words_to_read) {
		word_buf = ctrl->xfer.rx_buf;
		ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
		ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
		ctrl->xfer.rx_buf = word_buf + words_to_read;
	}

	if (bytes_to_read) {
		byte_buf = ctrl->xfer.rx_buf;
		rd_fifo = readl(ctrl->base + RD_FIFO);
		ctrl->xfer.rem_bytes -= bytes_to_read;
		for (i = 0; i < bytes_to_read; i++)
			*byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
		ctrl->xfer.rx_buf = byte_buf;
	}

	return IRQ_HANDLED;
}

static irqreturn_t pio_write(struct qcom_qspi *ctrl)
{
	const void *xfer_buf = ctrl->xfer.tx_buf;
	const int *word_buf;
	const char *byte_buf;
	unsigned int wr_fifo_bytes;
	unsigned int wr_fifo_words;
	unsigned int wr_size;
	unsigned int rem_words;

	wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
	wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;

	if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
		/* Process the last 1-3 bytes */
		wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
		ctrl->xfer.rem_bytes -= wr_size;

		byte_buf = xfer_buf;
		while (wr_size--)
			writel(*byte_buf++,
			       ctrl->base + PIO_DATAOUT_1B);
		ctrl->xfer.tx_buf = byte_buf;
	} else {
		/*
		 * Process all the whole words; to keep things simple we'll
		 * just wait for the next interrupt to handle the last 1-3
		 * bytes if we don't have an even number of words.
		 */
		rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
		wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;

		wr_size = min(rem_words, wr_fifo_words);
		ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;

		word_buf = xfer_buf;
		iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
		ctrl->xfer.tx_buf = word_buf + wr_size;

	}

	return IRQ_HANDLED;
}

static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
{
	u32 int_status;
	struct qcom_qspi *ctrl = dev_id;
	irqreturn_t ret = IRQ_NONE;
	unsigned long flags;

	spin_lock_irqsave(&ctrl->lock, flags);

	int_status = readl(ctrl->base + MSTR_INT_STATUS);
	writel(int_status, ctrl->base + MSTR_INT_STATUS);

	if (ctrl->xfer.dir == QSPI_WRITE) {
		if (int_status & WR_FIFO_EMPTY)
			ret = pio_write(ctrl);
	} else {
		if (int_status & RESP_FIFO_RDY)
			ret = pio_read(ctrl);
	}

	if (int_status & QSPI_ERR_IRQS) {
		if (int_status & RESP_FIFO_UNDERRUN)
			dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
		if (int_status & WR_FIFO_OVERRUN)
			dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
		if (int_status & HRESP_FROM_NOC_ERR)
			dev_err(ctrl->dev, "IRQ error: NOC response error\n");
		ret = IRQ_HANDLED;
	}

	if (!ctrl->xfer.rem_bytes) {
		writel(0, ctrl->base + MSTR_INT_EN);
		spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
	}

	spin_unlock_irqrestore(&ctrl->lock, flags);
	return ret;
}

static int qcom_qspi_probe(struct platform_device *pdev)
{
	int ret;
	struct device *dev;
	struct spi_master *master;
	struct qcom_qspi *ctrl;

	dev = &pdev->dev;

	master = spi_alloc_master(dev, sizeof(*ctrl));
	if (!master)
		return -ENOMEM;

	platform_set_drvdata(pdev, master);

	ctrl = spi_master_get_devdata(master);

	spin_lock_init(&ctrl->lock);
	ctrl->dev = dev;
	ctrl->base = devm_platform_ioremap_resource(pdev, 0);
	if (IS_ERR(ctrl->base)) {
		ret = PTR_ERR(ctrl->base);
		goto exit_probe_master_put;
	}

	ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
				  sizeof(*ctrl->clks), GFP_KERNEL);
	if (!ctrl->clks) {
		ret = -ENOMEM;
		goto exit_probe_master_put;
	}

	ctrl->clks[QSPI_CLK_CORE].id = "core";
	ctrl->clks[QSPI_CLK_IFACE].id = "iface";
	ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
	if (ret)
		goto exit_probe_master_put;

	ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
	if (IS_ERR(ctrl->icc_path_cpu_to_qspi)) {
		ret = dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
				    "Failed to get cpu path\n");
		goto exit_probe_master_put;
	}
	/* Set BW vote for register access */
	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
				Bps_to_icc(1000));
	if (ret) {
		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
				__func__, ret);
		goto exit_probe_master_put;
	}

	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
	if (ret) {
		dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
				__func__, ret);
		goto exit_probe_master_put;
	}

	ret = platform_get_irq(pdev, 0);
	if (ret < 0)
		goto exit_probe_master_put;
	ret = devm_request_irq(dev, ret, qcom_qspi_irq,
			IRQF_TRIGGER_HIGH, dev_name(dev), ctrl);
	if (ret) {
		dev_err(dev, "Failed to request irq %d\n", ret);
		goto exit_probe_master_put;
	}

	master->max_speed_hz = 300000000;
	master->num_chipselect = QSPI_NUM_CS;
	master->bus_num = -1;
	master->dev.of_node = pdev->dev.of_node;
	master->mode_bits = SPI_MODE_0 |
			    SPI_TX_DUAL | SPI_RX_DUAL |
			    SPI_TX_QUAD | SPI_RX_QUAD;
	master->flags = SPI_MASTER_HALF_DUPLEX;
	master->prepare_message = qcom_qspi_prepare_message;
	master->transfer_one = qcom_qspi_transfer_one;
	master->handle_err = qcom_qspi_handle_err;
	master->auto_runtime_pm = true;

	ctrl->opp_table = dev_pm_opp_set_clkname(&pdev->dev, "core");
	if (IS_ERR(ctrl->opp_table)) {
		ret = PTR_ERR(ctrl->opp_table);
		goto exit_probe_master_put;
	}
	/* OPP table is optional */
	ret = dev_pm_opp_of_add_table(&pdev->dev);
	if (ret && ret != -ENODEV) {
		dev_err(&pdev->dev, "invalid OPP table in device tree\n");
		goto exit_probe_put_clkname;
	}

	pm_runtime_use_autosuspend(dev);
	pm_runtime_set_autosuspend_delay(dev, 250);
	pm_runtime_enable(dev);

	ret = spi_register_master(master);
	if (!ret)
		return 0;

	pm_runtime_disable(dev);
	dev_pm_opp_of_remove_table(&pdev->dev);

exit_probe_put_clkname:
	dev_pm_opp_put_clkname(ctrl->opp_table);

exit_probe_master_put:
	spi_master_put(master);

	return ret;
}

static int qcom_qspi_remove(struct platform_device *pdev)
{
	struct spi_master *master = platform_get_drvdata(pdev);
	struct qcom_qspi *ctrl = spi_master_get_devdata(master);

	/* Unregister _before_ disabling pm_runtime() so we stop transfers */
	spi_unregister_master(master);

	pm_runtime_disable(&pdev->dev);
	dev_pm_opp_of_remove_table(&pdev->dev);
	dev_pm_opp_put_clkname(ctrl->opp_table);

	return 0;
}

static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
{
	struct spi_master *master = dev_get_drvdata(dev);
	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
	int ret;

	/* Drop the performance state vote */
	dev_pm_opp_set_rate(dev, 0);
	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);

	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
	if (ret) {
		dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
			__func__, ret);
		return ret;
	}

	return 0;
}

static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
{
	struct spi_master *master = dev_get_drvdata(dev);
	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
	int ret;

	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
	if (ret) {
		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
			__func__, ret);
		return ret;
	}

	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
	if (ret)
		return ret;

	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
}

static int __maybe_unused qcom_qspi_suspend(struct device *dev)
{
	struct spi_master *master = dev_get_drvdata(dev);
	int ret;

	ret = spi_master_suspend(master);
	if (ret)
		return ret;

	ret = pm_runtime_force_suspend(dev);
	if (ret)
		spi_master_resume(master);

	return ret;
}

static int __maybe_unused qcom_qspi_resume(struct device *dev)
{
	struct spi_master *master = dev_get_drvdata(dev);
	int ret;

	ret = pm_runtime_force_resume(dev);
	if (ret)
		return ret;

	ret = spi_master_resume(master);
	if (ret)
		pm_runtime_force_suspend(dev);

	return ret;
}

static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
	SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
			   qcom_qspi_runtime_resume, NULL)
	SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
};

static const struct of_device_id qcom_qspi_dt_match[] = {
	{ .compatible = "qcom,qspi-v1", },
	{ }
};
MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);

static struct platform_driver qcom_qspi_driver = {
	.driver = {
		.name		= "qcom_qspi",
		.pm		= &qcom_qspi_dev_pm_ops,
		.of_match_table = qcom_qspi_dt_match,
	},
	.probe = qcom_qspi_probe,
	.remove = qcom_qspi_remove,
};
module_platform_driver(qcom_qspi_driver);

MODULE_DESCRIPTION("SPI driver for QSPI cores");
MODULE_LICENSE("GPL v2");
Commit	Line	Data
04000dc6 GM	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
	3
	4	#include <linux/clk.h>
cff80645	5	#include <linux/interconnect.h>
04000dc6 GM	6	#include <linux/interrupt.h>
	7	#include <linux/io.h>
	8	#include <linux/module.h>
	9	#include <linux/of.h>
	10	#include <linux/of_platform.h>
	11	#include <linux/pm_runtime.h>
f79a158d	12	#include <linux/pm_opp.h>
04000dc6 GM	13	#include <linux/spi/spi.h>
	14	#include <linux/spi/spi-mem.h>
	15
	16
	17	#define QSPI_NUM_CS 2
	18	#define QSPI_BYTES_PER_WORD 4
	19
	20	#define MSTR_CONFIG 0x0000
	21	#define FULL_CYCLE_MODE BIT(3)
	22	#define FB_CLK_EN BIT(4)
	23	#define PIN_HOLDN BIT(6)
	24	#define PIN_WPN BIT(7)
	25	#define DMA_ENABLE BIT(8)
	26	#define BIG_ENDIAN_MODE BIT(9)
	27	#define SPI_MODE_MSK 0xc00
	28	#define SPI_MODE_SHFT 10
	29	#define CHIP_SELECT_NUM BIT(12)
	30	#define SBL_EN BIT(13)
	31	#define LPA_BASE_MSK 0x3c000
	32	#define LPA_BASE_SHFT 14
	33	#define TX_DATA_DELAY_MSK 0xc0000
	34	#define TX_DATA_DELAY_SHFT 18
	35	#define TX_CLK_DELAY_MSK 0x300000
	36	#define TX_CLK_DELAY_SHFT 20
	37	#define TX_CS_N_DELAY_MSK 0xc00000
	38	#define TX_CS_N_DELAY_SHFT 22
	39	#define TX_DATA_OE_DELAY_MSK 0x3000000
	40	#define TX_DATA_OE_DELAY_SHFT 24
	41
	42	#define AHB_MASTER_CFG 0x0004
	43	#define HMEM_TYPE_START_MID_TRANS_MSK 0x7
	44	#define HMEM_TYPE_START_MID_TRANS_SHFT 0
	45	#define HMEM_TYPE_LAST_TRANS_MSK 0x38
	46	#define HMEM_TYPE_LAST_TRANS_SHFT 3
	47	#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK 0xc0
	48	#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
	49	#define HMEMTYPE_READ_TRANS_MSK 0x700
	50	#define HMEMTYPE_READ_TRANS_SHFT 8
	51	#define HSHARED BIT(11)
	52	#define HINNERSHARED BIT(12)
	53
	54	#define MSTR_INT_EN 0x000C
	55	#define MSTR_INT_STATUS 0x0010
	56	#define RESP_FIFO_UNDERRUN BIT(0)
	57	#define RESP_FIFO_NOT_EMPTY BIT(1)
	58	#define RESP_FIFO_RDY BIT(2)
	59	#define HRESP_FROM_NOC_ERR BIT(3)
	60	#define WR_FIFO_EMPTY BIT(9)
	61	#define WR_FIFO_FULL BIT(10)
	62	#define WR_FIFO_OVERRUN BIT(11)
	63	#define TRANSACTION_DONE BIT(16)
	64	#define QSPI_ERR_IRQS (RESP_FIFO_UNDERRUN \| HRESP_FROM_NOC_ERR \| \
	65	WR_FIFO_OVERRUN)
	66	#define QSPI_ALL_IRQS (QSPI_ERR_IRQS \| RESP_FIFO_RDY \| \
	67	WR_FIFO_EMPTY \| WR_FIFO_FULL \| \
	68	TRANSACTION_DONE)
	69
	70	#define PIO_XFER_CTRL 0x0014
	71	#define REQUEST_COUNT_MSK 0xffff
	72
	73	#define PIO_XFER_CFG 0x0018
	74	#define TRANSFER_DIRECTION BIT(0)
	75	#define MULTI_IO_MODE_MSK 0xe
	76	#define MULTI_IO_MODE_SHFT 1
77	#define TRANSFER_FRAGMENT BIT(8)
78	#define SDR_1BIT 1
79	#define SDR_2BIT 2
80	#define SDR_4BIT 3
81	#define DDR_1BIT 5
82	#define DDR_2BIT 6
83	#define DDR_4BIT 7
84	#define DMA_DESC_SINGLE_SPI 1
85	#define DMA_DESC_DUAL_SPI 2
86	#define DMA_DESC_QUAD_SPI 3
87
88	#define PIO_XFER_STATUS 0x001c
89	#define WR_FIFO_BYTES_MSK 0xffff0000
90	#define WR_FIFO_BYTES_SHFT 16
91
92	#define PIO_DATAOUT_1B 0x0020
93	#define PIO_DATAOUT_4B 0x0024
94
478652f3 RC	95	#define RD_FIFO_CFG 0x0028
	96	#define CONTINUOUS_MODE BIT(0)
	97
04000dc6 GM	98	#define RD_FIFO_STATUS 0x002c
	99	#define FIFO_EMPTY BIT(11)
	100	#define WR_CNTS_MSK 0x7f0
	101	#define WR_CNTS_SHFT 4
	102	#define RDY_64BYTE BIT(3)
	103	#define RDY_32BYTE BIT(2)
	104	#define RDY_16BYTE BIT(1)
	105	#define FIFO_RDY BIT(0)
	106
04000dc6 GM	107	#define RD_FIFO_RESET 0x0030
	108	#define RESET_FIFO BIT(0)
	109
	110	#define CUR_MEM_ADDR 0x0048
	111	#define HW_VERSION 0x004c
	112	#define RD_FIFO 0x0050
	113	#define SAMPLING_CLK_CFG 0x0090
	114	#define SAMPLING_CLK_STATUS 0x0094
	115
	116
	117	enum qspi_dir {
	118	QSPI_READ,
	119	QSPI_WRITE,
	120	};
	121
	122	struct qspi_xfer {
	123	union {
	124	const void *tx_buf;
	125	void *rx_buf;
	126	};
	127	unsigned int rem_bytes;
	128	unsigned int buswidth;
	129	enum qspi_dir dir;
	130	bool is_last;
	131	};
	132
	133	enum qspi_clocks {
	134	QSPI_CLK_CORE,
	135	QSPI_CLK_IFACE,
	136	QSPI_NUM_CLKS
	137	};
	138
	139	struct qcom_qspi {
	140	void __iomem *base;
	141	struct device *dev;
b8d40d77	142	struct clk_bulk_data *clks;
04000dc6	143	struct qspi_xfer xfer;
cff80645	144	struct icc_path *icc_path_cpu_to_qspi;
f79a158d	145	struct opp_table *opp_table;
21243314	146	unsigned long last_speed;
cff80645	147	/* Lock to protect data accessed by IRQs */
04000dc6 GM	148	spinlock_t lock;
	149	};
	150
	151	static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
	152	unsigned int buswidth)
	153	{
	154	switch (buswidth) {
	155	case 1:
	156	return SDR_1BIT << MULTI_IO_MODE_SHFT;
	157	case 2:
	158	return SDR_2BIT << MULTI_IO_MODE_SHFT;
	159	case 4:
	160	return SDR_4BIT << MULTI_IO_MODE_SHFT;
	161	default:
	162	dev_warn_once(ctrl->dev,
	163	"Unexpected bus width: %u\n", buswidth);
	164	return SDR_1BIT << MULTI_IO_MODE_SHFT;
	165	}
	166	}
	167
	168	static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
	169	{
	170	u32 pio_xfer_cfg;
	171	const struct qspi_xfer *xfer;
	172
	173	xfer = &ctrl->xfer;
	174	pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
	175	pio_xfer_cfg &= ~TRANSFER_DIRECTION;
	176	pio_xfer_cfg \|= xfer->dir;
	177	if (xfer->is_last)
	178	pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
	179	else
	180	pio_xfer_cfg \|= TRANSFER_FRAGMENT;
	181	pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
	182	pio_xfer_cfg \|= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
	183
	184	writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
	185	}
	186
	187	static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
	188	{
	189	u32 pio_xfer_ctrl;
	190
	191	pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
	192	pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
	193	pio_xfer_ctrl \|= ctrl->xfer.rem_bytes;
	194	writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
	195	}
	196
	197	static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
	198	{
	199	u32 ints;
	200
	201	qcom_qspi_pio_xfer_cfg(ctrl);
	202
	203	/* Ack any previous interrupts that might be hanging around */
	204	writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
	205
	206	/* Setup new interrupts */
	207	if (ctrl->xfer.dir == QSPI_WRITE)
	208	ints = QSPI_ERR_IRQS \| WR_FIFO_EMPTY;
	209	else
	210	ints = QSPI_ERR_IRQS \| RESP_FIFO_RDY;
	211	writel(ints, ctrl->base + MSTR_INT_EN);
212
213	/* Kick off the transfer */
214	qcom_qspi_pio_xfer_ctrl(ctrl);
215	}
216
217	static void qcom_qspi_handle_err(struct spi_master *master,
218	struct spi_message *msg)
219	{
220	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
221	unsigned long flags;
222
223	spin_lock_irqsave(&ctrl->lock, flags);
224	writel(0, ctrl->base + MSTR_INT_EN);
225	ctrl->xfer.rem_bytes = 0;
226	spin_unlock_irqrestore(&ctrl->lock, flags);
227	}
228
21243314	229	static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
04000dc6	230	{
04000dc6	231	int ret;
cff80645	232	unsigned int avg_bw_cpu;
04000dc6	233
21243314 DA	234	if (speed_hz == ctrl->last_speed)
21243314 DA	235	return 0;
04000dc6 GM	236
04000dc6 GM	237	/* In regular operation (SBL_EN=1) core must be 4x transfer clock */
f79a158d	238	ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
04000dc6 GM	239	if (ret) {
	240	dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
	241	return ret;
	242	}
	243
cff80645 AA	244	/*
	245	* Set BW quota for CPU as driver supports FIFO mode only.
	246	* We don't have explicit peak requirement so keep it equal to avg_bw.
	247	*/
	248	avg_bw_cpu = Bps_to_icc(speed_hz);
	249	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
	250	if (ret) {
	251	dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
	252	__func__, ret);
	253	return ret;
	254	}
	255
21243314 DA	256	ctrl->last_speed = speed_hz;
	257
	258	return 0;
	259	}
	260
	261	static int qcom_qspi_transfer_one(struct spi_master *master,
	262	struct spi_device *slv,
	263	struct spi_transfer *xfer)
	264	{
	265	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
	266	int ret;
	267	unsigned long speed_hz;
	268	unsigned long flags;
	269
	270	speed_hz = slv->max_speed_hz;
	271	if (xfer->speed_hz)
	272	speed_hz = xfer->speed_hz;
	273
	274	ret = qcom_qspi_set_speed(ctrl, speed_hz);
	275	if (ret)
	276	return ret;
	277
04000dc6 GM	278	spin_lock_irqsave(&ctrl->lock, flags);
	279
	280	/* We are half duplex, so either rx or tx will be set */
	281	if (xfer->rx_buf) {
	282	ctrl->xfer.dir = QSPI_READ;
	283	ctrl->xfer.buswidth = xfer->rx_nbits;
	284	ctrl->xfer.rx_buf = xfer->rx_buf;
	285	} else {
	286	ctrl->xfer.dir = QSPI_WRITE;
	287	ctrl->xfer.buswidth = xfer->tx_nbits;
	288	ctrl->xfer.tx_buf = xfer->tx_buf;
	289	}
	290	ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
	291	&master->cur_msg->transfers);
	292	ctrl->xfer.rem_bytes = xfer->len;
	293	qcom_qspi_pio_xfer(ctrl);
	294
	295	spin_unlock_irqrestore(&ctrl->lock, flags);
	296
	297	/* We'll call spi_finalize_current_transfer() when done */
	298	return 1;
	299	}
	300
	301	static int qcom_qspi_prepare_message(struct spi_master *master,
	302	struct spi_message *message)
	303	{
	304	u32 mstr_cfg;
	305	struct qcom_qspi *ctrl;
	306	int tx_data_oe_delay = 1;
	307	int tx_data_delay = 1;
	308	unsigned long flags;
	309
	310	ctrl = spi_master_get_devdata(master);
	311	spin_lock_irqsave(&ctrl->lock, flags);
	312
	313	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
	314	mstr_cfg &= ~CHIP_SELECT_NUM;
	315	if (message->spi->chip_select)
	316	mstr_cfg \|= CHIP_SELECT_NUM;
	317
	318	mstr_cfg \|= FB_CLK_EN \| PIN_WPN \| PIN_HOLDN \| SBL_EN \| FULL_CYCLE_MODE;
	319	mstr_cfg &= ~(SPI_MODE_MSK \| TX_DATA_OE_DELAY_MSK \| TX_DATA_DELAY_MSK);
	320	mstr_cfg \|= message->spi->mode << SPI_MODE_SHFT;
	321	mstr_cfg \|= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
	322	mstr_cfg \|= tx_data_delay << TX_DATA_DELAY_SHFT;
	323	mstr_cfg &= ~DMA_ENABLE;
	324
	325	writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
	326	spin_unlock_irqrestore(&ctrl->lock, flags);
	327
	328	return 0;
	329	}
	330
	331	static irqreturn_t pio_read(struct qcom_qspi *ctrl)
	332	{
	333	u32 rd_fifo_status;
	334	u32 rd_fifo;
	335	unsigned int wr_cnts;
	336	unsigned int bytes_to_read;
	337	unsigned int words_to_read;
	338	u32 *word_buf;
	339	u8 *byte_buf;
	340	int i;
	341
342	rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
343
344	if (!(rd_fifo_status & FIFO_RDY)) {
345	dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
346	return IRQ_NONE;
347	}
348
349	wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
350	wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
351
352	words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
353	bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
354
355	if (words_to_read) {
356	word_buf = ctrl->xfer.rx_buf;
357	ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
358	ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
359	ctrl->xfer.rx_buf = word_buf + words_to_read;
360	}
361
362	if (bytes_to_read) {
363	byte_buf = ctrl->xfer.rx_buf;
364	rd_fifo = readl(ctrl->base + RD_FIFO);
365	ctrl->xfer.rem_bytes -= bytes_to_read;
366	for (i = 0; i < bytes_to_read; i++)
367	byte_buf++ = rd_fifo >> (i BITS_PER_BYTE);
368	ctrl->xfer.rx_buf = byte_buf;
369	}
370
371	return IRQ_HANDLED;
372	}
373
374	static irqreturn_t pio_write(struct qcom_qspi *ctrl)
375	{
376	const void *xfer_buf = ctrl->xfer.tx_buf;
377	const int *word_buf;
378	const char *byte_buf;
379	unsigned int wr_fifo_bytes;
380	unsigned int wr_fifo_words;
381	unsigned int wr_size;
382	unsigned int rem_words;
383
384	wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
385	wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
386
387	if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
388	/* Process the last 1-3 bytes */
389	wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
390	ctrl->xfer.rem_bytes -= wr_size;
391
392	byte_buf = xfer_buf;
393	while (wr_size--)
394	writel(*byte_buf++,
395	ctrl->base + PIO_DATAOUT_1B);
396	ctrl->xfer.tx_buf = byte_buf;
397	} else {
398	/*
399	* Process all the whole words; to keep things simple we'll
400	* just wait for the next interrupt to handle the last 1-3
401	* bytes if we don't have an even number of words.
402	*/
403	rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
404	wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
405
406	wr_size = min(rem_words, wr_fifo_words);
407	ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
408
409	word_buf = xfer_buf;
410	iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
411	ctrl->xfer.tx_buf = word_buf + wr_size;
412
413	}
414
415	return IRQ_HANDLED;
416	}
417
418	static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
419	{
420	u32 int_status;
421	struct qcom_qspi *ctrl = dev_id;
422	irqreturn_t ret = IRQ_NONE;
423	unsigned long flags;
424
425	spin_lock_irqsave(&ctrl->lock, flags);
426
427	int_status = readl(ctrl->base + MSTR_INT_STATUS);
428	writel(int_status, ctrl->base + MSTR_INT_STATUS);
429
430	if (ctrl->xfer.dir == QSPI_WRITE) {
431	if (int_status & WR_FIFO_EMPTY)
432	ret = pio_write(ctrl);
433	} else {
434	if (int_status & RESP_FIFO_RDY)
435	ret = pio_read(ctrl);
436	}
437
438	if (int_status & QSPI_ERR_IRQS) {
439	if (int_status & RESP_FIFO_UNDERRUN)
440	dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
441	if (int_status & WR_FIFO_OVERRUN)
442	dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
443	if (int_status & HRESP_FROM_NOC_ERR)
444	dev_err(ctrl->dev, "IRQ error: NOC response error\n");
445	ret = IRQ_HANDLED;
446	}
447
448	if (!ctrl->xfer.rem_bytes) {
449	writel(0, ctrl->base + MSTR_INT_EN);
450	spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
451	}
452
453	spin_unlock_irqrestore(&ctrl->lock, flags);
454	return ret;
455	}
456
457	static int qcom_qspi_probe(struct platform_device *pdev)
458	{
459	int ret;
460	struct device *dev;
04000dc6 GM	461	struct spi_master *master;
	462	struct qcom_qspi *ctrl;
	463
	464	dev = &pdev->dev;
	465
	466	master = spi_alloc_master(dev, sizeof(*ctrl));
	467	if (!master)
	468	return -ENOMEM;
	469
	470	platform_set_drvdata(pdev, master);
	471
	472	ctrl = spi_master_get_devdata(master);
	473
	474	spin_lock_init(&ctrl->lock);
	475	ctrl->dev = dev;
e0ea3cc2	476	ctrl->base = devm_platform_ioremap_resource(pdev, 0);
04000dc6 GM	477	if (IS_ERR(ctrl->base)) {
	478	ret = PTR_ERR(ctrl->base);
	479	goto exit_probe_master_put;
	480	}
	481
b8d40d77 MK	482	ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
	483	sizeof(*ctrl->clks), GFP_KERNEL);
	484	if (!ctrl->clks) {
	485	ret = -ENOMEM;
	486	goto exit_probe_master_put;
	487	}
	488
04000dc6 GM	489	ctrl->clks[QSPI_CLK_CORE].id = "core";
	490	ctrl->clks[QSPI_CLK_IFACE].id = "iface";
	491	ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
	492	if (ret)
	493	goto exit_probe_master_put;
	494
cff80645 AA	495	ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
cff80645 AA	496	if (IS_ERR(ctrl->icc_path_cpu_to_qspi)) {
03453268 KK	497	ret = dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
03453268 KK	498	"Failed to get cpu path\n");
cff80645 AA	499	goto exit_probe_master_put;
	500	}
	501	/* Set BW vote for register access */
	502	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
	503	Bps_to_icc(1000));
	504	if (ret) {
	505	dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
	506	__func__, ret);
	507	goto exit_probe_master_put;
	508	}
	509
	510	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
	511	if (ret) {
	512	dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
	513	__func__, ret);
	514	goto exit_probe_master_put;
	515	}
	516
04000dc6	517	ret = platform_get_irq(pdev, 0);
6b8ac10e	518	if (ret < 0)
04000dc6	519	goto exit_probe_master_put;
04000dc6 GM	520	ret = devm_request_irq(dev, ret, qcom_qspi_irq,
	521	IRQF_TRIGGER_HIGH, dev_name(dev), ctrl);
	522	if (ret) {
	523	dev_err(dev, "Failed to request irq %d\n", ret);
	524	goto exit_probe_master_put;
	525	}
	526
	527	master->max_speed_hz = 300000000;
	528	master->num_chipselect = QSPI_NUM_CS;
	529	master->bus_num = -1;
	530	master->dev.of_node = pdev->dev.of_node;
	531	master->mode_bits = SPI_MODE_0 \|
	532	SPI_TX_DUAL \| SPI_RX_DUAL \|
	533	SPI_TX_QUAD \| SPI_RX_QUAD;
	534	master->flags = SPI_MASTER_HALF_DUPLEX;
	535	master->prepare_message = qcom_qspi_prepare_message;
	536	master->transfer_one = qcom_qspi_transfer_one;
	537	master->handle_err = qcom_qspi_handle_err;
	538	master->auto_runtime_pm = true;
	539
f79a158d RN	540	ctrl->opp_table = dev_pm_opp_set_clkname(&pdev->dev, "core");
	541	if (IS_ERR(ctrl->opp_table)) {
	542	ret = PTR_ERR(ctrl->opp_table);
	543	goto exit_probe_master_put;
	544	}
	545	/* OPP table is optional */
	546	ret = dev_pm_opp_of_add_table(&pdev->dev);
062cf7fc	547	if (ret && ret != -ENODEV) {
f79a158d	548	dev_err(&pdev->dev, "invalid OPP table in device tree\n");
062cf7fc	549	goto exit_probe_put_clkname;
f79a158d RN	550	}
f79a158d RN	551
8592eb95 DA	552	pm_runtime_use_autosuspend(dev);
8592eb95 DA	553	pm_runtime_set_autosuspend_delay(dev, 250);
04000dc6 GM	554	pm_runtime_enable(dev);
	555
	556	ret = spi_register_master(master);
	557	if (!ret)
	558	return 0;
	559
	560	pm_runtime_disable(dev);
062cf7fc VK	561	dev_pm_opp_of_remove_table(&pdev->dev);
	562
	563	exit_probe_put_clkname:
f79a158d	564	dev_pm_opp_put_clkname(ctrl->opp_table);
04000dc6 GM	565
	566	exit_probe_master_put:
	567	spi_master_put(master);
	568
	569	return ret;
	570	}
	571
	572	static int qcom_qspi_remove(struct platform_device *pdev)
	573	{
	574	struct spi_master *master = platform_get_drvdata(pdev);
f79a158d	575	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
04000dc6 GM	576
	577	/* Unregister _before_ disabling pm_runtime() so we stop transfers */
	578	spi_unregister_master(master);
	579
	580	pm_runtime_disable(&pdev->dev);
062cf7fc	581	dev_pm_opp_of_remove_table(&pdev->dev);
f79a158d	582	dev_pm_opp_put_clkname(ctrl->opp_table);
04000dc6 GM	583
	584	return 0;
	585	}
	586
	587	static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
	588	{
	589	struct spi_master *master = dev_get_drvdata(dev);
	590	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
cff80645	591	int ret;
04000dc6	592
f79a158d RN	593	/* Drop the performance state vote */
f79a158d RN	594	dev_pm_opp_set_rate(dev, 0);
04000dc6 GM	595	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
04000dc6 GM	596
cff80645 AA	597	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
	598	if (ret) {
	599	dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
	600	__func__, ret);
	601	return ret;
	602	}
	603
04000dc6 GM	604	return 0;
	605	}
	606
	607	static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
	608	{
	609	struct spi_master *master = dev_get_drvdata(dev);
	610	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
cff80645 AA	611	int ret;
	612
	613	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
	614	if (ret) {
	615	dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
	616	__func__, ret);
	617	return ret;
	618	}
04000dc6	619
21243314 DA	620	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
	621	if (ret)
	622	return ret;
	623
	624	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
04000dc6 GM	625	}
	626
	627	static int __maybe_unused qcom_qspi_suspend(struct device *dev)
	628	{
	629	struct spi_master *master = dev_get_drvdata(dev);
	630	int ret;
	631
	632	ret = spi_master_suspend(master);
	633	if (ret)
	634	return ret;
	635
	636	ret = pm_runtime_force_suspend(dev);
	637	if (ret)
	638	spi_master_resume(master);
	639
	640	return ret;
	641	}
	642
	643	static int __maybe_unused qcom_qspi_resume(struct device *dev)
	644	{
	645	struct spi_master *master = dev_get_drvdata(dev);
	646	int ret;
	647
	648	ret = pm_runtime_force_resume(dev);
	649	if (ret)
	650	return ret;
	651
	652	ret = spi_master_resume(master);
	653	if (ret)
	654	pm_runtime_force_suspend(dev);
	655
	656	return ret;
	657	}
	658
	659	static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
	660	SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
	661	qcom_qspi_runtime_resume, NULL)
	662	SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
	663	};
	664
	665	static const struct of_device_id qcom_qspi_dt_match[] = {
	666	{ .compatible = "qcom,qspi-v1", },
	667	{ }
	668	};
	669	MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
	670
	671	static struct platform_driver qcom_qspi_driver = {
	672	.driver = {
	673	.name = "qcom_qspi",
	674	.pm = &qcom_qspi_dev_pm_ops,
	675	.of_match_table = qcom_qspi_dt_match,
	676	},
	677	.probe = qcom_qspi_probe,
	678	.remove = qcom_qspi_remove,
	679	};
	680	module_platform_driver(qcom_qspi_driver);
	681
	682	MODULE_DESCRIPTION("SPI driver for QSPI cores");
	683	MODULE_LICENSE("GPL v2");