[linux-block.git] / drivers / ufs / core / ufs-mcq.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved.
 *
 * Authors:
 *	Asutosh Das <quic_asutoshd@quicinc.com>
 *	Can Guo <quic_cang@quicinc.com>
 */

#include <asm/unaligned.h>
#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include "ufshcd-priv.h"

#define MAX_QUEUE_SUP GENMASK(7, 0)
#define UFS_MCQ_MIN_RW_QUEUES 2
#define UFS_MCQ_MIN_READ_QUEUES 0
#define UFS_MCQ_NUM_DEV_CMD_QUEUES 1
#define UFS_MCQ_MIN_POLL_QUEUES 0
#define QUEUE_EN_OFFSET 31
#define QUEUE_ID_OFFSET 16

#define MAX_DEV_CMD_ENTRIES	2
#define MCQ_CFG_MAC_MASK	GENMASK(16, 8)
#define MCQ_QCFG_SIZE		0x40
#define MCQ_ENTRY_SIZE_IN_DWORD	8

static int rw_queue_count_set(const char *val, const struct kernel_param *kp)
{
	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
				     num_possible_cpus());
}

static const struct kernel_param_ops rw_queue_count_ops = {
	.set = rw_queue_count_set,
	.get = param_get_uint,
};

static unsigned int rw_queues;
module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644);
MODULE_PARM_DESC(rw_queues,
		 "Number of interrupt driven I/O queues used for rw. Default value is nr_cpus");

static int read_queue_count_set(const char *val, const struct kernel_param *kp)
{
	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES,
				     num_possible_cpus());
}

static const struct kernel_param_ops read_queue_count_ops = {
	.set = read_queue_count_set,
	.get = param_get_uint,
};

static unsigned int read_queues;
module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644);
MODULE_PARM_DESC(read_queues,
		 "Number of interrupt driven read queues used for read. Default value is 0");

static int poll_queue_count_set(const char *val, const struct kernel_param *kp)
{
	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES,
				     num_possible_cpus());
}

static const struct kernel_param_ops poll_queue_count_ops = {
	.set = poll_queue_count_set,
	.get = param_get_uint,
};

static unsigned int poll_queues = 1;
module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644);
MODULE_PARM_DESC(poll_queues,
		 "Number of poll queues used for r/w. Default value is 1");

/**
 * ufshcd_mcq_config_mac - Set the #Max Activ Cmds.
 * @hba - per adapter instance
 * @max_active_cmds - maximum # of active commands to the device at any time.
 *
 * The controller won't send more than the max_active_cmds to the device at
 * any time.
 */
void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)
{
	u32 val;

	val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);
	val &= ~MCQ_CFG_MAC_MASK;
	val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds);
	ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);
}

/**
 * ufshcd_mcq_decide_queue_depth - decide the queue depth
 * @hba - per adapter instance
 *
 * Returns queue-depth on success, non-zero on error
 *
 * MAC - Max. Active Command of the Host Controller (HC)
 * HC wouldn't send more than this commands to the device.
 * It is mandatory to implement get_hba_mac() to enable MCQ mode.
 * Calculates and adjusts the queue depth based on the depth
 * supported by the HC and ufs device.
 */
int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
{
	int mac;

	/* Mandatory to implement get_hba_mac() */
	mac = ufshcd_mcq_vops_get_hba_mac(hba);
	if (mac < 0) {
		dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
		return mac;
	}

	WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
	/*
	 * max. value of bqueuedepth = 256, mac is host dependent.
	 * It is mandatory for UFS device to define bQueueDepth if
	 * shared queuing architecture is enabled.
	 */
	return min_t(int, mac, hba->dev_info.bqueuedepth);
}

static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
{
	int i;
	u32 hba_maxq, rem, tot_queues;
	struct Scsi_Host *host = hba->host;

	hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities);

	tot_queues = UFS_MCQ_NUM_DEV_CMD_QUEUES + read_queues + poll_queues +
			rw_queues;

	if (hba_maxq < tot_queues) {
		dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
			tot_queues, hba_maxq);
		return -EOPNOTSUPP;
	}

	rem = hba_maxq - UFS_MCQ_NUM_DEV_CMD_QUEUES;

	if (rw_queues) {
		hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
		rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];
	} else {
		rw_queues = num_possible_cpus();
	}

	if (poll_queues) {
		hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;
		rem -= hba->nr_queues[HCTX_TYPE_POLL];
	}

	if (read_queues) {
		hba->nr_queues[HCTX_TYPE_READ] = read_queues;
		rem -= hba->nr_queues[HCTX_TYPE_READ];
	}

	if (!hba->nr_queues[HCTX_TYPE_DEFAULT])
		hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,
							 num_possible_cpus());

	for (i = 0; i < HCTX_MAX_TYPES; i++)
		host->nr_hw_queues += hba->nr_queues[i];

	hba->nr_hw_queues = host->nr_hw_queues + UFS_MCQ_NUM_DEV_CMD_QUEUES;
	return 0;
}

int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)
{
	struct ufs_hw_queue *hwq;
	size_t utrdl_size, cqe_size;
	int i;

	for (i = 0; i < hba->nr_hw_queues; i++) {
		hwq = &hba->uhq[i];

		utrdl_size = sizeof(struct utp_transfer_req_desc) *
			     hwq->max_entries;
		hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,
							 &hwq->sqe_dma_addr,
							 GFP_KERNEL);
		if (!hwq->sqe_dma_addr) {
			dev_err(hba->dev, "SQE allocation failed\n");
			return -ENOMEM;
		}

		cqe_size = sizeof(struct cq_entry) * hwq->max_entries;
		hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,
							 &hwq->cqe_dma_addr,
							 GFP_KERNEL);
		if (!hwq->cqe_dma_addr) {
			dev_err(hba->dev, "CQE allocation failed\n");
			return -ENOMEM;
		}
	}

	return 0;
}


/* Operation and runtime registers configuration */
#define MCQ_CFG_n(r, i)	((r) + MCQ_QCFG_SIZE * (i))
#define MCQ_OPR_OFFSET_n(p, i) \
	(hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i))

static void __iomem *mcq_opr_base(struct ufs_hba *hba,
					 enum ufshcd_mcq_opr n, int i)
{
	struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];

	return opr->base + opr->stride * i;
}

void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
{
	struct ufs_hw_queue *hwq;
	u16 qsize;
	int i;

	for (i = 0; i < hba->nr_hw_queues; i++) {
		hwq = &hba->uhq[i];
		hwq->id = i;
		qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1;

		/* Submission Queue Lower Base Address */
		ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),
			      MCQ_CFG_n(REG_SQLBA, i));
		/* Submission Queue Upper Base Address */
		ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),
			      MCQ_CFG_n(REG_SQUBA, i));
		/* Submission Queue Doorbell Address Offset */
		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i),
			      MCQ_CFG_n(REG_SQDAO, i));
		/* Submission Queue Interrupt Status Address Offset */
		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i),
			      MCQ_CFG_n(REG_SQISAO, i));

		/* Completion Queue Lower Base Address */
		ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),
			      MCQ_CFG_n(REG_CQLBA, i));
		/* Completion Queue Upper Base Address */
		ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),
			      MCQ_CFG_n(REG_CQUBA, i));
		/* Completion Queue Doorbell Address Offset */
		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i),
			      MCQ_CFG_n(REG_CQDAO, i));
		/* Completion Queue Interrupt Status Address Offset */
		ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i),
			      MCQ_CFG_n(REG_CQISAO, i));

		/* Save the base addresses for quicker access */
		hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;
		hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;
		hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;
		hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;

		/* Enable Tail Entry Push Status interrupt only for non-poll queues */
		if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])
			writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);

		/* Completion Queue Enable|Size to Completion Queue Attribute */
		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize,
			      MCQ_CFG_n(REG_CQATTR, i));

		/*
		 * Submission Qeueue Enable|Size|Completion Queue ID to
		 * Submission Queue Attribute
		 */
		ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) | qsize |
			      (i << QUEUE_ID_OFFSET),
			      MCQ_CFG_n(REG_SQATTR, i));
	}
}

int ufshcd_mcq_init(struct ufs_hba *hba)
{
	struct Scsi_Host *host = hba->host;
	struct ufs_hw_queue *hwq;
	int ret, i;

	ret = ufshcd_mcq_config_nr_queues(hba);
	if (ret)
		return ret;

	ret = ufshcd_vops_mcq_config_resource(hba);
	if (ret)
		return ret;

	ret = ufshcd_mcq_vops_op_runtime_config(hba);
	if (ret) {
		dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",
			ret);
		return ret;
	}
	hba->uhq = devm_kzalloc(hba->dev,
				hba->nr_hw_queues * sizeof(struct ufs_hw_queue),
				GFP_KERNEL);
	if (!hba->uhq) {
		dev_err(hba->dev, "ufs hw queue memory allocation failed\n");
		return -ENOMEM;
	}

	for (i = 0; i < hba->nr_hw_queues; i++) {
		hwq = &hba->uhq[i];
		hwq->max_entries = hba->nutrs;
		spin_lock_init(&hwq->sq_lock);
	}

	/* The very first HW queue serves device commands */
	hba->dev_cmd_queue = &hba->uhq[0];
	/* Give dev_cmd_queue the minimal number of entries */
	hba->dev_cmd_queue->max_entries = MAX_DEV_CMD_ENTRIES;

	host->host_tagset = 1;
	return 0;
}
Commit	Line	Data
57b1c0ef AD	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2022 Qualcomm Innovation Center. All rights reserved.
	4	*
	5	* Authors:
	6	* Asutosh Das <quic_asutoshd@quicinc.com>
	7	* Can Guo <quic_cang@quicinc.com>
	8	*/
	9
	10	#include <asm/unaligned.h>
	11	#include <linux/dma-mapping.h>
	12	#include <linux/module.h>
	13	#include <linux/platform_device.h>
	14	#include "ufshcd-priv.h"
	15
	16	#define MAX_QUEUE_SUP GENMASK(7, 0)
	17	#define UFS_MCQ_MIN_RW_QUEUES 2
	18	#define UFS_MCQ_MIN_READ_QUEUES 0
	19	#define UFS_MCQ_NUM_DEV_CMD_QUEUES 1
	20	#define UFS_MCQ_MIN_POLL_QUEUES 0
2468da61 AD	21	#define QUEUE_EN_OFFSET 31
2468da61 AD	22	#define QUEUE_ID_OFFSET 16
57b1c0ef	23
7224c806 AD	24	#define MAX_DEV_CMD_ENTRIES 2
7224c806 AD	25	#define MCQ_CFG_MAC_MASK GENMASK(16, 8)
2468da61 AD	26	#define MCQ_QCFG_SIZE 0x40
2468da61 AD	27	#define MCQ_ENTRY_SIZE_IN_DWORD 8
7224c806	28
57b1c0ef AD	29	static int rw_queue_count_set(const char val, const struct kernel_param kp)
	30	{
	31	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_RW_QUEUES,
	32	num_possible_cpus());
	33	}
	34
	35	static const struct kernel_param_ops rw_queue_count_ops = {
	36	.set = rw_queue_count_set,
	37	.get = param_get_uint,
	38	};
	39
	40	static unsigned int rw_queues;
	41	module_param_cb(rw_queues, &rw_queue_count_ops, &rw_queues, 0644);
	42	MODULE_PARM_DESC(rw_queues,
	43	"Number of interrupt driven I/O queues used for rw. Default value is nr_cpus");
	44
	45	static int read_queue_count_set(const char val, const struct kernel_param kp)
	46	{
	47	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_READ_QUEUES,
	48	num_possible_cpus());
	49	}
	50
	51	static const struct kernel_param_ops read_queue_count_ops = {
	52	.set = read_queue_count_set,
	53	.get = param_get_uint,
	54	};
	55
	56	static unsigned int read_queues;
	57	module_param_cb(read_queues, &read_queue_count_ops, &read_queues, 0644);
	58	MODULE_PARM_DESC(read_queues,
	59	"Number of interrupt driven read queues used for read. Default value is 0");
	60
	61	static int poll_queue_count_set(const char val, const struct kernel_param kp)
	62	{
	63	return param_set_uint_minmax(val, kp, UFS_MCQ_MIN_POLL_QUEUES,
	64	num_possible_cpus());
	65	}
	66
	67	static const struct kernel_param_ops poll_queue_count_ops = {
	68	.set = poll_queue_count_set,
	69	.get = param_get_uint,
	70	};
	71
	72	static unsigned int poll_queues = 1;
	73	module_param_cb(poll_queues, &poll_queue_count_ops, &poll_queues, 0644);
	74	MODULE_PARM_DESC(poll_queues,
	75	"Number of poll queues used for r/w. Default value is 1");
	76
2468da61 AD	77	/**
	78	* ufshcd_mcq_config_mac - Set the #Max Activ Cmds.
	79	* @hba - per adapter instance
	80	* @max_active_cmds - maximum # of active commands to the device at any time.
	81	*
	82	* The controller won't send more than the max_active_cmds to the device at
	83	* any time.
	84	*/
	85	void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds)
	86	{
	87	u32 val;
	88
	89	val = ufshcd_readl(hba, REG_UFS_MCQ_CFG);
	90	val &= ~MCQ_CFG_MAC_MASK;
	91	val \|= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds);
	92	ufshcd_writel(hba, val, REG_UFS_MCQ_CFG);
	93	}
	94
7224c806 AD	95	/**
	96	* ufshcd_mcq_decide_queue_depth - decide the queue depth
	97	* @hba - per adapter instance
	98	*
	99	* Returns queue-depth on success, non-zero on error
	100	*
	101	* MAC - Max. Active Command of the Host Controller (HC)
	102	* HC wouldn't send more than this commands to the device.
	103	* It is mandatory to implement get_hba_mac() to enable MCQ mode.
	104	* Calculates and adjusts the queue depth based on the depth
	105	* supported by the HC and ufs device.
	106	*/
	107	int ufshcd_mcq_decide_queue_depth(struct ufs_hba *hba)
	108	{
	109	int mac;
	110
	111	/* Mandatory to implement get_hba_mac() */
	112	mac = ufshcd_mcq_vops_get_hba_mac(hba);
	113	if (mac < 0) {
	114	dev_err(hba->dev, "Failed to get mac, err=%d\n", mac);
	115	return mac;
	116	}
	117
	118	WARN_ON_ONCE(!hba->dev_info.bqueuedepth);
	119	/*
	120	* max. value of bqueuedepth = 256, mac is host dependent.
	121	* It is mandatory for UFS device to define bQueueDepth if
	122	* shared queuing architecture is enabled.
	123	*/
	124	return min_t(int, mac, hba->dev_info.bqueuedepth);
	125	}
	126
57b1c0ef AD	127	static int ufshcd_mcq_config_nr_queues(struct ufs_hba *hba)
	128	{
	129	int i;
	130	u32 hba_maxq, rem, tot_queues;
	131	struct Scsi_Host *host = hba->host;
	132
	133	hba_maxq = FIELD_GET(MAX_QUEUE_SUP, hba->mcq_capabilities);
	134
	135	tot_queues = UFS_MCQ_NUM_DEV_CMD_QUEUES + read_queues + poll_queues +
	136	rw_queues;
	137
	138	if (hba_maxq < tot_queues) {
	139	dev_err(hba->dev, "Total queues (%d) exceeds HC capacity (%d)\n",
	140	tot_queues, hba_maxq);
	141	return -EOPNOTSUPP;
	142	}
	143
	144	rem = hba_maxq - UFS_MCQ_NUM_DEV_CMD_QUEUES;
	145
	146	if (rw_queues) {
	147	hba->nr_queues[HCTX_TYPE_DEFAULT] = rw_queues;
	148	rem -= hba->nr_queues[HCTX_TYPE_DEFAULT];
	149	} else {
	150	rw_queues = num_possible_cpus();
	151	}
	152
	153	if (poll_queues) {
	154	hba->nr_queues[HCTX_TYPE_POLL] = poll_queues;
	155	rem -= hba->nr_queues[HCTX_TYPE_POLL];
	156	}
	157
	158	if (read_queues) {
	159	hba->nr_queues[HCTX_TYPE_READ] = read_queues;
	160	rem -= hba->nr_queues[HCTX_TYPE_READ];
	161	}
	162
	163	if (!hba->nr_queues[HCTX_TYPE_DEFAULT])
	164	hba->nr_queues[HCTX_TYPE_DEFAULT] = min3(rem, rw_queues,
	165	num_possible_cpus());
	166
	167	for (i = 0; i < HCTX_MAX_TYPES; i++)
	168	host->nr_hw_queues += hba->nr_queues[i];
	169
	170	hba->nr_hw_queues = host->nr_hw_queues + UFS_MCQ_NUM_DEV_CMD_QUEUES;
	171	return 0;
	172	}
	173
4682abfa AD	174	int ufshcd_mcq_memory_alloc(struct ufs_hba *hba)
	175	{
	176	struct ufs_hw_queue *hwq;
	177	size_t utrdl_size, cqe_size;
	178	int i;
	179
	180	for (i = 0; i < hba->nr_hw_queues; i++) {
	181	hwq = &hba->uhq[i];
	182
	183	utrdl_size = sizeof(struct utp_transfer_req_desc) *
	184	hwq->max_entries;
	185	hwq->sqe_base_addr = dmam_alloc_coherent(hba->dev, utrdl_size,
	186	&hwq->sqe_dma_addr,
	187	GFP_KERNEL);
	188	if (!hwq->sqe_dma_addr) {
	189	dev_err(hba->dev, "SQE allocation failed\n");
	190	return -ENOMEM;
	191	}
	192
	193	cqe_size = sizeof(struct cq_entry) * hwq->max_entries;
	194	hwq->cqe_base_addr = dmam_alloc_coherent(hba->dev, cqe_size,
	195	&hwq->cqe_dma_addr,
	196	GFP_KERNEL);
	197	if (!hwq->cqe_dma_addr) {
	198	dev_err(hba->dev, "CQE allocation failed\n");
	199	return -ENOMEM;
	200	}
	201	}
	202
	203	return 0;
	204	}
	205
	206
2468da61 AD	207	/* Operation and runtime registers configuration */
	208	#define MCQ_CFG_n(r, i) ((r) + MCQ_QCFG_SIZE * (i))
	209	#define MCQ_OPR_OFFSET_n(p, i) \
	210	(hba->mcq_opr[(p)].offset + hba->mcq_opr[(p)].stride * (i))
	211
	212	static void __iomem mcq_opr_base(struct ufs_hba hba,
	213	enum ufshcd_mcq_opr n, int i)
	214	{
	215	struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[n];
	216
	217	return opr->base + opr->stride * i;
	218	}
	219
	220	void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba)
	221	{
	222	struct ufs_hw_queue *hwq;
	223	u16 qsize;
	224	int i;
	225
	226	for (i = 0; i < hba->nr_hw_queues; i++) {
	227	hwq = &hba->uhq[i];
	228	hwq->id = i;
	229	qsize = hwq->max_entries * MCQ_ENTRY_SIZE_IN_DWORD - 1;
	230
	231	/* Submission Queue Lower Base Address */
	232	ufsmcq_writelx(hba, lower_32_bits(hwq->sqe_dma_addr),
	233	MCQ_CFG_n(REG_SQLBA, i));
	234	/* Submission Queue Upper Base Address */
	235	ufsmcq_writelx(hba, upper_32_bits(hwq->sqe_dma_addr),
	236	MCQ_CFG_n(REG_SQUBA, i));
	237	/* Submission Queue Doorbell Address Offset */
	238	ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQD, i),
	239	MCQ_CFG_n(REG_SQDAO, i));
	240	/* Submission Queue Interrupt Status Address Offset */
	241	ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_SQIS, i),
	242	MCQ_CFG_n(REG_SQISAO, i));
	243
	244	/* Completion Queue Lower Base Address */
	245	ufsmcq_writelx(hba, lower_32_bits(hwq->cqe_dma_addr),
	246	MCQ_CFG_n(REG_CQLBA, i));
	247	/* Completion Queue Upper Base Address */
	248	ufsmcq_writelx(hba, upper_32_bits(hwq->cqe_dma_addr),
	249	MCQ_CFG_n(REG_CQUBA, i));
	250	/* Completion Queue Doorbell Address Offset */
	251	ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQD, i),
	252	MCQ_CFG_n(REG_CQDAO, i));
	253	/* Completion Queue Interrupt Status Address Offset */
	254	ufsmcq_writelx(hba, MCQ_OPR_OFFSET_n(OPR_CQIS, i),
	255	MCQ_CFG_n(REG_CQISAO, i));
	256
	257	/* Save the base addresses for quicker access */
	258	hwq->mcq_sq_head = mcq_opr_base(hba, OPR_SQD, i) + REG_SQHP;
	259	hwq->mcq_sq_tail = mcq_opr_base(hba, OPR_SQD, i) + REG_SQTP;
	260	hwq->mcq_cq_head = mcq_opr_base(hba, OPR_CQD, i) + REG_CQHP;
	261	hwq->mcq_cq_tail = mcq_opr_base(hba, OPR_CQD, i) + REG_CQTP;
	262
	263	/* Enable Tail Entry Push Status interrupt only for non-poll queues */
	264	if (i < hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL])
	265	writel(1, mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIE);
	266
	267	/* Completion Queue Enable\|Size to Completion Queue Attribute */
	268	ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) \| qsize,
	269	MCQ_CFG_n(REG_CQATTR, i));
	270
271	/*
272	* Submission Qeueue Enable\|Size\|Completion Queue ID to
273	* Submission Queue Attribute
274	*/
275	ufsmcq_writel(hba, (1 << QUEUE_EN_OFFSET) \| qsize \|
276	(i << QUEUE_ID_OFFSET),
277	MCQ_CFG_n(REG_SQATTR, i));
278	}
279	}
280
57b1c0ef AD	281	int ufshcd_mcq_init(struct ufs_hba *hba)
57b1c0ef AD	282	{
0d33728f	283	struct Scsi_Host *host = hba->host;
4682abfa AD	284	struct ufs_hw_queue *hwq;
4682abfa AD	285	int ret, i;
57b1c0ef AD	286
57b1c0ef AD	287	ret = ufshcd_mcq_config_nr_queues(hba);
c263b4ef AD	288	if (ret)
c263b4ef AD	289	return ret;
57b1c0ef	290
c263b4ef	291	ret = ufshcd_vops_mcq_config_resource(hba);
4682abfa AD	292	if (ret)
	293	return ret;
	294
2468da61 AD	295	ret = ufshcd_mcq_vops_op_runtime_config(hba);
	296	if (ret) {
	297	dev_err(hba->dev, "Operation runtime config failed, ret=%d\n",
	298	ret);
	299	return ret;
	300	}
4682abfa AD	301	hba->uhq = devm_kzalloc(hba->dev,
	302	hba->nr_hw_queues * sizeof(struct ufs_hw_queue),
	303	GFP_KERNEL);
	304	if (!hba->uhq) {
	305	dev_err(hba->dev, "ufs hw queue memory allocation failed\n");
	306	return -ENOMEM;
	307	}
	308
	309	for (i = 0; i < hba->nr_hw_queues; i++) {
	310	hwq = &hba->uhq[i];
	311	hwq->max_entries = hba->nutrs;
22a2d563	312	spin_lock_init(&hwq->sq_lock);
4682abfa AD	313	}
	314
	315	/* The very first HW queue serves device commands */
	316	hba->dev_cmd_queue = &hba->uhq[0];
	317	/* Give dev_cmd_queue the minimal number of entries */
	318	hba->dev_cmd_queue->max_entries = MAX_DEV_CMD_ENTRIES;
	319
0d33728f	320	host->host_tagset = 1;
4682abfa	321	return 0;
57b1c0ef	322	}