[linux-block.git] / drivers / iommu / arm / arm-smmu / arm-smmu-qcom.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
 */

#include <linux/acpi.h>
#include <linux/adreno-smmu-priv.h>
#include <linux/of_device.h>
#include <linux/qcom_scm.h>

#include "arm-smmu.h"

struct qcom_smmu {
	struct arm_smmu_device smmu;
	bool bypass_quirk;
	u8 bypass_cbndx;
};

static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
{
	return container_of(smmu, struct qcom_smmu, smmu);
}

static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
		u32 reg)
{
	/*
	 * On the GPU device we want to process subsequent transactions after a
	 * fault to keep the GPU from hanging
	 */
	reg |= ARM_SMMU_SCTLR_HUPCF;

	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
}

#define QCOM_ADRENO_SMMU_GPU_SID 0

static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
{
	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
	int i;

	/*
	 * The GPU will always use SID 0 so that is a handy way to uniquely
	 * identify it and configure it for per-instance pagetables
	 */
	for (i = 0; i < fwspec->num_ids; i++) {
		u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);

		if (sid == QCOM_ADRENO_SMMU_GPU_SID)
			return true;
	}

	return false;
}

static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
		const void *cookie)
{
	struct arm_smmu_domain *smmu_domain = (void *)cookie;
	struct io_pgtable *pgtable =
		io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
	return &pgtable->cfg;
}

/*
 * Local implementation to configure TTBR0 with the specified pagetable config.
 * The GPU driver will call this to enable TTBR0 when per-instance pagetables
 * are active
 */

static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
		const struct io_pgtable_cfg *pgtbl_cfg)
{
	struct arm_smmu_domain *smmu_domain = (void *)cookie;
	struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];

	/* The domain must have split pagetables already enabled */
	if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
		return -EINVAL;

	/* If the pagetable config is NULL, disable TTBR0 */
	if (!pgtbl_cfg) {
		/* Do nothing if it is already disabled */
		if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
			return -EINVAL;

		/* Set TCR to the original configuration */
		cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
		cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
	} else {
		u32 tcr = cb->tcr[0];

		/* Don't call this again if TTBR0 is already enabled */
		if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
			return -EINVAL;

		tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
		tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);

		cb->tcr[0] = tcr;
		cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
		cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
	}

	arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);

	return 0;
}

static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
					       struct arm_smmu_device *smmu,
					       struct device *dev, int start)
{
	int count;

	/*
	 * Assign context bank 0 to the GPU device so the GPU hardware can
	 * switch pagetables
	 */
	if (qcom_adreno_smmu_is_gpu_device(dev)) {
		start = 0;
		count = 1;
	} else {
		start = 1;
		count = smmu->num_context_banks;
	}

	return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
}

static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
{
	struct adreno_smmu_priv *priv;

	/* Only enable split pagetables for the GPU device (SID 0) */
	if (!qcom_adreno_smmu_is_gpu_device(dev))
		return 0;

	/*
	 * All targets that use the qcom,adreno-smmu compatible string *should*
	 * be AARCH64 stage 1 but double check because the arm-smmu code assumes
	 * that is the case when the TTBR1 quirk is enabled
	 */
	if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
	    (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
		pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;

	/*
	 * Initialize private interface with GPU:
	 */

	priv = dev_get_drvdata(dev);
	priv->cookie = smmu_domain;
	priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
	priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;

	return 0;
}

static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
	{ .compatible = "qcom,adreno" },
	{ .compatible = "qcom,mdp4" },
	{ .compatible = "qcom,mdss" },
	{ .compatible = "qcom,sc7180-mdss" },
	{ .compatible = "qcom,sc7180-mss-pil" },
	{ .compatible = "qcom,sc8180x-mdss" },
	{ .compatible = "qcom,sdm845-mdss" },
	{ .compatible = "qcom,sdm845-mss-pil" },
	{ }
};

static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
{
	unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
	u32 reg;
	u32 smr;
	int i;

	/*
	 * With some firmware versions writes to S2CR of type FAULT are
	 * ignored, and writing BYPASS will end up written as FAULT in the
	 * register. Perform a write to S2CR to detect if this is the case and
	 * if so reserve a context bank to emulate bypass streams.
	 */
	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
	      FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
	      FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
	arm_smmu_gr0_write(smmu, last_s2cr, reg);
	reg = arm_smmu_gr0_read(smmu, last_s2cr);
	if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
		qsmmu->bypass_quirk = true;
		qsmmu->bypass_cbndx = smmu->num_context_banks - 1;

		set_bit(qsmmu->bypass_cbndx, smmu->context_map);

		arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);

		reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
		arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
	}

	for (i = 0; i < smmu->num_mapping_groups; i++) {
		smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));

		if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
			/* Ignore valid bit for SMR mask extraction. */
			smr &= ~ARM_SMMU_SMR_VALID;
			smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
			smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
			smmu->smrs[i].valid = true;

			smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
			smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
			smmu->s2crs[i].cbndx = 0xff;
		}
	}

	return 0;
}

static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
{
	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
	u32 cbndx = s2cr->cbndx;
	u32 type = s2cr->type;
	u32 reg;

	if (qsmmu->bypass_quirk) {
		if (type == S2CR_TYPE_BYPASS) {
			/*
			 * Firmware with quirky S2CR handling will substitute
			 * BYPASS writes with FAULT, so point the stream to the
			 * reserved context bank and ask for translation on the
			 * stream
			 */
			type = S2CR_TYPE_TRANS;
			cbndx = qsmmu->bypass_cbndx;
		} else if (type == S2CR_TYPE_FAULT) {
			/*
			 * Firmware with quirky S2CR handling will ignore FAULT
			 * writes, so trick it to write FAULT by asking for a
			 * BYPASS.
			 */
			type = S2CR_TYPE_BYPASS;
			cbndx = 0xff;
		}
	}

	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
	      FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
	      FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
}

static int qcom_smmu_def_domain_type(struct device *dev)
{
	const struct of_device_id *match =
		of_match_device(qcom_smmu_client_of_match, dev);

	return match ? IOMMU_DOMAIN_IDENTITY : 0;
}

static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
{
	int ret;

	/*
	 * To address performance degradation in non-real time clients,
	 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
	 * such as MTP and db845, whose firmwares implement secure monitor
	 * call handlers to turn on/off the wait-for-safe logic.
	 */
	ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
	if (ret)
		dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");

	return ret;
}

static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
{
	const struct device_node *np = smmu->dev->of_node;

	arm_mmu500_reset(smmu);

	if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
		return qcom_sdm845_smmu500_reset(smmu);

	return 0;
}

static const struct arm_smmu_impl qcom_smmu_impl = {
	.cfg_probe = qcom_smmu_cfg_probe,
	.def_domain_type = qcom_smmu_def_domain_type,
	.reset = qcom_smmu500_reset,
	.write_s2cr = qcom_smmu_write_s2cr,
};

static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
	.init_context = qcom_adreno_smmu_init_context,
	.def_domain_type = qcom_smmu_def_domain_type,
	.reset = qcom_smmu500_reset,
	.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
	.write_sctlr = qcom_adreno_smmu_write_sctlr,
};

static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
		const struct arm_smmu_impl *impl)
{
	struct qcom_smmu *qsmmu;

	/* Check to make sure qcom_scm has finished probing */
	if (!qcom_scm_is_available())
		return ERR_PTR(-EPROBE_DEFER);

	qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
	if (!qsmmu)
		return ERR_PTR(-ENOMEM);

	qsmmu->smmu.impl = impl;

	return &qsmmu->smmu;
}

static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
	{ .compatible = "qcom,msm8998-smmu-v2" },
	{ .compatible = "qcom,sc7180-smmu-500" },
	{ .compatible = "qcom,sc8180x-smmu-500" },
	{ .compatible = "qcom,sdm630-smmu-v2" },
	{ .compatible = "qcom,sdm845-smmu-500" },
	{ .compatible = "qcom,sm6125-smmu-500" },
	{ .compatible = "qcom,sm8150-smmu-500" },
	{ .compatible = "qcom,sm8250-smmu-500" },
	{ .compatible = "qcom,sm8350-smmu-500" },
	{ }
};

static struct acpi_platform_list qcom_acpi_platlist[] = {
	{ "LENOVO", "CB-01   ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
	{ "QCOM  ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
	{ }
};

struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
{
	const struct device_node *np = smmu->dev->of_node;

	if (np == NULL) {
		/* Match platform for ACPI boot */
		if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
			return qcom_smmu_create(smmu, &qcom_smmu_impl);
	}

	if (of_match_node(qcom_smmu_impl_of_match, np))
		return qcom_smmu_create(smmu, &qcom_smmu_impl);

	if (of_device_is_compatible(np, "qcom,adreno-smmu"))
		return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);

	return smmu;
}
Commit	Line	Data
759aaa10 VG	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (c) 2019, The Linux Foundation. All rights reserved.
	4	*/
	5
a51627c5	6	#include <linux/acpi.h>
5c7469c6	7	#include <linux/adreno-smmu-priv.h>
0e764a01	8	#include <linux/of_device.h>
759aaa10 VG	9	#include <linux/qcom_scm.h>
	10
	11	#include "arm-smmu.h"
	12
	13	struct qcom_smmu {
	14	struct arm_smmu_device smmu;
f9081b8f BA	15	bool bypass_quirk;
f9081b8f BA	16	u8 bypass_cbndx;
759aaa10 VG	17	};
759aaa10 VG	18
f9081b8f BA	19	static struct qcom_smmu to_qcom_smmu(struct arm_smmu_device smmu)
	20	{
	21	return container_of(smmu, struct qcom_smmu, smmu);
	22	}
	23
bffb2eaf RC	24	static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
	25	u32 reg)
	26	{
	27	/*
	28	* On the GPU device we want to process subsequent transactions after a
	29	* fault to keep the GPU from hanging
	30	*/
	31	reg \|= ARM_SMMU_SCTLR_HUPCF;
	32
	33	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
	34	}
	35
5c7469c6 JC	36	#define QCOM_ADRENO_SMMU_GPU_SID 0
	37
	38	static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
	39	{
	40	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
	41	int i;
	42
	43	/*
	44	* The GPU will always use SID 0 so that is a handy way to uniquely
	45	* identify it and configure it for per-instance pagetables
	46	*/
	47	for (i = 0; i < fwspec->num_ids; i++) {
	48	u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
	49
	50	if (sid == QCOM_ADRENO_SMMU_GPU_SID)
	51	return true;
	52	}
	53
	54	return false;
	55	}
	56
	57	static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
	58	const void *cookie)
	59	{
	60	struct arm_smmu_domain smmu_domain = (void )cookie;
	61	struct io_pgtable *pgtable =
	62	io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
	63	return &pgtable->cfg;
	64	}
	65
	66	/*
	67	* Local implementation to configure TTBR0 with the specified pagetable config.
	68	* The GPU driver will call this to enable TTBR0 when per-instance pagetables
	69	* are active
	70	*/
	71
	72	static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
	73	const struct io_pgtable_cfg *pgtbl_cfg)
	74	{
	75	struct arm_smmu_domain smmu_domain = (void )cookie;
	76	struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
	77	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
	78	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
	79
	80	/* The domain must have split pagetables already enabled */
	81	if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
	82	return -EINVAL;
	83
	84	/* If the pagetable config is NULL, disable TTBR0 */
	85	if (!pgtbl_cfg) {
	86	/* Do nothing if it is already disabled */
	87	if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
	88	return -EINVAL;
	89
	90	/* Set TCR to the original configuration */
	91	cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
	92	cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
	93	} else {
	94	u32 tcr = cb->tcr[0];
	95
	96	/* Don't call this again if TTBR0 is already enabled */
	97	if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
	98	return -EINVAL;
	99
100	tcr \|= arm_smmu_lpae_tcr(pgtbl_cfg);
101	tcr &= ~(ARM_SMMU_TCR_EPD0 \| ARM_SMMU_TCR_EPD1);
102
103	cb->tcr[0] = tcr;
104	cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
105	cb->ttbr[0] \|= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
106	}
107
108	arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
109
110	return 0;
111	}
112
113	static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
114	struct arm_smmu_device *smmu,
115	struct device *dev, int start)
116	{
117	int count;
118
119	/*
120	* Assign context bank 0 to the GPU device so the GPU hardware can
121	* switch pagetables
122	*/
123	if (qcom_adreno_smmu_is_gpu_device(dev)) {
124	start = 0;
125	count = 1;
126	} else {
127	start = 1;
128	count = smmu->num_context_banks;
129	}
130
131	return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
132	}
133
134	static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
135	struct io_pgtable_cfg pgtbl_cfg, struct device dev)
136	{
137	struct adreno_smmu_priv *priv;
138
139	/* Only enable split pagetables for the GPU device (SID 0) */
140	if (!qcom_adreno_smmu_is_gpu_device(dev))
141	return 0;
142
143	/*
144	* All targets that use the qcom,adreno-smmu compatible string should
145	* be AARCH64 stage 1 but double check because the arm-smmu code assumes
146	* that is the case when the TTBR1 quirk is enabled
147	*/
148	if ((smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
149	(smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
150	pgtbl_cfg->quirks \|= IO_PGTABLE_QUIRK_ARM_TTBR1;
151
152	/*
153	* Initialize private interface with GPU:
154	*/
155
156	priv = dev_get_drvdata(dev);
157	priv->cookie = smmu_domain;
158	priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
159	priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
160
161	return 0;
162	}
163
a082121b	164	static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
0e764a01 JC	165	{ .compatible = "qcom,adreno" },
	166	{ .compatible = "qcom,mdp4" },
	167	{ .compatible = "qcom,mdss" },
	168	{ .compatible = "qcom,sc7180-mdss" },
d100ff38	169	{ .compatible = "qcom,sc7180-mss-pil" },
1a7180ff	170	{ .compatible = "qcom,sc8180x-mdss" },
0e764a01	171	{ .compatible = "qcom,sdm845-mdss" },
d100ff38	172	{ .compatible = "qcom,sdm845-mss-pil" },
0e764a01 JC	173	{ }
	174	};
	175
07a7f2ca BA	176	static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
07a7f2ca BA	177	{
f9081b8f BA	178	unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
	179	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
	180	u32 reg;
07a7f2ca BA	181	u32 smr;
	182	int i;
	183
f9081b8f BA	184	/*
	185	* With some firmware versions writes to S2CR of type FAULT are
	186	* ignored, and writing BYPASS will end up written as FAULT in the
	187	* register. Perform a write to S2CR to detect if this is the case and
	188	* if so reserve a context bank to emulate bypass streams.
	189	*/
	190	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) \|
	191	FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) \|
	192	FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
	193	arm_smmu_gr0_write(smmu, last_s2cr, reg);
	194	reg = arm_smmu_gr0_read(smmu, last_s2cr);
	195	if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
	196	qsmmu->bypass_quirk = true;
	197	qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
	198
	199	set_bit(qsmmu->bypass_cbndx, smmu->context_map);
	200
aded8c7c BA	201	arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
aded8c7c BA	202
f9081b8f BA	203	reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
	204	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
	205	}
	206
07a7f2ca BA	207	for (i = 0; i < smmu->num_mapping_groups; i++) {
	208	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
	209
	210	if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
dead723e IM	211	/* Ignore valid bit for SMR mask extraction. */
dead723e IM	212	smr &= ~ARM_SMMU_SMR_VALID;
07a7f2ca BA	213	smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
	214	smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
	215	smmu->smrs[i].valid = true;
	216
	217	smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
	218	smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
	219	smmu->s2crs[i].cbndx = 0xff;
	220	}
	221	}
	222
	223	return 0;
	224	}
	225
f9081b8f BA	226	static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
	227	{
	228	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
	229	struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
	230	u32 cbndx = s2cr->cbndx;
	231	u32 type = s2cr->type;
	232	u32 reg;
	233
	234	if (qsmmu->bypass_quirk) {
	235	if (type == S2CR_TYPE_BYPASS) {
	236	/*
	237	* Firmware with quirky S2CR handling will substitute
	238	* BYPASS writes with FAULT, so point the stream to the
	239	* reserved context bank and ask for translation on the
	240	* stream
	241	*/
	242	type = S2CR_TYPE_TRANS;
	243	cbndx = qsmmu->bypass_cbndx;
	244	} else if (type == S2CR_TYPE_FAULT) {
	245	/*
	246	* Firmware with quirky S2CR handling will ignore FAULT
	247	* writes, so trick it to write FAULT by asking for a
	248	* BYPASS.
	249	*/
	250	type = S2CR_TYPE_BYPASS;
	251	cbndx = 0xff;
	252	}
	253	}
	254
	255	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) \|
	256	FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) \|
	257	FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
	258	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
	259	}
	260
0e764a01 JC	261	static int qcom_smmu_def_domain_type(struct device *dev)
	262	{
	263	const struct of_device_id *match =
	264	of_match_device(qcom_smmu_client_of_match, dev);
	265
	266	return match ? IOMMU_DOMAIN_IDENTITY : 0;
	267	}
	268
759aaa10 VG	269	static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
	270	{
	271	int ret;
	272
759aaa10 VG	273	/*
	274	* To address performance degradation in non-real time clients,
	275	* such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
	276	* such as MTP and db845, whose firmwares implement secure monitor
	277	* call handlers to turn on/off the wait-for-safe logic.
	278	*/
	279	ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
	280	if (ret)
	281	dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
	282
	283	return ret;
	284	}
	285
64510ede SPR	286	static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
	287	{
	288	const struct device_node *np = smmu->dev->of_node;
	289
	290	arm_mmu500_reset(smmu);
	291
	292	if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
	293	return qcom_sdm845_smmu500_reset(smmu);
	294
	295	return 0;
	296	}
	297
759aaa10	298	static const struct arm_smmu_impl qcom_smmu_impl = {
07a7f2ca	299	.cfg_probe = qcom_smmu_cfg_probe,
0e764a01	300	.def_domain_type = qcom_smmu_def_domain_type,
64510ede	301	.reset = qcom_smmu500_reset,
f9081b8f	302	.write_s2cr = qcom_smmu_write_s2cr,
759aaa10 VG	303	};
759aaa10 VG	304
5c7469c6 JC	305	static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
	306	.init_context = qcom_adreno_smmu_init_context,
	307	.def_domain_type = qcom_smmu_def_domain_type,
	308	.reset = qcom_smmu500_reset,
	309	.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
bffb2eaf	310	.write_sctlr = qcom_adreno_smmu_write_sctlr,
5c7469c6 JC	311	};
	312
	313	static struct arm_smmu_device qcom_smmu_create(struct arm_smmu_device smmu,
	314	const struct arm_smmu_impl *impl)
759aaa10 VG	315	{
	316	struct qcom_smmu *qsmmu;
	317
72b55c96 JS	318	/* Check to make sure qcom_scm has finished probing */
	319	if (!qcom_scm_is_available())
	320	return ERR_PTR(-EPROBE_DEFER);
	321
af9da914	322	qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
759aaa10 VG	323	if (!qsmmu)
	324	return ERR_PTR(-ENOMEM);
	325
5c7469c6	326	qsmmu->smmu.impl = impl;
759aaa10 VG	327
	328	return &qsmmu->smmu;
	329	}
5c7469c6	330
00597f9f	331	static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
b812834b	332	{ .compatible = "qcom,msm8998-smmu-v2" },
00597f9f	333	{ .compatible = "qcom,sc7180-smmu-500" },
1a7180ff	334	{ .compatible = "qcom,sc8180x-smmu-500" },
b812834b	335	{ .compatible = "qcom,sdm630-smmu-v2" },
00597f9f	336	{ .compatible = "qcom,sdm845-smmu-500" },
6321484d	337	{ .compatible = "qcom,sm6125-smmu-500" },
00597f9f SPR	338	{ .compatible = "qcom,sm8150-smmu-500" },
00597f9f SPR	339	{ .compatible = "qcom,sm8250-smmu-500" },
d8498b1e	340	{ .compatible = "qcom,sm8350-smmu-500" },
00597f9f SPR	341	{ }
	342	};
	343
a51627c5 SG	344	static struct acpi_platform_list qcom_acpi_platlist[] = {
	345	{ "LENOVO", "CB-01 ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
	346	{ "QCOM ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
	347	{ }
	348	};
	349
5c7469c6 JC	350	struct arm_smmu_device qcom_smmu_impl_init(struct arm_smmu_device smmu)
5c7469c6 JC	351	{
00597f9f	352	const struct device_node *np = smmu->dev->of_node;
5c7469c6	353
a51627c5 SG	354	if (np == NULL) {
	355	/* Match platform for ACPI boot */
	356	if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
	357	return qcom_smmu_create(smmu, &qcom_smmu_impl);
	358	}
	359
00597f9f SPR	360	if (of_match_node(qcom_smmu_impl_of_match, np))
	361	return qcom_smmu_create(smmu, &qcom_smmu_impl);
	362
	363	if (of_device_is_compatible(np, "qcom,adreno-smmu"))
	364	return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
	365
	366	return smmu;
5c7469c6	367	}