1 // SPDX-License-Identifier: GPL-2.0-only
3 * IOMMU API for ARM architected SMMU implementations.
5 * Copyright (C) 2013 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
18 #define pr_fmt(fmt) "arm-smmu: " fmt
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
38 #include <linux/fsl/mc.h>
41 #include "../../dma-iommu.h"
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
50 #define QCOM_DUMMY_VAL -1
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
69 static bool using_legacy_binding, using_generic_binding;
71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
85 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
88 * Setup an autosuspend delay to avoid bouncing runpm state.
89 * Otherwise, if a driver for a suspended consumer device
90 * unmaps buffers, it will runpm resume/suspend for each one.
92 * For example, when used by a GPU device, when an application
93 * or game exits, it can trigger unmapping 100s or 1000s of
94 * buffers. With a runpm cycle for each buffer, that adds up
95 * to 5-10sec worth of reprogramming the context bank, while
96 * the system appears to be locked up to the user.
98 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
99 pm_runtime_use_autosuspend(smmu->dev);
102 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
104 return container_of(dom, struct arm_smmu_domain, domain);
107 static struct platform_driver arm_smmu_driver;
108 static struct iommu_ops arm_smmu_ops;
110 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
111 static struct device_node *dev_get_dev_node(struct device *dev)
113 if (dev_is_pci(dev)) {
114 struct pci_bus *bus = to_pci_dev(dev)->bus;
116 while (!pci_is_root_bus(bus))
118 return of_node_get(bus->bridge->parent->of_node);
121 return of_node_get(dev->of_node);
124 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
126 *((__be32 *)data) = cpu_to_be32(alias);
127 return 0; /* Continue walking */
130 static int __find_legacy_master_phandle(struct device *dev, void *data)
132 struct of_phandle_iterator *it = *(void **)data;
133 struct device_node *np = it->node;
136 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
137 "#stream-id-cells", -1)
138 if (it->node == np) {
139 *(void **)data = dev;
143 return err == -ENOENT ? 0 : err;
146 static int arm_smmu_register_legacy_master(struct device *dev,
147 struct arm_smmu_device **smmu)
149 struct device *smmu_dev;
150 struct device_node *np;
151 struct of_phandle_iterator it;
157 np = dev_get_dev_node(dev);
158 if (!np || !of_property_present(np, "#stream-id-cells")) {
164 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
165 __find_legacy_master_phandle);
173 if (dev_is_pci(dev)) {
174 /* "mmu-masters" assumes Stream ID == Requester ID */
175 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
181 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
186 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
190 *smmu = dev_get_drvdata(smmu_dev);
191 of_phandle_iterator_args(&it, sids, it.cur_count);
192 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
197 static int arm_smmu_register_legacy_master(struct device *dev,
198 struct arm_smmu_device **smmu)
202 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
204 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
209 /* Wait for any pending TLB invalidations to complete */
210 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
211 int sync, int status)
213 unsigned int spin_cnt, delay;
216 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
217 return smmu->impl->tlb_sync(smmu, page, sync, status);
219 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
220 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
221 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
222 reg = arm_smmu_readl(smmu, page, status);
223 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
229 dev_err_ratelimited(smmu->dev,
230 "TLB sync timed out -- SMMU may be deadlocked\n");
233 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
237 spin_lock_irqsave(&smmu->global_sync_lock, flags);
238 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
239 ARM_SMMU_GR0_sTLBGSTATUS);
240 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
243 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
245 struct arm_smmu_device *smmu = smmu_domain->smmu;
248 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
249 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
250 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
251 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
254 static void arm_smmu_tlb_inv_context_s1(void *cookie)
256 struct arm_smmu_domain *smmu_domain = cookie;
258 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
259 * current CPU are visible beforehand.
262 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
263 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
264 arm_smmu_tlb_sync_context(smmu_domain);
267 static void arm_smmu_tlb_inv_context_s2(void *cookie)
269 struct arm_smmu_domain *smmu_domain = cookie;
270 struct arm_smmu_device *smmu = smmu_domain->smmu;
274 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
275 arm_smmu_tlb_sync_global(smmu);
278 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
279 size_t granule, void *cookie, int reg)
281 struct arm_smmu_domain *smmu_domain = cookie;
282 struct arm_smmu_device *smmu = smmu_domain->smmu;
283 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
284 int idx = cfg->cbndx;
286 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
289 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
290 iova = (iova >> 12) << 12;
293 arm_smmu_cb_write(smmu, idx, reg, iova);
295 } while (size -= granule);
298 iova |= (u64)cfg->asid << 48;
300 arm_smmu_cb_writeq(smmu, idx, reg, iova);
301 iova += granule >> 12;
302 } while (size -= granule);
306 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
307 size_t granule, void *cookie, int reg)
309 struct arm_smmu_domain *smmu_domain = cookie;
310 struct arm_smmu_device *smmu = smmu_domain->smmu;
311 int idx = smmu_domain->cfg.cbndx;
313 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
318 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
319 arm_smmu_cb_writeq(smmu, idx, reg, iova);
321 arm_smmu_cb_write(smmu, idx, reg, iova);
322 iova += granule >> 12;
323 } while (size -= granule);
326 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
327 size_t granule, void *cookie)
329 struct arm_smmu_domain *smmu_domain = cookie;
330 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
332 if (cfg->flush_walk_prefer_tlbiasid) {
333 arm_smmu_tlb_inv_context_s1(cookie);
335 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
336 ARM_SMMU_CB_S1_TLBIVA);
337 arm_smmu_tlb_sync_context(cookie);
341 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
342 unsigned long iova, size_t granule,
345 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
346 ARM_SMMU_CB_S1_TLBIVAL);
349 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
350 size_t granule, void *cookie)
352 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
353 ARM_SMMU_CB_S2_TLBIIPAS2);
354 arm_smmu_tlb_sync_context(cookie);
357 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
358 unsigned long iova, size_t granule,
361 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
362 ARM_SMMU_CB_S2_TLBIIPAS2L);
365 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
366 size_t granule, void *cookie)
368 arm_smmu_tlb_inv_context_s2(cookie);
371 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
372 * almost negligible, but the benefit of getting the first one in as far ahead
373 * of the sync as possible is significant, hence we don't just make this a
374 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
377 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
378 unsigned long iova, size_t granule,
381 struct arm_smmu_domain *smmu_domain = cookie;
382 struct arm_smmu_device *smmu = smmu_domain->smmu;
384 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
387 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
390 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
391 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
392 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
393 .tlb_add_page = arm_smmu_tlb_add_page_s1,
396 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
397 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
398 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
399 .tlb_add_page = arm_smmu_tlb_add_page_s2,
402 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
403 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
404 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
405 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
408 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
410 u32 fsr, fsynr, cbfrsynra;
412 struct arm_smmu_domain *smmu_domain = dev;
413 struct arm_smmu_device *smmu = smmu_domain->smmu;
414 int idx = smmu_domain->cfg.cbndx;
417 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
418 if (!(fsr & ARM_SMMU_FSR_FAULT))
421 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
422 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
423 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
425 ret = report_iommu_fault(&smmu_domain->domain, NULL, iova,
426 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
429 dev_err_ratelimited(smmu->dev,
430 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
431 fsr, iova, fsynr, cbfrsynra, idx);
433 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
437 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
439 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
440 struct arm_smmu_device *smmu = dev;
441 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
442 DEFAULT_RATELIMIT_BURST);
444 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
445 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
446 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
447 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
452 if (__ratelimit(&rs)) {
453 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
454 (gfsr & ARM_SMMU_sGFSR_USF))
456 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
460 "Unexpected global fault, this could be serious\n");
462 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
463 gfsr, gfsynr0, gfsynr1, gfsynr2);
466 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
470 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
471 struct io_pgtable_cfg *pgtbl_cfg)
473 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
474 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
475 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
484 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
485 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
486 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
487 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
489 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
492 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
497 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
498 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
501 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
503 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
506 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
507 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
509 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
512 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
515 /* MAIRs (stage-1 only) */
517 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
518 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
519 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
521 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
522 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
527 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
531 struct arm_smmu_cb *cb = &smmu->cbs[idx];
532 struct arm_smmu_cfg *cfg = cb->cfg;
534 /* Unassigned context banks only need disabling */
536 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
540 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
543 if (smmu->version > ARM_SMMU_V1) {
544 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
545 reg = ARM_SMMU_CBA2R_VA64;
548 /* 16-bit VMIDs live in CBA2R */
549 if (smmu->features & ARM_SMMU_FEAT_VMID16)
550 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
552 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
556 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
557 if (smmu->version < ARM_SMMU_V2)
558 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
561 * Use the weakest shareability/memory types, so they are
562 * overridden by the ttbcr/pte.
565 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
566 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
567 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
568 ARM_SMMU_CBAR_S1_MEMATTR_WB);
569 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
570 /* 8-bit VMIDs live in CBAR */
571 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
573 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
577 * We must write this before the TTBRs, since it determines the
578 * access behaviour of some fields (in particular, ASID[15:8]).
580 if (stage1 && smmu->version > ARM_SMMU_V1)
581 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
585 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
586 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
587 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
588 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
590 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
592 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
596 /* MAIRs (stage-1 only) */
598 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
599 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
603 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
604 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
606 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
607 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
608 reg |= ARM_SMMU_SCTLR_E;
610 if (smmu->impl && smmu->impl->write_sctlr)
611 smmu->impl->write_sctlr(smmu, idx, reg);
613 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
616 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
617 struct arm_smmu_device *smmu,
618 struct device *dev, unsigned int start)
620 if (smmu->impl && smmu->impl->alloc_context_bank)
621 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
623 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
626 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
627 struct arm_smmu_device *smmu,
630 int irq, start, ret = 0;
631 unsigned long ias, oas;
632 struct io_pgtable_ops *pgtbl_ops;
633 struct io_pgtable_cfg pgtbl_cfg;
634 enum io_pgtable_fmt fmt;
635 struct iommu_domain *domain = &smmu_domain->domain;
636 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
637 irqreturn_t (*context_fault)(int irq, void *dev);
639 mutex_lock(&smmu_domain->init_mutex);
640 if (smmu_domain->smmu)
644 * Mapping the requested stage onto what we support is surprisingly
645 * complicated, mainly because the spec allows S1+S2 SMMUs without
646 * support for nested translation. That means we end up with the
649 * Requested Supported Actual
659 * Note that you can't actually request stage-2 mappings.
661 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
662 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
663 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
664 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
667 * Choosing a suitable context format is even more fiddly. Until we
668 * grow some way for the caller to express a preference, and/or move
669 * the decision into the io-pgtable code where it arguably belongs,
670 * just aim for the closest thing to the rest of the system, and hope
671 * that the hardware isn't esoteric enough that we can't assume AArch64
672 * support to be a superset of AArch32 support...
674 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
676 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
677 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
678 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
679 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
680 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
681 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
682 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
683 ARM_SMMU_FEAT_FMT_AARCH64_16K |
684 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
685 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
687 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
692 switch (smmu_domain->stage) {
693 case ARM_SMMU_DOMAIN_S1:
694 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
695 start = smmu->num_s2_context_banks;
697 oas = smmu->ipa_size;
698 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
699 fmt = ARM_64_LPAE_S1;
700 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
701 fmt = ARM_32_LPAE_S1;
702 ias = min(ias, 32UL);
703 oas = min(oas, 40UL);
706 ias = min(ias, 32UL);
707 oas = min(oas, 32UL);
709 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
711 case ARM_SMMU_DOMAIN_NESTED:
713 * We will likely want to change this if/when KVM gets
716 case ARM_SMMU_DOMAIN_S2:
717 cfg->cbar = CBAR_TYPE_S2_TRANS;
719 ias = smmu->ipa_size;
721 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
722 fmt = ARM_64_LPAE_S2;
724 fmt = ARM_32_LPAE_S2;
725 ias = min(ias, 40UL);
726 oas = min(oas, 40UL);
728 if (smmu->version == ARM_SMMU_V2)
729 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
731 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
738 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
743 smmu_domain->smmu = smmu;
746 if (smmu->version < ARM_SMMU_V2) {
747 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
748 cfg->irptndx %= smmu->num_context_irqs;
750 cfg->irptndx = cfg->cbndx;
753 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
754 cfg->vmid = cfg->cbndx + 1;
756 cfg->asid = cfg->cbndx;
758 pgtbl_cfg = (struct io_pgtable_cfg) {
759 .pgsize_bitmap = smmu->pgsize_bitmap,
762 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
763 .tlb = smmu_domain->flush_ops,
764 .iommu_dev = smmu->dev,
767 if (smmu->impl && smmu->impl->init_context) {
768 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
773 if (smmu_domain->pgtbl_quirks)
774 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
776 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
782 /* Update the domain's page sizes to reflect the page table format */
783 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
785 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
786 domain->geometry.aperture_start = ~0UL << ias;
787 domain->geometry.aperture_end = ~0UL;
789 domain->geometry.aperture_end = (1UL << ias) - 1;
792 domain->geometry.force_aperture = true;
794 /* Initialise the context bank with our page table cfg */
795 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
796 arm_smmu_write_context_bank(smmu, cfg->cbndx);
799 * Request context fault interrupt. Do this last to avoid the
800 * handler seeing a half-initialised domain state.
802 irq = smmu->irqs[cfg->irptndx];
804 if (smmu->impl && smmu->impl->context_fault)
805 context_fault = smmu->impl->context_fault;
807 context_fault = arm_smmu_context_fault;
809 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
810 "arm-smmu-context-fault", smmu_domain);
812 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
814 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
817 mutex_unlock(&smmu_domain->init_mutex);
819 /* Publish page table ops for map/unmap */
820 smmu_domain->pgtbl_ops = pgtbl_ops;
824 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
825 smmu_domain->smmu = NULL;
827 mutex_unlock(&smmu_domain->init_mutex);
831 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
833 struct arm_smmu_device *smmu = smmu_domain->smmu;
834 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
840 ret = arm_smmu_rpm_get(smmu);
845 * Disable the context bank and free the page tables before freeing
848 smmu->cbs[cfg->cbndx].cfg = NULL;
849 arm_smmu_write_context_bank(smmu, cfg->cbndx);
851 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
852 irq = smmu->irqs[cfg->irptndx];
853 devm_free_irq(smmu->dev, irq, smmu_domain);
856 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
857 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
859 arm_smmu_rpm_put(smmu);
862 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
864 struct arm_smmu_domain *smmu_domain;
867 * Allocate the domain and initialise some of its data structures.
868 * We can't really do anything meaningful until we've added a
871 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
875 mutex_init(&smmu_domain->init_mutex);
876 spin_lock_init(&smmu_domain->cb_lock);
879 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
881 if (arm_smmu_init_domain_context(smmu_domain, cfg->smmu, dev)) {
887 return &smmu_domain->domain;
890 static void arm_smmu_domain_free(struct iommu_domain *domain)
892 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
895 * Free the domain resources. We assume that all devices have
896 * already been detached.
898 arm_smmu_destroy_domain_context(smmu_domain);
902 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
904 struct arm_smmu_smr *smr = smmu->smrs + idx;
905 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
906 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
908 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
909 reg |= ARM_SMMU_SMR_VALID;
910 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
913 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
915 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
918 if (smmu->impl && smmu->impl->write_s2cr) {
919 smmu->impl->write_s2cr(smmu, idx);
923 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
924 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
925 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
927 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
928 smmu->smrs[idx].valid)
929 reg |= ARM_SMMU_S2CR_EXIDVALID;
930 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
933 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
935 arm_smmu_write_s2cr(smmu, idx);
937 arm_smmu_write_smr(smmu, idx);
941 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
942 * should be called after sCR0 is written.
944 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
952 * If we've had to accommodate firmware memory regions, we may
953 * have live SMRs by now; tread carefully...
955 * Somewhat perversely, not having a free SMR for this test implies we
956 * can get away without it anyway, as we'll only be able to 'allocate'
957 * these SMRs for the ID/mask values we're already trusting to be OK.
959 for (i = 0; i < smmu->num_mapping_groups; i++)
960 if (!smmu->smrs[i].valid)
965 * SMR.ID bits may not be preserved if the corresponding MASK
966 * bits are set, so check each one separately. We can reject
967 * masters later if they try to claim IDs outside these masks.
969 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
970 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
971 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
972 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
974 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
975 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
976 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
977 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
980 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
982 struct arm_smmu_smr *smrs = smmu->smrs;
983 int i, free_idx = -ENOSPC;
985 /* Stream indexing is blissfully easy */
989 /* Validating SMRs is... less so */
990 for (i = 0; i < smmu->num_mapping_groups; ++i) {
991 if (!smrs[i].valid) {
993 * Note the first free entry we come across, which
994 * we'll claim in the end if nothing else matches.
1001 * If the new entry is _entirely_ matched by an existing entry,
1002 * then reuse that, with the guarantee that there also cannot
1003 * be any subsequent conflicting entries. In normal use we'd
1004 * expect simply identical entries for this case, but there's
1005 * no harm in accommodating the generalisation.
1007 if ((mask & smrs[i].mask) == mask &&
1008 !((id ^ smrs[i].id) & ~smrs[i].mask))
1011 * If the new entry has any other overlap with an existing one,
1012 * though, then there always exists at least one stream ID
1013 * which would cause a conflict, and we can't allow that risk.
1015 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1022 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1024 if (--smmu->s2crs[idx].count)
1027 smmu->s2crs[idx] = s2cr_init_val;
1029 smmu->smrs[idx].valid = false;
1034 static int arm_smmu_master_alloc_smes(struct device *dev)
1036 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1037 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1038 struct arm_smmu_device *smmu = cfg->smmu;
1039 struct arm_smmu_smr *smrs = smmu->smrs;
1042 mutex_lock(&smmu->stream_map_mutex);
1043 /* Figure out a viable stream map entry allocation */
1044 for_each_cfg_sme(cfg, fwspec, i, idx) {
1045 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1046 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1048 if (idx != INVALID_SMENDX) {
1053 ret = arm_smmu_find_sme(smmu, sid, mask);
1058 if (smrs && smmu->s2crs[idx].count == 0) {
1060 smrs[idx].mask = mask;
1061 smrs[idx].valid = true;
1063 smmu->s2crs[idx].count++;
1064 cfg->smendx[i] = (s16)idx;
1067 /* It worked! Now, poke the actual hardware */
1068 for_each_cfg_sme(cfg, fwspec, i, idx)
1069 arm_smmu_write_sme(smmu, idx);
1071 mutex_unlock(&smmu->stream_map_mutex);
1076 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1077 cfg->smendx[i] = INVALID_SMENDX;
1079 mutex_unlock(&smmu->stream_map_mutex);
1083 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1084 struct iommu_fwspec *fwspec)
1086 struct arm_smmu_device *smmu = cfg->smmu;
1089 mutex_lock(&smmu->stream_map_mutex);
1090 for_each_cfg_sme(cfg, fwspec, i, idx) {
1091 if (arm_smmu_free_sme(smmu, idx))
1092 arm_smmu_write_sme(smmu, idx);
1093 cfg->smendx[i] = INVALID_SMENDX;
1095 mutex_unlock(&smmu->stream_map_mutex);
1098 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1099 enum arm_smmu_s2cr_type type,
1100 u8 cbndx, struct iommu_fwspec *fwspec)
1102 struct arm_smmu_device *smmu = cfg->smmu;
1103 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1106 for_each_cfg_sme(cfg, fwspec, i, idx) {
1107 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1110 s2cr[idx].type = type;
1111 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1112 s2cr[idx].cbndx = cbndx;
1113 arm_smmu_write_s2cr(smmu, idx);
1117 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1119 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1120 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1121 struct arm_smmu_master_cfg *cfg;
1122 struct arm_smmu_device *smmu;
1126 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1127 * domains between of_xlate() and probe_device() - we have no way to cope
1128 * with that, so until ARM gets converted to rely on groups and default
1129 * domains, just say no (but more politely than by dereferencing NULL).
1130 * This should be at least a WARN_ON once that's sorted.
1132 cfg = dev_iommu_priv_get(dev);
1138 ret = arm_smmu_rpm_get(smmu);
1142 /* Ensure that the domain is finalised */
1143 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1148 * Sanity check the domain. We don't support domains across
1151 if (smmu_domain->smmu != smmu) {
1156 /* Looks ok, so add the device to the domain */
1157 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1158 smmu_domain->cfg.cbndx, fwspec);
1159 arm_smmu_rpm_use_autosuspend(smmu);
1161 arm_smmu_rpm_put(smmu);
1165 static int arm_smmu_attach_dev_type(struct device *dev,
1166 enum arm_smmu_s2cr_type type)
1168 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1169 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1170 struct arm_smmu_device *smmu;
1177 ret = arm_smmu_rpm_get(smmu);
1181 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1182 arm_smmu_rpm_use_autosuspend(smmu);
1183 arm_smmu_rpm_put(smmu);
1187 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1190 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1193 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1194 .attach_dev = arm_smmu_attach_dev_identity,
1197 static struct iommu_domain arm_smmu_identity_domain = {
1198 .type = IOMMU_DOMAIN_IDENTITY,
1199 .ops = &arm_smmu_identity_ops,
1202 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1205 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1208 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1209 .attach_dev = arm_smmu_attach_dev_blocked,
1212 static struct iommu_domain arm_smmu_blocked_domain = {
1213 .type = IOMMU_DOMAIN_BLOCKED,
1214 .ops = &arm_smmu_blocked_ops,
1217 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1218 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1219 int prot, gfp_t gfp, size_t *mapped)
1221 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1222 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1228 arm_smmu_rpm_get(smmu);
1229 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1230 arm_smmu_rpm_put(smmu);
1235 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1236 size_t pgsize, size_t pgcount,
1237 struct iommu_iotlb_gather *iotlb_gather)
1239 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1240 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1246 arm_smmu_rpm_get(smmu);
1247 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1248 arm_smmu_rpm_put(smmu);
1253 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1255 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1256 struct arm_smmu_device *smmu = smmu_domain->smmu;
1258 if (smmu_domain->flush_ops) {
1259 arm_smmu_rpm_get(smmu);
1260 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1261 arm_smmu_rpm_put(smmu);
1265 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1266 struct iommu_iotlb_gather *gather)
1268 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1269 struct arm_smmu_device *smmu = smmu_domain->smmu;
1274 arm_smmu_rpm_get(smmu);
1275 if (smmu->version == ARM_SMMU_V2 ||
1276 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1277 arm_smmu_tlb_sync_context(smmu_domain);
1279 arm_smmu_tlb_sync_global(smmu);
1280 arm_smmu_rpm_put(smmu);
1283 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1286 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1287 struct arm_smmu_device *smmu = smmu_domain->smmu;
1288 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1289 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1290 struct device *dev = smmu->dev;
1294 unsigned long va, flags;
1295 int ret, idx = cfg->cbndx;
1296 phys_addr_t addr = 0;
1298 ret = arm_smmu_rpm_get(smmu);
1302 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1303 va = iova & ~0xfffUL;
1304 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1305 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1307 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1309 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1310 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1312 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1314 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1316 arm_smmu_rpm_put(smmu);
1317 return ops->iova_to_phys(ops, iova);
1320 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1321 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1322 if (phys & ARM_SMMU_CB_PAR_F) {
1323 dev_err(dev, "translation fault!\n");
1324 dev_err(dev, "PAR = 0x%llx\n", phys);
1328 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1330 arm_smmu_rpm_put(smmu);
1335 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1338 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1339 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1344 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1345 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1346 return arm_smmu_iova_to_phys_hard(domain, iova);
1348 return ops->iova_to_phys(ops, iova);
1351 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1353 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1356 case IOMMU_CAP_CACHE_COHERENCY:
1358 * It's overwhelmingly the case in practice that when the pagetable
1359 * walk interface is connected to a coherent interconnect, all the
1360 * translation interfaces are too. Furthermore if the device is
1361 * natively coherent, then its translation interface must also be.
1363 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1364 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1365 case IOMMU_CAP_NOEXEC:
1366 case IOMMU_CAP_DEFERRED_FLUSH:
1374 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1376 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1379 return dev ? dev_get_drvdata(dev) : NULL;
1382 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1384 struct arm_smmu_device *smmu = NULL;
1385 struct arm_smmu_master_cfg *cfg;
1386 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1389 if (using_legacy_binding) {
1390 ret = arm_smmu_register_legacy_master(dev, &smmu);
1393 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1394 * will allocate/initialise a new one. Thus we need to update fwspec for
1397 fwspec = dev_iommu_fwspec_get(dev);
1401 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1405 for (i = 0; i < fwspec->num_ids; i++) {
1406 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1407 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1409 if (sid & ~smmu->streamid_mask) {
1410 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1411 sid, smmu->streamid_mask);
1414 if (mask & ~smmu->smr_mask_mask) {
1415 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1416 mask, smmu->smr_mask_mask);
1422 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1428 dev_iommu_priv_set(dev, cfg);
1430 cfg->smendx[i] = INVALID_SMENDX;
1432 ret = arm_smmu_rpm_get(smmu);
1436 ret = arm_smmu_master_alloc_smes(dev);
1437 arm_smmu_rpm_put(smmu);
1442 device_link_add(dev, smmu->dev,
1443 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1445 return &smmu->iommu;
1450 iommu_fwspec_free(dev);
1451 return ERR_PTR(ret);
1454 static void arm_smmu_release_device(struct device *dev)
1456 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1457 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1460 ret = arm_smmu_rpm_get(cfg->smmu);
1464 arm_smmu_master_free_smes(cfg, fwspec);
1466 arm_smmu_rpm_put(cfg->smmu);
1471 static void arm_smmu_probe_finalize(struct device *dev)
1473 struct arm_smmu_master_cfg *cfg;
1474 struct arm_smmu_device *smmu;
1476 cfg = dev_iommu_priv_get(dev);
1479 if (smmu->impl && smmu->impl->probe_finalize)
1480 smmu->impl->probe_finalize(smmu, dev);
1483 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1485 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1486 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1487 struct arm_smmu_device *smmu = cfg->smmu;
1488 struct iommu_group *group = NULL;
1491 mutex_lock(&smmu->stream_map_mutex);
1492 for_each_cfg_sme(cfg, fwspec, i, idx) {
1493 if (group && smmu->s2crs[idx].group &&
1494 group != smmu->s2crs[idx].group) {
1495 mutex_unlock(&smmu->stream_map_mutex);
1496 return ERR_PTR(-EINVAL);
1499 group = smmu->s2crs[idx].group;
1503 mutex_unlock(&smmu->stream_map_mutex);
1504 return iommu_group_ref_get(group);
1507 if (dev_is_pci(dev))
1508 group = pci_device_group(dev);
1509 else if (dev_is_fsl_mc(dev))
1510 group = fsl_mc_device_group(dev);
1512 group = generic_device_group(dev);
1514 /* Remember group for faster lookups */
1516 for_each_cfg_sme(cfg, fwspec, i, idx)
1517 smmu->s2crs[idx].group = group;
1519 mutex_unlock(&smmu->stream_map_mutex);
1523 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1525 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1528 mutex_lock(&smmu_domain->init_mutex);
1529 if (smmu_domain->smmu)
1532 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1533 mutex_unlock(&smmu_domain->init_mutex);
1538 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1539 unsigned long quirks)
1541 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1544 mutex_lock(&smmu_domain->init_mutex);
1545 if (smmu_domain->smmu)
1548 smmu_domain->pgtbl_quirks = quirks;
1549 mutex_unlock(&smmu_domain->init_mutex);
1554 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1558 if (args->args_count > 0)
1559 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1561 if (args->args_count > 1)
1562 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1563 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1564 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1566 return iommu_fwspec_add_ids(dev, &fwid, 1);
1569 static void arm_smmu_get_resv_regions(struct device *dev,
1570 struct list_head *head)
1572 struct iommu_resv_region *region;
1573 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1575 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1576 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1580 list_add_tail(®ion->list, head);
1582 iommu_dma_get_resv_regions(dev, head);
1585 static int arm_smmu_def_domain_type(struct device *dev)
1587 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1588 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1590 if (using_legacy_binding)
1591 return IOMMU_DOMAIN_IDENTITY;
1593 if (impl && impl->def_domain_type)
1594 return impl->def_domain_type(dev);
1599 static struct iommu_ops arm_smmu_ops = {
1600 .identity_domain = &arm_smmu_identity_domain,
1601 .blocked_domain = &arm_smmu_blocked_domain,
1602 .capable = arm_smmu_capable,
1603 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1604 .probe_device = arm_smmu_probe_device,
1605 .release_device = arm_smmu_release_device,
1606 .probe_finalize = arm_smmu_probe_finalize,
1607 .device_group = arm_smmu_device_group,
1608 .of_xlate = arm_smmu_of_xlate,
1609 .get_resv_regions = arm_smmu_get_resv_regions,
1610 .def_domain_type = arm_smmu_def_domain_type,
1611 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1612 .owner = THIS_MODULE,
1613 .default_domain_ops = &(const struct iommu_domain_ops) {
1614 .attach_dev = arm_smmu_attach_dev,
1615 .map_pages = arm_smmu_map_pages,
1616 .unmap_pages = arm_smmu_unmap_pages,
1617 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1618 .iotlb_sync = arm_smmu_iotlb_sync,
1619 .iova_to_phys = arm_smmu_iova_to_phys,
1620 .enable_nesting = arm_smmu_enable_nesting,
1621 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1622 .free = arm_smmu_domain_free,
1626 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1631 /* clear global FSR */
1632 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1633 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1636 * Reset stream mapping groups: Initial values mark all SMRn as
1637 * invalid and all S2CRn as bypass unless overridden.
1639 for (i = 0; i < smmu->num_mapping_groups; ++i)
1640 arm_smmu_write_sme(smmu, i);
1642 /* Make sure all context banks are disabled and clear CB_FSR */
1643 for (i = 0; i < smmu->num_context_banks; ++i) {
1644 arm_smmu_write_context_bank(smmu, i);
1645 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1648 /* Invalidate the TLB, just in case */
1649 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1650 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1652 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1654 /* Enable fault reporting */
1655 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1656 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1658 /* Disable TLB broadcasting. */
1659 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1661 /* Enable client access, handling unmatched streams as appropriate */
1662 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1664 reg |= ARM_SMMU_sCR0_USFCFG;
1666 reg &= ~ARM_SMMU_sCR0_USFCFG;
1668 /* Disable forced broadcasting */
1669 reg &= ~ARM_SMMU_sCR0_FB;
1671 /* Don't upgrade barriers */
1672 reg &= ~(ARM_SMMU_sCR0_BSU);
1674 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1675 reg |= ARM_SMMU_sCR0_VMID16EN;
1677 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1678 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1680 if (smmu->impl && smmu->impl->reset)
1681 smmu->impl->reset(smmu);
1683 /* Push the button */
1684 arm_smmu_tlb_sync_global(smmu);
1685 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1688 static int arm_smmu_id_size_to_bits(int size)
1707 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1711 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1714 dev_notice(smmu->dev, "probing hardware configuration...\n");
1715 dev_notice(smmu->dev, "SMMUv%d with:\n",
1716 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1719 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1721 /* Restrict available stages based on module parameter */
1722 if (force_stage == 1)
1723 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1724 else if (force_stage == 2)
1725 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1727 if (id & ARM_SMMU_ID0_S1TS) {
1728 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1729 dev_notice(smmu->dev, "\tstage 1 translation\n");
1732 if (id & ARM_SMMU_ID0_S2TS) {
1733 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1734 dev_notice(smmu->dev, "\tstage 2 translation\n");
1737 if (id & ARM_SMMU_ID0_NTS) {
1738 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1739 dev_notice(smmu->dev, "\tnested translation\n");
1742 if (!(smmu->features &
1743 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1744 dev_err(smmu->dev, "\tno translation support!\n");
1748 if ((id & ARM_SMMU_ID0_S1TS) &&
1749 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1750 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1751 dev_notice(smmu->dev, "\taddress translation ops\n");
1755 * In order for DMA API calls to work properly, we must defer to what
1756 * the FW says about coherency, regardless of what the hardware claims.
1757 * Fortunately, this also opens up a workaround for systems where the
1758 * ID register value has ended up configured incorrectly.
1760 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1761 if (cttw_fw || cttw_reg)
1762 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1763 cttw_fw ? "" : "non-");
1764 if (cttw_fw != cttw_reg)
1765 dev_notice(smmu->dev,
1766 "\t(IDR0.CTTW overridden by FW configuration)\n");
1768 /* Max. number of entries we have for stream matching/indexing */
1769 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1770 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1773 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1775 smmu->streamid_mask = size - 1;
1776 if (id & ARM_SMMU_ID0_SMS) {
1777 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1778 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1781 "stream-matching supported, but no SMRs present!\n");
1785 /* Zero-initialised to mark as invalid */
1786 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1791 dev_notice(smmu->dev,
1792 "\tstream matching with %u register groups", size);
1794 /* s2cr->type == 0 means translation, so initialise explicitly */
1795 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1799 for (i = 0; i < size; i++)
1800 smmu->s2crs[i] = s2cr_init_val;
1802 smmu->num_mapping_groups = size;
1803 mutex_init(&smmu->stream_map_mutex);
1804 spin_lock_init(&smmu->global_sync_lock);
1806 if (smmu->version < ARM_SMMU_V2 ||
1807 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1808 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1809 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1810 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1814 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1815 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1817 /* Check for size mismatch of SMMU address space from mapped region */
1818 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1819 if (smmu->numpage != 2 * size << smmu->pgshift)
1821 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1822 2 * size << smmu->pgshift, smmu->numpage);
1823 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1824 smmu->numpage = size;
1826 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1827 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1828 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1829 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1832 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1833 smmu->num_context_banks, smmu->num_s2_context_banks);
1834 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1835 sizeof(*smmu->cbs), GFP_KERNEL);
1840 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1841 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1842 smmu->ipa_size = size;
1844 /* The output mask is also applied for bypass */
1845 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1846 smmu->pa_size = size;
1848 if (id & ARM_SMMU_ID2_VMID16)
1849 smmu->features |= ARM_SMMU_FEAT_VMID16;
1852 * What the page table walker can address actually depends on which
1853 * descriptor format is in use, but since a) we don't know that yet,
1854 * and b) it can vary per context bank, this will have to do...
1856 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1858 "failed to set DMA mask for table walker\n");
1860 if (smmu->version < ARM_SMMU_V2) {
1861 smmu->va_size = smmu->ipa_size;
1862 if (smmu->version == ARM_SMMU_V1_64K)
1863 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1865 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1866 smmu->va_size = arm_smmu_id_size_to_bits(size);
1867 if (id & ARM_SMMU_ID2_PTFS_4K)
1868 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1869 if (id & ARM_SMMU_ID2_PTFS_16K)
1870 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1871 if (id & ARM_SMMU_ID2_PTFS_64K)
1872 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1875 if (smmu->impl && smmu->impl->cfg_probe) {
1876 ret = smmu->impl->cfg_probe(smmu);
1881 /* Now we've corralled the various formats, what'll it do? */
1882 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1883 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1884 if (smmu->features &
1885 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1886 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1887 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1888 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1889 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1890 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1892 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1893 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1895 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1896 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1897 smmu->pgsize_bitmap);
1900 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1901 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1902 smmu->va_size, smmu->ipa_size);
1904 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1905 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1906 smmu->ipa_size, smmu->pa_size);
1911 struct arm_smmu_match_data {
1912 enum arm_smmu_arch_version version;
1913 enum arm_smmu_implementation model;
1916 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1917 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1919 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1920 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1921 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1922 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1923 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1924 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1926 static const struct of_device_id arm_smmu_of_match[] = {
1927 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1928 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1929 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1930 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1931 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1932 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1933 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1934 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1937 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1940 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1945 case ACPI_IORT_SMMU_V1:
1946 case ACPI_IORT_SMMU_CORELINK_MMU400:
1947 smmu->version = ARM_SMMU_V1;
1948 smmu->model = GENERIC_SMMU;
1950 case ACPI_IORT_SMMU_CORELINK_MMU401:
1951 smmu->version = ARM_SMMU_V1_64K;
1952 smmu->model = GENERIC_SMMU;
1954 case ACPI_IORT_SMMU_V2:
1955 smmu->version = ARM_SMMU_V2;
1956 smmu->model = GENERIC_SMMU;
1958 case ACPI_IORT_SMMU_CORELINK_MMU500:
1959 smmu->version = ARM_SMMU_V2;
1960 smmu->model = ARM_MMU500;
1962 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1963 smmu->version = ARM_SMMU_V2;
1964 smmu->model = CAVIUM_SMMUV2;
1973 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1974 u32 *global_irqs, u32 *pmu_irqs)
1976 struct device *dev = smmu->dev;
1977 struct acpi_iort_node *node =
1978 *(struct acpi_iort_node **)dev_get_platdata(dev);
1979 struct acpi_iort_smmu *iort_smmu;
1982 /* Retrieve SMMU1/2 specific data */
1983 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1985 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1989 /* Ignore the configuration access interrupt */
1993 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1994 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2000 u32 *global_irqs, u32 *pmu_irqs)
2006 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2007 u32 *global_irqs, u32 *pmu_irqs)
2009 const struct arm_smmu_match_data *data;
2010 struct device *dev = smmu->dev;
2011 bool legacy_binding;
2013 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2014 return dev_err_probe(dev, -ENODEV,
2015 "missing #global-interrupts property\n");
2018 data = of_device_get_match_data(dev);
2019 smmu->version = data->version;
2020 smmu->model = data->model;
2022 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2023 if (legacy_binding && !using_generic_binding) {
2024 if (!using_legacy_binding) {
2025 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2026 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2028 using_legacy_binding = true;
2029 } else if (!legacy_binding && !using_legacy_binding) {
2030 using_generic_binding = true;
2032 dev_err(dev, "not probing due to mismatched DT properties\n");
2036 if (of_dma_is_coherent(dev->of_node))
2037 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2042 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2044 struct list_head rmr_list;
2045 struct iommu_resv_region *e;
2049 INIT_LIST_HEAD(&rmr_list);
2050 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2053 * Rather than trying to look at existing mappings that
2054 * are setup by the firmware and then invalidate the ones
2055 * that do no have matching RMR entries, just disable the
2056 * SMMU until it gets enabled again in the reset routine.
2058 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2059 reg |= ARM_SMMU_sCR0_CLIENTPD;
2060 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2062 list_for_each_entry(e, &rmr_list, list) {
2063 struct iommu_iort_rmr_data *rmr;
2066 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2067 for (i = 0; i < rmr->num_sids; i++) {
2068 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2072 if (smmu->s2crs[idx].count == 0) {
2073 smmu->smrs[idx].id = rmr->sids[i];
2074 smmu->smrs[idx].mask = 0;
2075 smmu->smrs[idx].valid = true;
2077 smmu->s2crs[idx].count++;
2078 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2079 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2085 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2086 cnt == 1 ? "" : "s");
2087 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2090 static int arm_smmu_device_probe(struct platform_device *pdev)
2092 struct resource *res;
2093 struct arm_smmu_device *smmu;
2094 struct device *dev = &pdev->dev;
2095 int num_irqs, i, err;
2096 u32 global_irqs, pmu_irqs;
2097 irqreturn_t (*global_fault)(int irq, void *dev);
2099 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2101 dev_err(dev, "failed to allocate arm_smmu_device\n");
2107 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2109 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2113 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2114 if (IS_ERR(smmu->base))
2115 return PTR_ERR(smmu->base);
2116 smmu->ioaddr = res->start;
2119 * The resource size should effectively match the value of SMMU_TOP;
2120 * stash that temporarily until we know PAGESIZE to validate it with.
2122 smmu->numpage = resource_size(res);
2124 smmu = arm_smmu_impl_init(smmu);
2126 return PTR_ERR(smmu);
2128 num_irqs = platform_irq_count(pdev);
2130 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2131 if (smmu->num_context_irqs <= 0)
2132 return dev_err_probe(dev, -ENODEV,
2133 "found %d interrupts but expected at least %d\n",
2134 num_irqs, global_irqs + pmu_irqs + 1);
2136 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2137 sizeof(*smmu->irqs), GFP_KERNEL);
2139 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2140 smmu->num_context_irqs);
2142 for (i = 0; i < smmu->num_context_irqs; i++) {
2143 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2147 smmu->irqs[i] = irq;
2150 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2152 dev_err(dev, "failed to get clocks %d\n", err);
2155 smmu->num_clks = err;
2157 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2161 err = arm_smmu_device_cfg_probe(smmu);
2165 if (smmu->version == ARM_SMMU_V2) {
2166 if (smmu->num_context_banks > smmu->num_context_irqs) {
2168 "found only %d context irq(s) but %d required\n",
2169 smmu->num_context_irqs, smmu->num_context_banks);
2173 /* Ignore superfluous interrupts */
2174 smmu->num_context_irqs = smmu->num_context_banks;
2177 if (smmu->impl && smmu->impl->global_fault)
2178 global_fault = smmu->impl->global_fault;
2180 global_fault = arm_smmu_global_fault;
2182 for (i = 0; i < global_irqs; i++) {
2183 int irq = platform_get_irq(pdev, i);
2188 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2189 "arm-smmu global fault", smmu);
2191 return dev_err_probe(dev, err,
2192 "failed to request global IRQ %d (%u)\n",
2196 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2197 "smmu.%pa", &smmu->ioaddr);
2199 dev_err(dev, "Failed to register iommu in sysfs\n");
2203 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2204 using_legacy_binding ? NULL : dev);
2206 dev_err(dev, "Failed to register iommu\n");
2207 iommu_device_sysfs_remove(&smmu->iommu);
2211 platform_set_drvdata(pdev, smmu);
2213 /* Check for RMRs and install bypass SMRs if any */
2214 arm_smmu_rmr_install_bypass_smr(smmu);
2216 arm_smmu_device_reset(smmu);
2217 arm_smmu_test_smr_masks(smmu);
2220 * We want to avoid touching dev->power.lock in fastpaths unless
2221 * it's really going to do something useful - pm_runtime_enabled()
2222 * can serve as an ideal proxy for that decision. So, conditionally
2223 * enable pm_runtime.
2225 if (dev->pm_domain) {
2226 pm_runtime_set_active(dev);
2227 pm_runtime_enable(dev);
2233 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2235 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2237 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2238 dev_notice(&pdev->dev, "disabling translation\n");
2240 arm_smmu_rpm_get(smmu);
2241 /* Turn the thing off */
2242 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2243 arm_smmu_rpm_put(smmu);
2245 if (pm_runtime_enabled(smmu->dev))
2246 pm_runtime_force_suspend(smmu->dev);
2248 clk_bulk_disable(smmu->num_clks, smmu->clks);
2250 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2253 static void arm_smmu_device_remove(struct platform_device *pdev)
2255 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2257 iommu_device_unregister(&smmu->iommu);
2258 iommu_device_sysfs_remove(&smmu->iommu);
2260 arm_smmu_device_shutdown(pdev);
2263 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2265 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2268 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2272 arm_smmu_device_reset(smmu);
2277 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2279 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2281 clk_bulk_disable(smmu->num_clks, smmu->clks);
2286 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2289 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2291 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2295 if (pm_runtime_suspended(dev))
2298 ret = arm_smmu_runtime_resume(dev);
2300 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2305 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2308 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2310 if (pm_runtime_suspended(dev))
2313 ret = arm_smmu_runtime_suspend(dev);
2318 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2322 static const struct dev_pm_ops arm_smmu_pm_ops = {
2323 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2324 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2325 arm_smmu_runtime_resume, NULL)
2328 static struct platform_driver arm_smmu_driver = {
2331 .of_match_table = arm_smmu_of_match,
2332 .pm = &arm_smmu_pm_ops,
2333 .suppress_bind_attrs = true,
2335 .probe = arm_smmu_device_probe,
2336 .remove_new = arm_smmu_device_remove,
2337 .shutdown = arm_smmu_device_shutdown,
2339 module_platform_driver(arm_smmu_driver);
2341 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2342 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2343 MODULE_ALIAS("platform:arm-smmu");
2344 MODULE_LICENSE("GPL v2");