drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/io-pgtable.h>
  43 #include <linux/iommu.h>
  44 #include <linux/iopoll.h>
  45 #include <linux/init.h>
  46 #include <linux/moduleparam.h>
  47 #include <linux/of.h>
  48 #include <linux/of_address.h>
  49 #include <linux/of_device.h>
  50 #include <linux/of_iommu.h>
  51 #include <linux/pci.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/pm_runtime.h>
  54 #include <linux/slab.h>
  55 #include <linux/spinlock.h>
  56
  57 #include <linux/amba/bus.h>
  58 #include <linux/fsl/mc.h>
  59
  60 #include "arm-smmu-regs.h"
  61
  62 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
  63
  64 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
  65 #define ARM_MMU500_ACR_S2CRB_TLBEN      (1 << 10)
  66 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
  67
  68 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  69 #define TLB_SPIN_COUNT                  10
  70
  71 /* Maximum number of context banks per SMMU */
  72 #define ARM_SMMU_MAX_CBS                128
  73
  74 /* SMMU global address space */
  75 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  76 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  77
  78 /*
  79  * SMMU global address space with conditional offset to access secure
  80  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  81  * nsGFSYNR0: 0x450)
  82  */
  83 #define ARM_SMMU_GR0_NS(smmu)                                           \
  84         ((smmu)->base +                                                 \
  85                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  86                         ? 0x400 : 0))
  87
  88 /*
  89  * Some 64-bit registers only make sense to write atomically, but in such
  90  * cases all the data relevant to AArch32 formats lies within the lower word,
  91  * therefore this actually makes more sense than it might first appear.
  92  */
  93 #ifdef CONFIG_64BIT
  94 #define smmu_write_atomic_lq            writeq_relaxed
  95 #else
  96 #define smmu_write_atomic_lq            writel_relaxed
  97 #endif
  98
  99 /* Translation context bank */
 100 #define ARM_SMMU_CB(smmu, n)    ((smmu)->cb_base + ((n) << (smmu)->pgshift))
 101
 102 #define MSI_IOVA_BASE                   0x8000000
 103 #define MSI_IOVA_LENGTH                 0x100000
 104
 105 static int force_stage;
 106 /*
 107  * not really modular, but the easiest way to keep compat with existing
 108  * bootargs behaviour is to continue using module_param() here.
 109  */
 110 module_param(force_stage, int, S_IRUGO);
 111 MODULE_PARM_DESC(force_stage,
 112         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 113 static bool disable_bypass =
 114         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
 115 module_param(disable_bypass, bool, S_IRUGO);
 116 MODULE_PARM_DESC(disable_bypass,
 117         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 118
 119 enum arm_smmu_arch_version {
 120         ARM_SMMU_V1,
 121         ARM_SMMU_V1_64K,
 122         ARM_SMMU_V2,
 123 };
 124
 125 enum arm_smmu_implementation {
 126         GENERIC_SMMU,
 127         ARM_MMU500,
 128         CAVIUM_SMMUV2,
 129         QCOM_SMMUV2,
 130 };
 131
 132 struct arm_smmu_s2cr {
 133         struct iommu_group              *group;
 134         int                             count;
 135         enum arm_smmu_s2cr_type         type;
 136         enum arm_smmu_s2cr_privcfg      privcfg;
 137         u8                              cbndx;
 138 };
 139
 140 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 141         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 142 }
 143
 144 struct arm_smmu_smr {
 145         u16                             mask;
 146         u16                             id;
 147         bool                            valid;
 148 };
 149
 150 struct arm_smmu_cb {
 151         u64                             ttbr[2];
 152         u32                             tcr[2];
 153         u32                             mair[2];
 154         struct arm_smmu_cfg             *cfg;
 155 };
 156
 157 struct arm_smmu_master_cfg {
 158         struct arm_smmu_device          *smmu;
 159         s16                             smendx[];
 160 };
 161 #define INVALID_SMENDX                  -1
 162 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 163 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 164 #define fwspec_smendx(fw, i) \
 165         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 166 #define for_each_cfg_sme(fw, i, idx) \
 167         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 168
 169 struct arm_smmu_device {
 170         struct device                   *dev;
 171
 172         void __iomem                    *base;
 173         void __iomem                    *cb_base;
 174         unsigned long                   pgshift;
 175
 176 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 177 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 178 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 179 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 180 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 181 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 182 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 183 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 184 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 185 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 186 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 187 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 188 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 189         u32                             features;
 190
 191 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 192         u32                             options;
 193         enum arm_smmu_arch_version      version;
 194         enum arm_smmu_implementation    model;
 195
 196         u32                             num_context_banks;
 197         u32                             num_s2_context_banks;
 198         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 199         struct arm_smmu_cb              *cbs;
 200         atomic_t                        irptndx;
 201
 202         u32                             num_mapping_groups;
 203         u16                             streamid_mask;
 204         u16                             smr_mask_mask;
 205         struct arm_smmu_smr             *smrs;
 206         struct arm_smmu_s2cr            *s2crs;
 207         struct mutex                    stream_map_mutex;
 208
 209         unsigned long                   va_size;
 210         unsigned long                   ipa_size;
 211         unsigned long                   pa_size;
 212         unsigned long                   pgsize_bitmap;
 213
 214         u32                             num_global_irqs;
 215         u32                             num_context_irqs;
 216         unsigned int                    *irqs;
 217         struct clk_bulk_data            *clks;
 218         int                             num_clks;
 219
 220         u32                             cavium_id_base; /* Specific to Cavium */
 221
 222         spinlock_t                      global_sync_lock;
 223
 224         /* IOMMU core code handle */
 225         struct iommu_device             iommu;
 226 };
 227
 228 enum arm_smmu_context_fmt {
 229         ARM_SMMU_CTX_FMT_NONE,
 230         ARM_SMMU_CTX_FMT_AARCH64,
 231         ARM_SMMU_CTX_FMT_AARCH32_L,
 232         ARM_SMMU_CTX_FMT_AARCH32_S,
 233 };
 234
 235 struct arm_smmu_cfg {
 236         u8                              cbndx;
 237         u8                              irptndx;
 238         union {
 239                 u16                     asid;
 240                 u16                     vmid;
 241         };
 242         u32                             cbar;
 243         enum arm_smmu_context_fmt       fmt;
 244 };
 245 #define INVALID_IRPTNDX                 0xff
 246
 247 enum arm_smmu_domain_stage {
 248         ARM_SMMU_DOMAIN_S1 = 0,
 249         ARM_SMMU_DOMAIN_S2,
 250         ARM_SMMU_DOMAIN_NESTED,
 251         ARM_SMMU_DOMAIN_BYPASS,
 252 };
 253
 254 struct arm_smmu_domain {
 255         struct arm_smmu_device          *smmu;
 256         struct io_pgtable_ops           *pgtbl_ops;
 257         const struct iommu_gather_ops   *tlb_ops;
 258         struct arm_smmu_cfg             cfg;
 259         enum arm_smmu_domain_stage      stage;
 260         bool                            non_strict;
 261         struct mutex                    init_mutex; /* Protects smmu pointer */
 262         spinlock_t                      cb_lock; /* Serialises ATS1* ops and TLB syncs */
 263         struct iommu_domain             domain;
 264 };
 265
 266 struct arm_smmu_option_prop {
 267         u32 opt;
 268         const char *prop;
 269 };
 270
 271 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 272
 273 static bool using_legacy_binding, using_generic_binding;
 274
 275 static struct arm_smmu_option_prop arm_smmu_options[] = {
 276         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 277         { 0, NULL},
 278 };
 279
 280 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 281 {
 282         if (pm_runtime_enabled(smmu->dev))
 283                 return pm_runtime_get_sync(smmu->dev);
 284
 285         return 0;
 286 }
 287
 288 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 289 {
 290         if (pm_runtime_enabled(smmu->dev))
 291                 pm_runtime_put(smmu->dev);
 292 }
 293
 294 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 295 {
 296         return container_of(dom, struct arm_smmu_domain, domain);
 297 }
 298
 299 static void parse_driver_options(struct arm_smmu_device *smmu)
 300 {
 301         int i = 0;
 302
 303         do {
 304                 if (of_property_read_bool(smmu->dev->of_node,
 305                                                 arm_smmu_options[i].prop)) {
 306                         smmu->options |= arm_smmu_options[i].opt;
 307                         dev_notice(smmu->dev, "option %s\n",
 308                                 arm_smmu_options[i].prop);
 309                 }
 310         } while (arm_smmu_options[++i].opt);
 311 }
 312
 313 static struct device_node *dev_get_dev_node(struct device *dev)
 314 {
 315         if (dev_is_pci(dev)) {
 316                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 317
 318                 while (!pci_is_root_bus(bus))
 319                         bus = bus->parent;
 320                 return of_node_get(bus->bridge->parent->of_node);
 321         }
 322
 323         return of_node_get(dev->of_node);
 324 }
 325
 326 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 327 {
 328         *((__be32 *)data) = cpu_to_be32(alias);
 329         return 0; /* Continue walking */
 330 }
 331
 332 static int __find_legacy_master_phandle(struct device *dev, void *data)
 333 {
 334         struct of_phandle_iterator *it = *(void **)data;
 335         struct device_node *np = it->node;
 336         int err;
 337
 338         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 339                             "#stream-id-cells", 0)
 340                 if (it->node == np) {
 341                         *(void **)data = dev;
 342                         return 1;
 343                 }
 344         it->node = np;
 345         return err == -ENOENT ? 0 : err;
 346 }
 347
 348 static struct platform_driver arm_smmu_driver;
 349 static struct iommu_ops arm_smmu_ops;
 350
 351 static int arm_smmu_register_legacy_master(struct device *dev,
 352                                            struct arm_smmu_device **smmu)
 353 {
 354         struct device *smmu_dev;
 355         struct device_node *np;
 356         struct of_phandle_iterator it;
 357         void *data = &it;
 358         u32 *sids;
 359         __be32 pci_sid;
 360         int err;
 361
 362         np = dev_get_dev_node(dev);
 363         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 364                 of_node_put(np);
 365                 return -ENODEV;
 366         }
 367
 368         it.node = np;
 369         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 370                                      __find_legacy_master_phandle);
 371         smmu_dev = data;
 372         of_node_put(np);
 373         if (err == 0)
 374                 return -ENODEV;
 375         if (err < 0)
 376                 return err;
 377
 378         if (dev_is_pci(dev)) {
 379                 /* "mmu-masters" assumes Stream ID == Requester ID */
 380                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 381                                        &pci_sid);
 382                 it.cur = &pci_sid;
 383                 it.cur_count = 1;
 384         }
 385
 386         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 387                                 &arm_smmu_ops);
 388         if (err)
 389                 return err;
 390
 391         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 392         if (!sids)
 393                 return -ENOMEM;
 394
 395         *smmu = dev_get_drvdata(smmu_dev);
 396         of_phandle_iterator_args(&it, sids, it.cur_count);
 397         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 398         kfree(sids);
 399         return err;
 400 }
 401
 402 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 403 {
 404         int idx;
 405
 406         do {
 407                 idx = find_next_zero_bit(map, end, start);
 408                 if (idx == end)
 409                         return -ENOSPC;
 410         } while (test_and_set_bit(idx, map));
 411
 412         return idx;
 413 }
 414
 415 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 416 {
 417         clear_bit(idx, map);
 418 }
 419
 420 /* Wait for any pending TLB invalidations to complete */
 421 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
 422                                 void __iomem *sync, void __iomem *status)
 423 {
 424         unsigned int spin_cnt, delay;
 425
 426         writel_relaxed(0, sync);
 427         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 428                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 429                         if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
 430                                 return;
 431                         cpu_relax();
 432                 }
 433                 udelay(delay);
 434         }
 435         dev_err_ratelimited(smmu->dev,
 436                             "TLB sync timed out -- SMMU may be deadlocked\n");
 437 }
 438
 439 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 440 {
 441         void __iomem *base = ARM_SMMU_GR0(smmu);
 442         unsigned long flags;
 443
 444         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 445         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
 446                             base + ARM_SMMU_GR0_sTLBGSTATUS);
 447         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 448 }
 449
 450 static void arm_smmu_tlb_sync_context(void *cookie)
 451 {
 452         struct arm_smmu_domain *smmu_domain = cookie;
 453         struct arm_smmu_device *smmu = smmu_domain->smmu;
 454         void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 455         unsigned long flags;
 456
 457         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 458         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
 459                             base + ARM_SMMU_CB_TLBSTATUS);
 460         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 461 }
 462
 463 static void arm_smmu_tlb_sync_vmid(void *cookie)
 464 {
 465         struct arm_smmu_domain *smmu_domain = cookie;
 466
 467         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 468 }
 469
 470 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 471 {
 472         struct arm_smmu_domain *smmu_domain = cookie;
 473         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 474         void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 475
 476         /*
 477          * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
 478          * cleared by the current CPU are visible to the SMMU before the TLBI.
 479          */
 480         writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
 481         arm_smmu_tlb_sync_context(cookie);
 482 }
 483
 484 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 485 {
 486         struct arm_smmu_domain *smmu_domain = cookie;
 487         struct arm_smmu_device *smmu = smmu_domain->smmu;
 488         void __iomem *base = ARM_SMMU_GR0(smmu);
 489
 490         /* NOTE: see above */
 491         writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 492         arm_smmu_tlb_sync_global(smmu);
 493 }
 494
 495 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 496                                           size_t granule, bool leaf, void *cookie)
 497 {
 498         struct arm_smmu_domain *smmu_domain = cookie;
 499         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 500         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 501         void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 502
 503         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 504                 wmb();
 505
 506         if (stage1) {
 507                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 508
 509                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 510                         iova &= ~12UL;
 511                         iova |= cfg->asid;
 512                         do {
 513                                 writel_relaxed(iova, reg);
 514                                 iova += granule;
 515                         } while (size -= granule);
 516                 } else {
 517                         iova >>= 12;
 518                         iova |= (u64)cfg->asid << 48;
 519                         do {
 520                                 writeq_relaxed(iova, reg);
 521                                 iova += granule >> 12;
 522                         } while (size -= granule);
 523                 }
 524         } else {
 525                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 526                               ARM_SMMU_CB_S2_TLBIIPAS2;
 527                 iova >>= 12;
 528                 do {
 529                         smmu_write_atomic_lq(iova, reg);
 530                         iova += granule >> 12;
 531                 } while (size -= granule);
 532         }
 533 }
 534
 535 /*
 536  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 537  * almost negligible, but the benefit of getting the first one in as far ahead
 538  * of the sync as possible is significant, hence we don't just make this a
 539  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 540  */
 541 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 542                                          size_t granule, bool leaf, void *cookie)
 543 {
 544         struct arm_smmu_domain *smmu_domain = cookie;
 545         void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
 546
 547         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 548                 wmb();
 549
 550         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 551 }
 552
 553 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 554         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 555         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 556         .tlb_sync       = arm_smmu_tlb_sync_context,
 557 };
 558
 559 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 560         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 561         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 562         .tlb_sync       = arm_smmu_tlb_sync_context,
 563 };
 564
 565 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 566         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 567         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 568         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 569 };
 570
 571 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 572 {
 573         u32 fsr, fsynr, cbfrsynra;
 574         unsigned long iova;
 575         struct iommu_domain *domain = dev;
 576         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 577         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 578         struct arm_smmu_device *smmu = smmu_domain->smmu;
 579         void __iomem *gr1_base = ARM_SMMU_GR1(smmu);
 580         void __iomem *cb_base;
 581
 582         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 583         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 584
 585         if (!(fsr & FSR_FAULT))
 586                 return IRQ_NONE;
 587
 588         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 589         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 590         cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
 591
 592         dev_err_ratelimited(smmu->dev,
 593         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 594                             fsr, iova, fsynr, cbfrsynra, cfg->cbndx);
 595
 596         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 597         return IRQ_HANDLED;
 598 }
 599
 600 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 601 {
 602         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 603         struct arm_smmu_device *smmu = dev;
 604         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 605
 606         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 607         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 608         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 609         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 610
 611         if (!gfsr)
 612                 return IRQ_NONE;
 613
 614         dev_err_ratelimited(smmu->dev,
 615                 "Unexpected global fault, this could be serious\n");
 616         dev_err_ratelimited(smmu->dev,
 617                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 618                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 619
 620         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 621         return IRQ_HANDLED;
 622 }
 623
 624 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 625                                        struct io_pgtable_cfg *pgtbl_cfg)
 626 {
 627         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 628         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 629         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 630
 631         cb->cfg = cfg;
 632
 633         /* TTBCR */
 634         if (stage1) {
 635                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 636                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 637                 } else {
 638                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 639                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 640                         cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
 641                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 642                                 cb->tcr[1] |= TTBCR2_AS;
 643                 }
 644         } else {
 645                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 646         }
 647
 648         /* TTBRs */
 649         if (stage1) {
 650                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 651                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 652                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 653                 } else {
 654                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 655                         cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 656                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 657                         cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 658                 }
 659         } else {
 660                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 661         }
 662
 663         /* MAIRs (stage-1 only) */
 664         if (stage1) {
 665                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 666                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 667                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 668                 } else {
 669                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 670                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 671                 }
 672         }
 673 }
 674
 675 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 676 {
 677         u32 reg;
 678         bool stage1;
 679         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 680         struct arm_smmu_cfg *cfg = cb->cfg;
 681         void __iomem *cb_base, *gr1_base;
 682
 683         cb_base = ARM_SMMU_CB(smmu, idx);
 684
 685         /* Unassigned context banks only need disabling */
 686         if (!cfg) {
 687                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 688                 return;
 689         }
 690
 691         gr1_base = ARM_SMMU_GR1(smmu);
 692         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 693
 694         /* CBA2R */
 695         if (smmu->version > ARM_SMMU_V1) {
 696                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 697                         reg = CBA2R_RW64_64BIT;
 698                 else
 699                         reg = CBA2R_RW64_32BIT;
 700                 /* 16-bit VMIDs live in CBA2R */
 701                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 702                         reg |= cfg->vmid << CBA2R_VMID_SHIFT;
 703
 704                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
 705         }
 706
 707         /* CBAR */
 708         reg = cfg->cbar;
 709         if (smmu->version < ARM_SMMU_V2)
 710                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 711
 712         /*
 713          * Use the weakest shareability/memory types, so they are
 714          * overridden by the ttbcr/pte.
 715          */
 716         if (stage1) {
 717                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 718                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 719         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 720                 /* 8-bit VMIDs live in CBAR */
 721                 reg |= cfg->vmid << CBAR_VMID_SHIFT;
 722         }
 723         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
 724
 725         /*
 726          * TTBCR
 727          * We must write this before the TTBRs, since it determines the
 728          * access behaviour of some fields (in particular, ASID[15:8]).
 729          */
 730         if (stage1 && smmu->version > ARM_SMMU_V1)
 731                 writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
 732         writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
 733
 734         /* TTBRs */
 735         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 736                 writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 737                 writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 738                 writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 739         } else {
 740                 writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 741                 if (stage1)
 742                         writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 743         }
 744
 745         /* MAIRs (stage-1 only) */
 746         if (stage1) {
 747                 writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
 748                 writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
 749         }
 750
 751         /* SCTLR */
 752         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 753         if (stage1)
 754                 reg |= SCTLR_S1_ASIDPNE;
 755         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 756                 reg |= SCTLR_E;
 757
 758         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 759 }
 760
 761 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 762                                         struct arm_smmu_device *smmu)
 763 {
 764         int irq, start, ret = 0;
 765         unsigned long ias, oas;
 766         struct io_pgtable_ops *pgtbl_ops;
 767         struct io_pgtable_cfg pgtbl_cfg;
 768         enum io_pgtable_fmt fmt;
 769         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 770         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 771
 772         mutex_lock(&smmu_domain->init_mutex);
 773         if (smmu_domain->smmu)
 774                 goto out_unlock;
 775
 776         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 777                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 778                 smmu_domain->smmu = smmu;
 779                 goto out_unlock;
 780         }
 781
 782         /*
 783          * Mapping the requested stage onto what we support is surprisingly
 784          * complicated, mainly because the spec allows S1+S2 SMMUs without
 785          * support for nested translation. That means we end up with the
 786          * following table:
 787          *
 788          * Requested        Supported        Actual
 789          *     S1               N              S1
 790          *     S1             S1+S2            S1
 791          *     S1               S2             S2
 792          *     S1               S1             S1
 793          *     N                N              N
 794          *     N              S1+S2            S2
 795          *     N                S2             S2
 796          *     N                S1             S1
 797          *
 798          * Note that you can't actually request stage-2 mappings.
 799          */
 800         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 801                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 802         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 803                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 804
 805         /*
 806          * Choosing a suitable context format is even more fiddly. Until we
 807          * grow some way for the caller to express a preference, and/or move
 808          * the decision into the io-pgtable code where it arguably belongs,
 809          * just aim for the closest thing to the rest of the system, and hope
 810          * that the hardware isn't esoteric enough that we can't assume AArch64
 811          * support to be a superset of AArch32 support...
 812          */
 813         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 814                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 815         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 816             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 817             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 818             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 819                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 820         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 821             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 822                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 823                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 824                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 825
 826         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 827                 ret = -EINVAL;
 828                 goto out_unlock;
 829         }
 830
 831         switch (smmu_domain->stage) {
 832         case ARM_SMMU_DOMAIN_S1:
 833                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 834                 start = smmu->num_s2_context_banks;
 835                 ias = smmu->va_size;
 836                 oas = smmu->ipa_size;
 837                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 838                         fmt = ARM_64_LPAE_S1;
 839                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 840                         fmt = ARM_32_LPAE_S1;
 841                         ias = min(ias, 32UL);
 842                         oas = min(oas, 40UL);
 843                 } else {
 844                         fmt = ARM_V7S;
 845                         ias = min(ias, 32UL);
 846                         oas = min(oas, 32UL);
 847                 }
 848                 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
 849                 break;
 850         case ARM_SMMU_DOMAIN_NESTED:
 851                 /*
 852                  * We will likely want to change this if/when KVM gets
 853                  * involved.
 854                  */
 855         case ARM_SMMU_DOMAIN_S2:
 856                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 857                 start = 0;
 858                 ias = smmu->ipa_size;
 859                 oas = smmu->pa_size;
 860                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 861                         fmt = ARM_64_LPAE_S2;
 862                 } else {
 863                         fmt = ARM_32_LPAE_S2;
 864                         ias = min(ias, 40UL);
 865                         oas = min(oas, 40UL);
 866                 }
 867                 if (smmu->version == ARM_SMMU_V2)
 868                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 869                 else
 870                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 871                 break;
 872         default:
 873                 ret = -EINVAL;
 874                 goto out_unlock;
 875         }
 876         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 877                                       smmu->num_context_banks);
 878         if (ret < 0)
 879                 goto out_unlock;
 880
 881         cfg->cbndx = ret;
 882         if (smmu->version < ARM_SMMU_V2) {
 883                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 884                 cfg->irptndx %= smmu->num_context_irqs;
 885         } else {
 886                 cfg->irptndx = cfg->cbndx;
 887         }
 888
 889         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 890                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
 891         else
 892                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
 893
 894         pgtbl_cfg = (struct io_pgtable_cfg) {
 895                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 896                 .ias            = ias,
 897                 .oas            = oas,
 898                 .tlb            = smmu_domain->tlb_ops,
 899                 .iommu_dev      = smmu->dev,
 900         };
 901
 902         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 903                 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
 904
 905         if (smmu_domain->non_strict)
 906                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 907
 908         smmu_domain->smmu = smmu;
 909         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 910         if (!pgtbl_ops) {
 911                 ret = -ENOMEM;
 912                 goto out_clear_smmu;
 913         }
 914
 915         /* Update the domain's page sizes to reflect the page table format */
 916         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 917         domain->geometry.aperture_end = (1UL << ias) - 1;
 918         domain->geometry.force_aperture = true;
 919
 920         /* Initialise the context bank with our page table cfg */
 921         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 922         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 923
 924         /*
 925          * Request context fault interrupt. Do this last to avoid the
 926          * handler seeing a half-initialised domain state.
 927          */
 928         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 929         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 930                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 931         if (ret < 0) {
 932                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 933                         cfg->irptndx, irq);
 934                 cfg->irptndx = INVALID_IRPTNDX;
 935         }
 936
 937         mutex_unlock(&smmu_domain->init_mutex);
 938
 939         /* Publish page table ops for map/unmap */
 940         smmu_domain->pgtbl_ops = pgtbl_ops;
 941         return 0;
 942
 943 out_clear_smmu:
 944         smmu_domain->smmu = NULL;
 945 out_unlock:
 946         mutex_unlock(&smmu_domain->init_mutex);
 947         return ret;
 948 }
 949
 950 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 951 {
 952         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 953         struct arm_smmu_device *smmu = smmu_domain->smmu;
 954         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 955         int ret, irq;
 956
 957         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 958                 return;
 959
 960         ret = arm_smmu_rpm_get(smmu);
 961         if (ret < 0)
 962                 return;
 963
 964         /*
 965          * Disable the context bank and free the page tables before freeing
 966          * it.
 967          */
 968         smmu->cbs[cfg->cbndx].cfg = NULL;
 969         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 970
 971         if (cfg->irptndx != INVALID_IRPTNDX) {
 972                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 973                 devm_free_irq(smmu->dev, irq, domain);
 974         }
 975
 976         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 977         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 978
 979         arm_smmu_rpm_put(smmu);
 980 }
 981
 982 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 983 {
 984         struct arm_smmu_domain *smmu_domain;
 985
 986         if (type != IOMMU_DOMAIN_UNMANAGED &&
 987             type != IOMMU_DOMAIN_DMA &&
 988             type != IOMMU_DOMAIN_IDENTITY)
 989                 return NULL;
 990         /*
 991          * Allocate the domain and initialise some of its data structures.
 992          * We can't really do anything meaningful until we've added a
 993          * master.
 994          */
 995         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 996         if (!smmu_domain)
 997                 return NULL;
 998
 999         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1000             iommu_get_dma_cookie(&smmu_domain->domain))) {
1001                 kfree(smmu_domain);
1002                 return NULL;
1003         }
1004
1005         mutex_init(&smmu_domain->init_mutex);
1006         spin_lock_init(&smmu_domain->cb_lock);
1007
1008         return &smmu_domain->domain;
1009 }
1010
1011 static void arm_smmu_domain_free(struct iommu_domain *domain)
1012 {
1013         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1014
1015         /*
1016          * Free the domain resources. We assume that all devices have
1017          * already been detached.
1018          */
1019         iommu_put_dma_cookie(domain);
1020         arm_smmu_destroy_domain_context(domain);
1021         kfree(smmu_domain);
1022 }
1023
1024 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1025 {
1026         struct arm_smmu_smr *smr = smmu->smrs + idx;
1027         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1028
1029         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1030                 reg |= SMR_VALID;
1031         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1032 }
1033
1034 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1035 {
1036         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1037         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1038                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1039                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1040
1041         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1042             smmu->smrs[idx].valid)
1043                 reg |= S2CR_EXIDVALID;
1044         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1045 }
1046
1047 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1048 {
1049         arm_smmu_write_s2cr(smmu, idx);
1050         if (smmu->smrs)
1051                 arm_smmu_write_smr(smmu, idx);
1052 }
1053
1054 /*
1055  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1056  * should be called after sCR0 is written.
1057  */
1058 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1059 {
1060         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1061         u32 smr;
1062
1063         if (!smmu->smrs)
1064                 return;
1065
1066         /*
1067          * SMR.ID bits may not be preserved if the corresponding MASK
1068          * bits are set, so check each one separately. We can reject
1069          * masters later if they try to claim IDs outside these masks.
1070          */
1071         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1072         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1073         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1074         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1075
1076         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1077         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1078         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1079         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1080 }
1081
1082 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1083 {
1084         struct arm_smmu_smr *smrs = smmu->smrs;
1085         int i, free_idx = -ENOSPC;
1086
1087         /* Stream indexing is blissfully easy */
1088         if (!smrs)
1089                 return id;
1090
1091         /* Validating SMRs is... less so */
1092         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1093                 if (!smrs[i].valid) {
1094                         /*
1095                          * Note the first free entry we come across, which
1096                          * we'll claim in the end if nothing else matches.
1097                          */
1098                         if (free_idx < 0)
1099                                 free_idx = i;
1100                         continue;
1101                 }
1102                 /*
1103                  * If the new entry is _entirely_ matched by an existing entry,
1104                  * then reuse that, with the guarantee that there also cannot
1105                  * be any subsequent conflicting entries. In normal use we'd
1106                  * expect simply identical entries for this case, but there's
1107                  * no harm in accommodating the generalisation.
1108                  */
1109                 if ((mask & smrs[i].mask) == mask &&
1110                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1111                         return i;
1112                 /*
1113                  * If the new entry has any other overlap with an existing one,
1114                  * though, then there always exists at least one stream ID
1115                  * which would cause a conflict, and we can't allow that risk.
1116                  */
1117                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1118                         return -EINVAL;
1119         }
1120
1121         return free_idx;
1122 }
1123
1124 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1125 {
1126         if (--smmu->s2crs[idx].count)
1127                 return false;
1128
1129         smmu->s2crs[idx] = s2cr_init_val;
1130         if (smmu->smrs)
1131                 smmu->smrs[idx].valid = false;
1132
1133         return true;
1134 }
1135
1136 static int arm_smmu_master_alloc_smes(struct device *dev)
1137 {
1138         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1139         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1140         struct arm_smmu_device *smmu = cfg->smmu;
1141         struct arm_smmu_smr *smrs = smmu->smrs;
1142         struct iommu_group *group;
1143         int i, idx, ret;
1144
1145         mutex_lock(&smmu->stream_map_mutex);
1146         /* Figure out a viable stream map entry allocation */
1147         for_each_cfg_sme(fwspec, i, idx) {
1148                 u16 sid = fwspec->ids[i];
1149                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1150
1151                 if (idx != INVALID_SMENDX) {
1152                         ret = -EEXIST;
1153                         goto out_err;
1154                 }
1155
1156                 ret = arm_smmu_find_sme(smmu, sid, mask);
1157                 if (ret < 0)
1158                         goto out_err;
1159
1160                 idx = ret;
1161                 if (smrs && smmu->s2crs[idx].count == 0) {
1162                         smrs[idx].id = sid;
1163                         smrs[idx].mask = mask;
1164                         smrs[idx].valid = true;
1165                 }
1166                 smmu->s2crs[idx].count++;
1167                 cfg->smendx[i] = (s16)idx;
1168         }
1169
1170         group = iommu_group_get_for_dev(dev);
1171         if (!group)
1172                 group = ERR_PTR(-ENOMEM);
1173         if (IS_ERR(group)) {
1174                 ret = PTR_ERR(group);
1175                 goto out_err;
1176         }
1177         iommu_group_put(group);
1178
1179         /* It worked! Now, poke the actual hardware */
1180         for_each_cfg_sme(fwspec, i, idx) {
1181                 arm_smmu_write_sme(smmu, idx);
1182                 smmu->s2crs[idx].group = group;
1183         }
1184
1185         mutex_unlock(&smmu->stream_map_mutex);
1186         return 0;
1187
1188 out_err:
1189         while (i--) {
1190                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1191                 cfg->smendx[i] = INVALID_SMENDX;
1192         }
1193         mutex_unlock(&smmu->stream_map_mutex);
1194         return ret;
1195 }
1196
1197 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1198 {
1199         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1200         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1201         int i, idx;
1202
1203         mutex_lock(&smmu->stream_map_mutex);
1204         for_each_cfg_sme(fwspec, i, idx) {
1205                 if (arm_smmu_free_sme(smmu, idx))
1206                         arm_smmu_write_sme(smmu, idx);
1207                 cfg->smendx[i] = INVALID_SMENDX;
1208         }
1209         mutex_unlock(&smmu->stream_map_mutex);
1210 }
1211
1212 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1213                                       struct iommu_fwspec *fwspec)
1214 {
1215         struct arm_smmu_device *smmu = smmu_domain->smmu;
1216         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1217         u8 cbndx = smmu_domain->cfg.cbndx;
1218         enum arm_smmu_s2cr_type type;
1219         int i, idx;
1220
1221         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1222                 type = S2CR_TYPE_BYPASS;
1223         else
1224                 type = S2CR_TYPE_TRANS;
1225
1226         for_each_cfg_sme(fwspec, i, idx) {
1227                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1228                         continue;
1229
1230                 s2cr[idx].type = type;
1231                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1232                 s2cr[idx].cbndx = cbndx;
1233                 arm_smmu_write_s2cr(smmu, idx);
1234         }
1235         return 0;
1236 }
1237
1238 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1239 {
1240         int ret;
1241         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1242         struct arm_smmu_device *smmu;
1243         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1244
1245         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1246                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1247                 return -ENXIO;
1248         }
1249
1250         /*
1251          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1252          * domains between of_xlate() and add_device() - we have no way to cope
1253          * with that, so until ARM gets converted to rely on groups and default
1254          * domains, just say no (but more politely than by dereferencing NULL).
1255          * This should be at least a WARN_ON once that's sorted.
1256          */
1257         if (!fwspec->iommu_priv)
1258                 return -ENODEV;
1259
1260         smmu = fwspec_smmu(fwspec);
1261
1262         ret = arm_smmu_rpm_get(smmu);
1263         if (ret < 0)
1264                 return ret;
1265
1266         /* Ensure that the domain is finalised */
1267         ret = arm_smmu_init_domain_context(domain, smmu);
1268         if (ret < 0)
1269                 goto rpm_put;
1270
1271         /*
1272          * Sanity check the domain. We don't support domains across
1273          * different SMMUs.
1274          */
1275         if (smmu_domain->smmu != smmu) {
1276                 dev_err(dev,
1277                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1278                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1279                 ret = -EINVAL;
1280                 goto rpm_put;
1281         }
1282
1283         /* Looks ok, so add the device to the domain */
1284         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1285
1286 rpm_put:
1287         arm_smmu_rpm_put(smmu);
1288         return ret;
1289 }
1290
1291 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1292                         phys_addr_t paddr, size_t size, int prot)
1293 {
1294         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1295         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1296         int ret;
1297
1298         if (!ops)
1299                 return -ENODEV;
1300
1301         arm_smmu_rpm_get(smmu);
1302         ret = ops->map(ops, iova, paddr, size, prot);
1303         arm_smmu_rpm_put(smmu);
1304
1305         return ret;
1306 }
1307
1308 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1309                              size_t size)
1310 {
1311         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1312         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1313         size_t ret;
1314
1315         if (!ops)
1316                 return 0;
1317
1318         arm_smmu_rpm_get(smmu);
1319         ret = ops->unmap(ops, iova, size);
1320         arm_smmu_rpm_put(smmu);
1321
1322         return ret;
1323 }
1324
1325 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1326 {
1327         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1328         struct arm_smmu_device *smmu = smmu_domain->smmu;
1329
1330         if (smmu_domain->tlb_ops) {
1331                 arm_smmu_rpm_get(smmu);
1332                 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1333                 arm_smmu_rpm_put(smmu);
1334         }
1335 }
1336
1337 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1338 {
1339         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1340         struct arm_smmu_device *smmu = smmu_domain->smmu;
1341
1342         if (smmu_domain->tlb_ops) {
1343                 arm_smmu_rpm_get(smmu);
1344                 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1345                 arm_smmu_rpm_put(smmu);
1346         }
1347 }
1348
1349 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1350                                               dma_addr_t iova)
1351 {
1352         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1353         struct arm_smmu_device *smmu = smmu_domain->smmu;
1354         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1355         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1356         struct device *dev = smmu->dev;
1357         void __iomem *cb_base;
1358         u32 tmp;
1359         u64 phys;
1360         unsigned long va, flags;
1361         int ret;
1362
1363         ret = arm_smmu_rpm_get(smmu);
1364         if (ret < 0)
1365                 return 0;
1366
1367         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1368
1369         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1370         /* ATS1 registers can only be written atomically */
1371         va = iova & ~0xfffUL;
1372         if (smmu->version == ARM_SMMU_V2)
1373                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1374         else /* Register is only 32-bit in v1 */
1375                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1376
1377         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1378                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1379                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1380                 dev_err(dev,
1381                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1382                         &iova);
1383                 return ops->iova_to_phys(ops, iova);
1384         }
1385
1386         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1387         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1388         if (phys & CB_PAR_F) {
1389                 dev_err(dev, "translation fault!\n");
1390                 dev_err(dev, "PAR = 0x%llx\n", phys);
1391                 return 0;
1392         }
1393
1394         arm_smmu_rpm_put(smmu);
1395
1396         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1397 }
1398
1399 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1400                                         dma_addr_t iova)
1401 {
1402         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1403         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1404
1405         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1406                 return iova;
1407
1408         if (!ops)
1409                 return 0;
1410
1411         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1412                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1413                 return arm_smmu_iova_to_phys_hard(domain, iova);
1414
1415         return ops->iova_to_phys(ops, iova);
1416 }
1417
1418 static bool arm_smmu_capable(enum iommu_cap cap)
1419 {
1420         switch (cap) {
1421         case IOMMU_CAP_CACHE_COHERENCY:
1422                 /*
1423                  * Return true here as the SMMU can always send out coherent
1424                  * requests.
1425                  */
1426                 return true;
1427         case IOMMU_CAP_NOEXEC:
1428                 return true;
1429         default:
1430                 return false;
1431         }
1432 }
1433
1434 static int arm_smmu_match_node(struct device *dev, void *data)
1435 {
1436         return dev->fwnode == data;
1437 }
1438
1439 static
1440 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1441 {
1442         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1443                                                 fwnode, arm_smmu_match_node);
1444         put_device(dev);
1445         return dev ? dev_get_drvdata(dev) : NULL;
1446 }
1447
1448 static int arm_smmu_add_device(struct device *dev)
1449 {
1450         struct arm_smmu_device *smmu;
1451         struct arm_smmu_master_cfg *cfg;
1452         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1453         int i, ret;
1454
1455         if (using_legacy_binding) {
1456                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1457
1458                 /*
1459                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1460                  * will allocate/initialise a new one. Thus we need to update fwspec for
1461                  * later use.
1462                  */
1463                 fwspec = dev_iommu_fwspec_get(dev);
1464                 if (ret)
1465                         goto out_free;
1466         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1467                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1468         } else {
1469                 return -ENODEV;
1470         }
1471
1472         ret = -EINVAL;
1473         for (i = 0; i < fwspec->num_ids; i++) {
1474                 u16 sid = fwspec->ids[i];
1475                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1476
1477                 if (sid & ~smmu->streamid_mask) {
1478                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1479                                 sid, smmu->streamid_mask);
1480                         goto out_free;
1481                 }
1482                 if (mask & ~smmu->smr_mask_mask) {
1483                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1484                                 mask, smmu->smr_mask_mask);
1485                         goto out_free;
1486                 }
1487         }
1488
1489         ret = -ENOMEM;
1490         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1491                       GFP_KERNEL);
1492         if (!cfg)
1493                 goto out_free;
1494
1495         cfg->smmu = smmu;
1496         fwspec->iommu_priv = cfg;
1497         while (i--)
1498                 cfg->smendx[i] = INVALID_SMENDX;
1499
1500         ret = arm_smmu_rpm_get(smmu);
1501         if (ret < 0)
1502                 goto out_cfg_free;
1503
1504         ret = arm_smmu_master_alloc_smes(dev);
1505         arm_smmu_rpm_put(smmu);
1506
1507         if (ret)
1508                 goto out_cfg_free;
1509
1510         iommu_device_link(&smmu->iommu, dev);
1511
1512         device_link_add(dev, smmu->dev,
1513                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1514
1515         return 0;
1516
1517 out_cfg_free:
1518         kfree(cfg);
1519 out_free:
1520         iommu_fwspec_free(dev);
1521         return ret;
1522 }
1523
1524 static void arm_smmu_remove_device(struct device *dev)
1525 {
1526         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1527         struct arm_smmu_master_cfg *cfg;
1528         struct arm_smmu_device *smmu;
1529         int ret;
1530
1531         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1532                 return;
1533
1534         cfg  = fwspec->iommu_priv;
1535         smmu = cfg->smmu;
1536
1537         ret = arm_smmu_rpm_get(smmu);
1538         if (ret < 0)
1539                 return;
1540
1541         iommu_device_unlink(&smmu->iommu, dev);
1542         arm_smmu_master_free_smes(fwspec);
1543
1544         arm_smmu_rpm_put(smmu);
1545
1546         iommu_group_remove_device(dev);
1547         kfree(fwspec->iommu_priv);
1548         iommu_fwspec_free(dev);
1549 }
1550
1551 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1552 {
1553         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1554         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1555         struct iommu_group *group = NULL;
1556         int i, idx;
1557
1558         for_each_cfg_sme(fwspec, i, idx) {
1559                 if (group && smmu->s2crs[idx].group &&
1560                     group != smmu->s2crs[idx].group)
1561                         return ERR_PTR(-EINVAL);
1562
1563                 group = smmu->s2crs[idx].group;
1564         }
1565
1566         if (group)
1567                 return iommu_group_ref_get(group);
1568
1569         if (dev_is_pci(dev))
1570                 group = pci_device_group(dev);
1571         else if (dev_is_fsl_mc(dev))
1572                 group = fsl_mc_device_group(dev);
1573         else
1574                 group = generic_device_group(dev);
1575
1576         return group;
1577 }
1578
1579 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1580                                     enum iommu_attr attr, void *data)
1581 {
1582         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1583
1584         switch(domain->type) {
1585         case IOMMU_DOMAIN_UNMANAGED:
1586                 switch (attr) {
1587                 case DOMAIN_ATTR_NESTING:
1588                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1589                         return 0;
1590                 default:
1591                         return -ENODEV;
1592                 }
1593                 break;
1594         case IOMMU_DOMAIN_DMA:
1595                 switch (attr) {
1596                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1597                         *(int *)data = smmu_domain->non_strict;
1598                         return 0;
1599                 default:
1600                         return -ENODEV;
1601                 }
1602                 break;
1603         default:
1604                 return -EINVAL;
1605         }
1606 }
1607
1608 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1609                                     enum iommu_attr attr, void *data)
1610 {
1611         int ret = 0;
1612         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1613
1614         mutex_lock(&smmu_domain->init_mutex);
1615
1616         switch(domain->type) {
1617         case IOMMU_DOMAIN_UNMANAGED:
1618                 switch (attr) {
1619                 case DOMAIN_ATTR_NESTING:
1620                         if (smmu_domain->smmu) {
1621                                 ret = -EPERM;
1622                                 goto out_unlock;
1623                         }
1624
1625                         if (*(int *)data)
1626                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1627                         else
1628                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1629                         break;
1630                 default:
1631                         ret = -ENODEV;
1632                 }
1633                 break;
1634         case IOMMU_DOMAIN_DMA:
1635                 switch (attr) {
1636                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1637                         smmu_domain->non_strict = *(int *)data;
1638                         break;
1639                 default:
1640                         ret = -ENODEV;
1641                 }
1642                 break;
1643         default:
1644                 ret = -EINVAL;
1645         }
1646 out_unlock:
1647         mutex_unlock(&smmu_domain->init_mutex);
1648         return ret;
1649 }
1650
1651 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1652 {
1653         u32 mask, fwid = 0;
1654
1655         if (args->args_count > 0)
1656                 fwid |= (u16)args->args[0];
1657
1658         if (args->args_count > 1)
1659                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1660         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1661                 fwid |= (u16)mask << SMR_MASK_SHIFT;
1662
1663         return iommu_fwspec_add_ids(dev, &fwid, 1);
1664 }
1665
1666 static void arm_smmu_get_resv_regions(struct device *dev,
1667                                       struct list_head *head)
1668 {
1669         struct iommu_resv_region *region;
1670         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1671
1672         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1673                                          prot, IOMMU_RESV_SW_MSI);
1674         if (!region)
1675                 return;
1676
1677         list_add_tail(&region->list, head);
1678
1679         iommu_dma_get_resv_regions(dev, head);
1680 }
1681
1682 static void arm_smmu_put_resv_regions(struct device *dev,
1683                                       struct list_head *head)
1684 {
1685         struct iommu_resv_region *entry, *next;
1686
1687         list_for_each_entry_safe(entry, next, head, list)
1688                 kfree(entry);
1689 }
1690
1691 static struct iommu_ops arm_smmu_ops = {
1692         .capable                = arm_smmu_capable,
1693         .domain_alloc           = arm_smmu_domain_alloc,
1694         .domain_free            = arm_smmu_domain_free,
1695         .attach_dev             = arm_smmu_attach_dev,
1696         .map                    = arm_smmu_map,
1697         .unmap                  = arm_smmu_unmap,
1698         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1699         .iotlb_sync             = arm_smmu_iotlb_sync,
1700         .iova_to_phys           = arm_smmu_iova_to_phys,
1701         .add_device             = arm_smmu_add_device,
1702         .remove_device          = arm_smmu_remove_device,
1703         .device_group           = arm_smmu_device_group,
1704         .domain_get_attr        = arm_smmu_domain_get_attr,
1705         .domain_set_attr        = arm_smmu_domain_set_attr,
1706         .of_xlate               = arm_smmu_of_xlate,
1707         .get_resv_regions       = arm_smmu_get_resv_regions,
1708         .put_resv_regions       = arm_smmu_put_resv_regions,
1709         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1710 };
1711
1712 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1713 {
1714         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1715         int i;
1716         u32 reg, major;
1717
1718         /* clear global FSR */
1719         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1720         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1721
1722         /*
1723          * Reset stream mapping groups: Initial values mark all SMRn as
1724          * invalid and all S2CRn as bypass unless overridden.
1725          */
1726         for (i = 0; i < smmu->num_mapping_groups; ++i)
1727                 arm_smmu_write_sme(smmu, i);
1728
1729         if (smmu->model == ARM_MMU500) {
1730                 /*
1731                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1732                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1733                  * bit is only present in MMU-500r2 onwards.
1734                  */
1735                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1736                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1737                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1738                 if (major >= 2)
1739                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1740                 /*
1741                  * Allow unmatched Stream IDs to allocate bypass
1742                  * TLB entries for reduced latency.
1743                  */
1744                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1745                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1746         }
1747
1748         /* Make sure all context banks are disabled and clear CB_FSR  */
1749         for (i = 0; i < smmu->num_context_banks; ++i) {
1750                 void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
1751
1752                 arm_smmu_write_context_bank(smmu, i);
1753                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1754                 /*
1755                  * Disable MMU-500's not-particularly-beneficial next-page
1756                  * prefetcher for the sake of errata #841119 and #826419.
1757                  */
1758                 if (smmu->model == ARM_MMU500) {
1759                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1760                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1761                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1762                 }
1763         }
1764
1765         /* Invalidate the TLB, just in case */
1766         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1767         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1768
1769         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1770
1771         /* Enable fault reporting */
1772         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1773
1774         /* Disable TLB broadcasting. */
1775         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1776
1777         /* Enable client access, handling unmatched streams as appropriate */
1778         reg &= ~sCR0_CLIENTPD;
1779         if (disable_bypass)
1780                 reg |= sCR0_USFCFG;
1781         else
1782                 reg &= ~sCR0_USFCFG;
1783
1784         /* Disable forced broadcasting */
1785         reg &= ~sCR0_FB;
1786
1787         /* Don't upgrade barriers */
1788         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1789
1790         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1791                 reg |= sCR0_VMID16EN;
1792
1793         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1794                 reg |= sCR0_EXIDENABLE;
1795
1796         /* Push the button */
1797         arm_smmu_tlb_sync_global(smmu);
1798         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1799 }
1800
1801 static int arm_smmu_id_size_to_bits(int size)
1802 {
1803         switch (size) {
1804         case 0:
1805                 return 32;
1806         case 1:
1807                 return 36;
1808         case 2:
1809                 return 40;
1810         case 3:
1811                 return 42;
1812         case 4:
1813                 return 44;
1814         case 5:
1815         default:
1816                 return 48;
1817         }
1818 }
1819
1820 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1821 {
1822         unsigned long size;
1823         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1824         u32 id;
1825         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1826         int i;
1827
1828         dev_notice(smmu->dev, "probing hardware configuration...\n");
1829         dev_notice(smmu->dev, "SMMUv%d with:\n",
1830                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1831
1832         /* ID0 */
1833         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1834
1835         /* Restrict available stages based on module parameter */
1836         if (force_stage == 1)
1837                 id &= ~(ID0_S2TS | ID0_NTS);
1838         else if (force_stage == 2)
1839                 id &= ~(ID0_S1TS | ID0_NTS);
1840
1841         if (id & ID0_S1TS) {
1842                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1843                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1844         }
1845
1846         if (id & ID0_S2TS) {
1847                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1848                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1849         }
1850
1851         if (id & ID0_NTS) {
1852                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1853                 dev_notice(smmu->dev, "\tnested translation\n");
1854         }
1855
1856         if (!(smmu->features &
1857                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1858                 dev_err(smmu->dev, "\tno translation support!\n");
1859                 return -ENODEV;
1860         }
1861
1862         if ((id & ID0_S1TS) &&
1863                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1864                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1865                 dev_notice(smmu->dev, "\taddress translation ops\n");
1866         }
1867
1868         /*
1869          * In order for DMA API calls to work properly, we must defer to what
1870          * the FW says about coherency, regardless of what the hardware claims.
1871          * Fortunately, this also opens up a workaround for systems where the
1872          * ID register value has ended up configured incorrectly.
1873          */
1874         cttw_reg = !!(id & ID0_CTTW);
1875         if (cttw_fw || cttw_reg)
1876                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1877                            cttw_fw ? "" : "non-");
1878         if (cttw_fw != cttw_reg)
1879                 dev_notice(smmu->dev,
1880                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1881
1882         /* Max. number of entries we have for stream matching/indexing */
1883         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1884                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1885                 size = 1 << 16;
1886         } else {
1887                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1888         }
1889         smmu->streamid_mask = size - 1;
1890         if (id & ID0_SMS) {
1891                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1892                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1893                 if (size == 0) {
1894                         dev_err(smmu->dev,
1895                                 "stream-matching supported, but no SMRs present!\n");
1896                         return -ENODEV;
1897                 }
1898
1899                 /* Zero-initialised to mark as invalid */
1900                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1901                                           GFP_KERNEL);
1902                 if (!smmu->smrs)
1903                         return -ENOMEM;
1904
1905                 dev_notice(smmu->dev,
1906                            "\tstream matching with %lu register groups", size);
1907         }
1908         /* s2cr->type == 0 means translation, so initialise explicitly */
1909         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1910                                          GFP_KERNEL);
1911         if (!smmu->s2crs)
1912                 return -ENOMEM;
1913         for (i = 0; i < size; i++)
1914                 smmu->s2crs[i] = s2cr_init_val;
1915
1916         smmu->num_mapping_groups = size;
1917         mutex_init(&smmu->stream_map_mutex);
1918         spin_lock_init(&smmu->global_sync_lock);
1919
1920         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1921                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1922                 if (!(id & ID0_PTFS_NO_AARCH32S))
1923                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1924         }
1925
1926         /* ID1 */
1927         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1928         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1929
1930         /* Check for size mismatch of SMMU address space from mapped region */
1931         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1932         size <<= smmu->pgshift;
1933         if (smmu->cb_base != gr0_base + size)
1934                 dev_warn(smmu->dev,
1935                         "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n",
1936                         size * 2, (smmu->cb_base - gr0_base) * 2);
1937
1938         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1939         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1940         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1941                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1942                 return -ENODEV;
1943         }
1944         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1945                    smmu->num_context_banks, smmu->num_s2_context_banks);
1946         /*
1947          * Cavium CN88xx erratum #27704.
1948          * Ensure ASID and VMID allocation is unique across all SMMUs in
1949          * the system.
1950          */
1951         if (smmu->model == CAVIUM_SMMUV2) {
1952                 smmu->cavium_id_base =
1953                         atomic_add_return(smmu->num_context_banks,
1954                                           &cavium_smmu_context_count);
1955                 smmu->cavium_id_base -= smmu->num_context_banks;
1956                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1957         }
1958         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1959                                  sizeof(*smmu->cbs), GFP_KERNEL);
1960         if (!smmu->cbs)
1961                 return -ENOMEM;
1962
1963         /* ID2 */
1964         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1965         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1966         smmu->ipa_size = size;
1967
1968         /* The output mask is also applied for bypass */
1969         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1970         smmu->pa_size = size;
1971
1972         if (id & ID2_VMID16)
1973                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1974
1975         /*
1976          * What the page table walker can address actually depends on which
1977          * descriptor format is in use, but since a) we don't know that yet,
1978          * and b) it can vary per context bank, this will have to do...
1979          */
1980         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1981                 dev_warn(smmu->dev,
1982                          "failed to set DMA mask for table walker\n");
1983
1984         if (smmu->version < ARM_SMMU_V2) {
1985                 smmu->va_size = smmu->ipa_size;
1986                 if (smmu->version == ARM_SMMU_V1_64K)
1987                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1988         } else {
1989                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1990                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1991                 if (id & ID2_PTFS_4K)
1992                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1993                 if (id & ID2_PTFS_16K)
1994                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1995                 if (id & ID2_PTFS_64K)
1996                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1997         }
1998
1999         /* Now we've corralled the various formats, what'll it do? */
2000         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2001                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2002         if (smmu->features &
2003             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2004                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2005         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2006                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2007         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2008                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2009
2010         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2011                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2012         else
2013                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2014         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2015                    smmu->pgsize_bitmap);
2016
2017
2018         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2019                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2020                            smmu->va_size, smmu->ipa_size);
2021
2022         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2023                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2024                            smmu->ipa_size, smmu->pa_size);
2025
2026         return 0;
2027 }
2028
2029 struct arm_smmu_match_data {
2030         enum arm_smmu_arch_version version;
2031         enum arm_smmu_implementation model;
2032 };
2033
2034 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
2035 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
2036
2037 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2038 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2039 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2040 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2041 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2042 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
2043
2044 static const struct of_device_id arm_smmu_of_match[] = {
2045         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2046         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2047         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2048         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2049         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2050         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2051         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
2052         { },
2053 };
2054
2055 #ifdef CONFIG_ACPI
2056 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2057 {
2058         int ret = 0;
2059
2060         switch (model) {
2061         case ACPI_IORT_SMMU_V1:
2062         case ACPI_IORT_SMMU_CORELINK_MMU400:
2063                 smmu->version = ARM_SMMU_V1;
2064                 smmu->model = GENERIC_SMMU;
2065                 break;
2066         case ACPI_IORT_SMMU_CORELINK_MMU401:
2067                 smmu->version = ARM_SMMU_V1_64K;
2068                 smmu->model = GENERIC_SMMU;
2069                 break;
2070         case ACPI_IORT_SMMU_V2:
2071                 smmu->version = ARM_SMMU_V2;
2072                 smmu->model = GENERIC_SMMU;
2073                 break;
2074         case ACPI_IORT_SMMU_CORELINK_MMU500:
2075                 smmu->version = ARM_SMMU_V2;
2076                 smmu->model = ARM_MMU500;
2077                 break;
2078         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2079                 smmu->version = ARM_SMMU_V2;
2080                 smmu->model = CAVIUM_SMMUV2;
2081                 break;
2082         default:
2083                 ret = -ENODEV;
2084         }
2085
2086         return ret;
2087 }
2088
2089 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2090                                       struct arm_smmu_device *smmu)
2091 {
2092         struct device *dev = smmu->dev;
2093         struct acpi_iort_node *node =
2094                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2095         struct acpi_iort_smmu *iort_smmu;
2096         int ret;
2097
2098         /* Retrieve SMMU1/2 specific data */
2099         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2100
2101         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2102         if (ret < 0)
2103                 return ret;
2104
2105         /* Ignore the configuration access interrupt */
2106         smmu->num_global_irqs = 1;
2107
2108         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2109                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2110
2111         return 0;
2112 }
2113 #else
2114 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2115                                              struct arm_smmu_device *smmu)
2116 {
2117         return -ENODEV;
2118 }
2119 #endif
2120
2121 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2122                                     struct arm_smmu_device *smmu)
2123 {
2124         const struct arm_smmu_match_data *data;
2125         struct device *dev = &pdev->dev;
2126         bool legacy_binding;
2127
2128         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2129                                  &smmu->num_global_irqs)) {
2130                 dev_err(dev, "missing #global-interrupts property\n");
2131                 return -ENODEV;
2132         }
2133
2134         data = of_device_get_match_data(dev);
2135         smmu->version = data->version;
2136         smmu->model = data->model;
2137
2138         parse_driver_options(smmu);
2139
2140         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2141         if (legacy_binding && !using_generic_binding) {
2142                 if (!using_legacy_binding)
2143                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2144                 using_legacy_binding = true;
2145         } else if (!legacy_binding && !using_legacy_binding) {
2146                 using_generic_binding = true;
2147         } else {
2148                 dev_err(dev, "not probing due to mismatched DT properties\n");
2149                 return -ENODEV;
2150         }
2151
2152         if (of_dma_is_coherent(dev->of_node))
2153                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2154
2155         return 0;
2156 }
2157
2158 static void arm_smmu_bus_init(void)
2159 {
2160         /* Oh, for a proper bus abstraction */
2161         if (!iommu_present(&platform_bus_type))
2162                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2163 #ifdef CONFIG_ARM_AMBA
2164         if (!iommu_present(&amba_bustype))
2165                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2166 #endif
2167 #ifdef CONFIG_PCI
2168         if (!iommu_present(&pci_bus_type)) {
2169                 pci_request_acs();
2170                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2171         }
2172 #endif
2173 #ifdef CONFIG_FSL_MC_BUS
2174         if (!iommu_present(&fsl_mc_bus_type))
2175                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2176 #endif
2177 }
2178
2179 static int arm_smmu_device_probe(struct platform_device *pdev)
2180 {
2181         struct resource *res;
2182         resource_size_t ioaddr;
2183         struct arm_smmu_device *smmu;
2184         struct device *dev = &pdev->dev;
2185         int num_irqs, i, err;
2186
2187         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2188         if (!smmu) {
2189                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2190                 return -ENOMEM;
2191         }
2192         smmu->dev = dev;
2193
2194         if (dev->of_node)
2195                 err = arm_smmu_device_dt_probe(pdev, smmu);
2196         else
2197                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2198
2199         if (err)
2200                 return err;
2201
2202         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2203         ioaddr = res->start;
2204         smmu->base = devm_ioremap_resource(dev, res);
2205         if (IS_ERR(smmu->base))
2206                 return PTR_ERR(smmu->base);
2207         smmu->cb_base = smmu->base + resource_size(res) / 2;
2208
2209         num_irqs = 0;
2210         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2211                 num_irqs++;
2212                 if (num_irqs > smmu->num_global_irqs)
2213                         smmu->num_context_irqs++;
2214         }
2215
2216         if (!smmu->num_context_irqs) {
2217                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2218                         num_irqs, smmu->num_global_irqs + 1);
2219                 return -ENODEV;
2220         }
2221
2222         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2223                                   GFP_KERNEL);
2224         if (!smmu->irqs) {
2225                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2226                 return -ENOMEM;
2227         }
2228
2229         for (i = 0; i < num_irqs; ++i) {
2230                 int irq = platform_get_irq(pdev, i);
2231
2232                 if (irq < 0) {
2233                         dev_err(dev, "failed to get irq index %d\n", i);
2234                         return -ENODEV;
2235                 }
2236                 smmu->irqs[i] = irq;
2237         }
2238
2239         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2240         if (err < 0) {
2241                 dev_err(dev, "failed to get clocks %d\n", err);
2242                 return err;
2243         }
2244         smmu->num_clks = err;
2245
2246         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2247         if (err)
2248                 return err;
2249
2250         err = arm_smmu_device_cfg_probe(smmu);
2251         if (err)
2252                 return err;
2253
2254         if (smmu->version == ARM_SMMU_V2) {
2255                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2256                         dev_err(dev,
2257                               "found only %d context irq(s) but %d required\n",
2258                               smmu->num_context_irqs, smmu->num_context_banks);
2259                         return -ENODEV;
2260                 }
2261
2262                 /* Ignore superfluous interrupts */
2263                 smmu->num_context_irqs = smmu->num_context_banks;
2264         }
2265
2266         for (i = 0; i < smmu->num_global_irqs; ++i) {
2267                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2268                                        arm_smmu_global_fault,
2269                                        IRQF_SHARED,
2270                                        "arm-smmu global fault",
2271                                        smmu);
2272                 if (err) {
2273                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2274                                 i, smmu->irqs[i]);
2275                         return err;
2276                 }
2277         }
2278
2279         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2280                                      "smmu.%pa", &ioaddr);
2281         if (err) {
2282                 dev_err(dev, "Failed to register iommu in sysfs\n");
2283                 return err;
2284         }
2285
2286         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2287         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2288
2289         err = iommu_device_register(&smmu->iommu);
2290         if (err) {
2291                 dev_err(dev, "Failed to register iommu\n");
2292                 return err;
2293         }
2294
2295         platform_set_drvdata(pdev, smmu);
2296         arm_smmu_device_reset(smmu);
2297         arm_smmu_test_smr_masks(smmu);
2298
2299         /*
2300          * We want to avoid touching dev->power.lock in fastpaths unless
2301          * it's really going to do something useful - pm_runtime_enabled()
2302          * can serve as an ideal proxy for that decision. So, conditionally
2303          * enable pm_runtime.
2304          */
2305         if (dev->pm_domain) {
2306                 pm_runtime_set_active(dev);
2307                 pm_runtime_enable(dev);
2308         }
2309
2310         /*
2311          * For ACPI and generic DT bindings, an SMMU will be probed before
2312          * any device which might need it, so we want the bus ops in place
2313          * ready to handle default domain setup as soon as any SMMU exists.
2314          */
2315         if (!using_legacy_binding)
2316                 arm_smmu_bus_init();
2317
2318         return 0;
2319 }
2320
2321 /*
2322  * With the legacy DT binding in play, though, we have no guarantees about
2323  * probe order, but then we're also not doing default domains, so we can
2324  * delay setting bus ops until we're sure every possible SMMU is ready,
2325  * and that way ensure that no add_device() calls get missed.
2326  */
2327 static int arm_smmu_legacy_bus_init(void)
2328 {
2329         if (using_legacy_binding)
2330                 arm_smmu_bus_init();
2331         return 0;
2332 }
2333 device_initcall_sync(arm_smmu_legacy_bus_init);
2334
2335 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2336 {
2337         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2338
2339         if (!smmu)
2340                 return;
2341
2342         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2343                 dev_err(&pdev->dev, "removing device with active domains!\n");
2344
2345         arm_smmu_rpm_get(smmu);
2346         /* Turn the thing off */
2347         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2348         arm_smmu_rpm_put(smmu);
2349
2350         if (pm_runtime_enabled(smmu->dev))
2351                 pm_runtime_force_suspend(smmu->dev);
2352         else
2353                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2354
2355         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2356 }
2357
2358 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2359 {
2360         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2361         int ret;
2362
2363         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2364         if (ret)
2365                 return ret;
2366
2367         arm_smmu_device_reset(smmu);
2368
2369         return 0;
2370 }
2371
2372 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2373 {
2374         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2375
2376         clk_bulk_disable(smmu->num_clks, smmu->clks);
2377
2378         return 0;
2379 }
2380
2381 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2382 {
2383         if (pm_runtime_suspended(dev))
2384                 return 0;
2385
2386         return arm_smmu_runtime_resume(dev);
2387 }
2388
2389 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2390 {
2391         if (pm_runtime_suspended(dev))
2392                 return 0;
2393
2394         return arm_smmu_runtime_suspend(dev);
2395 }
2396
2397 static const struct dev_pm_ops arm_smmu_pm_ops = {
2398         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2399         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2400                            arm_smmu_runtime_resume, NULL)
2401 };
2402
2403 static struct platform_driver arm_smmu_driver = {
2404         .driver = {
2405                 .name                   = "arm-smmu",
2406                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2407                 .pm                     = &arm_smmu_pm_ops,
2408                 .suppress_bind_attrs    = true,
2409         },
2410         .probe  = arm_smmu_device_probe,
2411         .shutdown = arm_smmu_device_shutdown,
2412 };
2413 builtin_platform_driver(arm_smmu_driver);