drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/dma-direct.h>
  49 #include <linux/crash_dump.h>
  50 #include <asm/irq_remapping.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/iommu.h>
  53
  54 #include "irq_remapping.h"
  55 #include "intel-pasid.h"
  56
  57 #define ROOT_SIZE               VTD_PAGE_SIZE
  58 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  59
  60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  64
  65 #define IOAPIC_RANGE_START      (0xfee00000)
  66 #define IOAPIC_RANGE_END        (0xfeefffff)
  67 #define IOVA_START_ADDR         (0x1000)
  68
  69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  70
  71 #define MAX_AGAW_WIDTH 64
  72 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  73
  74 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  76
  77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  78    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  79 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  80                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  81 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  82
  83 /* IO virtual address start page frame number */
  84 #define IOVA_START_PFN          (1)
  85
  86 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186 int intel_iommu_tboot_noforce;
 187
 188 /*
 189  * 0: Present
 190  * 1-11: Reserved
 191  * 12-63: Context Ptr (12 - (haw-1))
 192  * 64-127: Reserved
 193  */
 194 struct root_entry {
 195         u64     lo;
 196         u64     hi;
 197 };
 198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 199
 200 /*
 201  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 202  * if marked present.
 203  */
 204 static phys_addr_t root_entry_lctp(struct root_entry *re)
 205 {
 206         if (!(re->lo & 1))
 207                 return 0;
 208
 209         return re->lo & VTD_PAGE_MASK;
 210 }
 211
 212 /*
 213  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 214  * if marked present.
 215  */
 216 static phys_addr_t root_entry_uctp(struct root_entry *re)
 217 {
 218         if (!(re->hi & 1))
 219                 return 0;
 220
 221         return re->hi & VTD_PAGE_MASK;
 222 }
 223 /*
 224  * low 64 bits:
 225  * 0: present
 226  * 1: fault processing disable
 227  * 2-3: translation type
 228  * 12-63: address space root
 229  * high 64 bits:
 230  * 0-2: address width
 231  * 3-6: aval
 232  * 8-23: domain id
 233  */
 234 struct context_entry {
 235         u64 lo;
 236         u64 hi;
 237 };
 238
 239 static inline void context_clear_pasid_enable(struct context_entry *context)
 240 {
 241         context->lo &= ~(1ULL << 11);
 242 }
 243
 244 static inline bool context_pasid_enabled(struct context_entry *context)
 245 {
 246         return !!(context->lo & (1ULL << 11));
 247 }
 248
 249 static inline void context_set_copied(struct context_entry *context)
 250 {
 251         context->hi |= (1ull << 3);
 252 }
 253
 254 static inline bool context_copied(struct context_entry *context)
 255 {
 256         return !!(context->hi & (1ULL << 3));
 257 }
 258
 259 static inline bool __context_present(struct context_entry *context)
 260 {
 261         return (context->lo & 1);
 262 }
 263
 264 static inline bool context_present(struct context_entry *context)
 265 {
 266         return context_pasid_enabled(context) ?
 267              __context_present(context) :
 268              __context_present(context) && !context_copied(context);
 269 }
 270
 271 static inline void context_set_present(struct context_entry *context)
 272 {
 273         context->lo |= 1;
 274 }
 275
 276 static inline void context_set_fault_enable(struct context_entry *context)
 277 {
 278         context->lo &= (((u64)-1) << 2) | 1;
 279 }
 280
 281 static inline void context_set_translation_type(struct context_entry *context,
 282                                                 unsigned long value)
 283 {
 284         context->lo &= (((u64)-1) << 4) | 3;
 285         context->lo |= (value & 3) << 2;
 286 }
 287
 288 static inline void context_set_address_root(struct context_entry *context,
 289                                             unsigned long value)
 290 {
 291         context->lo &= ~VTD_PAGE_MASK;
 292         context->lo |= value & VTD_PAGE_MASK;
 293 }
 294
 295 static inline void context_set_address_width(struct context_entry *context,
 296                                              unsigned long value)
 297 {
 298         context->hi |= value & 7;
 299 }
 300
 301 static inline void context_set_domain_id(struct context_entry *context,
 302                                          unsigned long value)
 303 {
 304         context->hi |= (value & ((1 << 16) - 1)) << 8;
 305 }
 306
 307 static inline int context_domain_id(struct context_entry *c)
 308 {
 309         return((c->hi >> 8) & 0xffff);
 310 }
 311
 312 static inline void context_clear_entry(struct context_entry *context)
 313 {
 314         context->lo = 0;
 315         context->hi = 0;
 316 }
 317
 318 /*
 319  * 0: readable
 320  * 1: writable
 321  * 2-6: reserved
 322  * 7: super page
 323  * 8-10: available
 324  * 11: snoop behavior
 325  * 12-63: Host physcial address
 326  */
 327 struct dma_pte {
 328         u64 val;
 329 };
 330
 331 static inline void dma_clear_pte(struct dma_pte *pte)
 332 {
 333         pte->val = 0;
 334 }
 335
 336 static inline u64 dma_pte_addr(struct dma_pte *pte)
 337 {
 338 #ifdef CONFIG_64BIT
 339         return pte->val & VTD_PAGE_MASK;
 340 #else
 341         /* Must have a full atomic 64-bit read */
 342         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 343 #endif
 344 }
 345
 346 static inline bool dma_pte_present(struct dma_pte *pte)
 347 {
 348         return (pte->val & 3) != 0;
 349 }
 350
 351 static inline bool dma_pte_superpage(struct dma_pte *pte)
 352 {
 353         return (pte->val & DMA_PTE_LARGE_PAGE);
 354 }
 355
 356 static inline int first_pte_in_page(struct dma_pte *pte)
 357 {
 358         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 359 }
 360
 361 /*
 362  * This domain is a statically identity mapping domain.
 363  *      1. This domain creats a static 1:1 mapping to all usable memory.
 364  *      2. It maps to each iommu if successful.
 365  *      3. Each iommu mapps to this domain if successful.
 366  */
 367 static struct dmar_domain *si_domain;
 368 static int hw_pass_through = 1;
 369
 370 /*
 371  * Domain represents a virtual machine, more than one devices
 372  * across iommus may be owned in one domain, e.g. kvm guest.
 373  */
 374 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 375
 376 /* si_domain contains mulitple devices */
 377 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 378
 379 #define for_each_domain_iommu(idx, domain)                      \
 380         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 381                 if (domain->iommu_refcnt[idx])
 382
 383 struct dmar_rmrr_unit {
 384         struct list_head list;          /* list of rmrr units   */
 385         struct acpi_dmar_header *hdr;   /* ACPI header          */
 386         u64     base_address;           /* reserved base address*/
 387         u64     end_address;            /* reserved end address */
 388         struct dmar_dev_scope *devices; /* target devices */
 389         int     devices_cnt;            /* target device count */
 390         struct iommu_resv_region *resv; /* reserved region handle */
 391 };
 392
 393 struct dmar_atsr_unit {
 394         struct list_head list;          /* list of ATSR units */
 395         struct acpi_dmar_header *hdr;   /* ACPI header */
 396         struct dmar_dev_scope *devices; /* target devices */
 397         int devices_cnt;                /* target device count */
 398         u8 include_all:1;               /* include all ports */
 399 };
 400
 401 static LIST_HEAD(dmar_atsr_units);
 402 static LIST_HEAD(dmar_rmrr_units);
 403
 404 #define for_each_rmrr_units(rmrr) \
 405         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 406
 407 /* bitmap for indexing intel_iommus */
 408 static int g_num_of_iommus;
 409
 410 static void domain_exit(struct dmar_domain *domain);
 411 static void domain_remove_dev_info(struct dmar_domain *domain);
 412 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 413                                      struct device *dev);
 414 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 415 static void domain_context_clear(struct intel_iommu *iommu,
 416                                  struct device *dev);
 417 static int domain_detach_iommu(struct dmar_domain *domain,
 418                                struct intel_iommu *iommu);
 419
 420 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 421 int dmar_disabled = 0;
 422 #else
 423 int dmar_disabled = 1;
 424 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 425
 426 int intel_iommu_enabled = 0;
 427 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 428
 429 static int dmar_map_gfx = 1;
 430 static int dmar_forcedac;
 431 static int intel_iommu_strict;
 432 static int intel_iommu_superpage = 1;
 433 static int intel_iommu_ecs = 1;
 434 static int intel_iommu_pasid28;
 435 static int iommu_identity_mapping;
 436
 437 #define IDENTMAP_ALL            1
 438 #define IDENTMAP_GFX            2
 439 #define IDENTMAP_AZALIA         4
 440
 441 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 442  * level" translation of DMA requests-without-PASID doesn't actually happen
 443  * unless you also set the NESTE bit in an extended context-entry. Which of
 444  * course means that SVM doesn't work because it's trying to do nested
 445  * translation of the physical addresses it finds in the process page tables,
 446  * through the IOVA->phys mapping found in the "second level" page tables.
 447  *
 448  * The VT-d specification was retroactively changed to change the definition
 449  * of the capability bits and pretend that Broadwell/Skylake never happened...
 450  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 451  * for some reason it was the PASID capability bit which was redefined (from
 452  * bit 28 on BDW/SKL to bit 40 in future).
 453  *
 454  * So our test for ECS needs to eschew those implementations which set the old
 455  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 456  * Unless we are working around the 'pasid28' limitations, that is, by putting
 457  * the device into passthrough mode for normal DMA and thus masking the bug.
 458  */
 459 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 460                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 461 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 462  * or new capability bits are set. */
 463 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 464                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 465
 466 int intel_iommu_gfx_mapped;
 467 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 468
 469 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 470 static DEFINE_SPINLOCK(device_domain_lock);
 471 static LIST_HEAD(device_domain_list);
 472
 473 /*
 474  * Iterate over elements in device_domain_list and call the specified
 475  * callback @fn against each element. This helper should only be used
 476  * in the context where the device_domain_lock has already been holden.
 477  */
 478 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
 479                                      void *data), void *data)
 480 {
 481         int ret = 0;
 482         struct device_domain_info *info;
 483
 484         assert_spin_locked(&device_domain_lock);
 485         list_for_each_entry(info, &device_domain_list, global) {
 486                 ret = fn(info, data);
 487                 if (ret)
 488                         return ret;
 489         }
 490
 491         return 0;
 492 }
 493
 494 const struct iommu_ops intel_iommu_ops;
 495
 496 static bool translation_pre_enabled(struct intel_iommu *iommu)
 497 {
 498         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 499 }
 500
 501 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 502 {
 503         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 504 }
 505
 506 static void init_translation_status(struct intel_iommu *iommu)
 507 {
 508         u32 gsts;
 509
 510         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 511         if (gsts & DMA_GSTS_TES)
 512                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 513 }
 514
 515 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 516 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 517 {
 518         return container_of(dom, struct dmar_domain, domain);
 519 }
 520
 521 static int __init intel_iommu_setup(char *str)
 522 {
 523         if (!str)
 524                 return -EINVAL;
 525         while (*str) {
 526                 if (!strncmp(str, "on", 2)) {
 527                         dmar_disabled = 0;
 528                         pr_info("IOMMU enabled\n");
 529                 } else if (!strncmp(str, "off", 3)) {
 530                         dmar_disabled = 1;
 531                         pr_info("IOMMU disabled\n");
 532                 } else if (!strncmp(str, "igfx_off", 8)) {
 533                         dmar_map_gfx = 0;
 534                         pr_info("Disable GFX device mapping\n");
 535                 } else if (!strncmp(str, "forcedac", 8)) {
 536                         pr_info("Forcing DAC for PCI devices\n");
 537                         dmar_forcedac = 1;
 538                 } else if (!strncmp(str, "strict", 6)) {
 539                         pr_info("Disable batched IOTLB flush\n");
 540                         intel_iommu_strict = 1;
 541                 } else if (!strncmp(str, "sp_off", 6)) {
 542                         pr_info("Disable supported super page\n");
 543                         intel_iommu_superpage = 0;
 544                 } else if (!strncmp(str, "ecs_off", 7)) {
 545                         printk(KERN_INFO
 546                                 "Intel-IOMMU: disable extended context table support\n");
 547                         intel_iommu_ecs = 0;
 548                 } else if (!strncmp(str, "pasid28", 7)) {
 549                         printk(KERN_INFO
 550                                 "Intel-IOMMU: enable pre-production PASID support\n");
 551                         intel_iommu_pasid28 = 1;
 552                         iommu_identity_mapping |= IDENTMAP_GFX;
 553                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 554                         printk(KERN_INFO
 555                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 556                         intel_iommu_tboot_noforce = 1;
 557                 }
 558
 559                 str += strcspn(str, ",");
 560                 while (*str == ',')
 561                         str++;
 562         }
 563         return 0;
 564 }
 565 __setup("intel_iommu=", intel_iommu_setup);
 566
 567 static struct kmem_cache *iommu_domain_cache;
 568 static struct kmem_cache *iommu_devinfo_cache;
 569
 570 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 571 {
 572         struct dmar_domain **domains;
 573         int idx = did >> 8;
 574
 575         domains = iommu->domains[idx];
 576         if (!domains)
 577                 return NULL;
 578
 579         return domains[did & 0xff];
 580 }
 581
 582 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 583                              struct dmar_domain *domain)
 584 {
 585         struct dmar_domain **domains;
 586         int idx = did >> 8;
 587
 588         if (!iommu->domains[idx]) {
 589                 size_t size = 256 * sizeof(struct dmar_domain *);
 590                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 591         }
 592
 593         domains = iommu->domains[idx];
 594         if (WARN_ON(!domains))
 595                 return;
 596         else
 597                 domains[did & 0xff] = domain;
 598 }
 599
 600 void *alloc_pgtable_page(int node)
 601 {
 602         struct page *page;
 603         void *vaddr = NULL;
 604
 605         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 606         if (page)
 607                 vaddr = page_address(page);
 608         return vaddr;
 609 }
 610
 611 void free_pgtable_page(void *vaddr)
 612 {
 613         free_page((unsigned long)vaddr);
 614 }
 615
 616 static inline void *alloc_domain_mem(void)
 617 {
 618         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 619 }
 620
 621 static void free_domain_mem(void *vaddr)
 622 {
 623         kmem_cache_free(iommu_domain_cache, vaddr);
 624 }
 625
 626 static inline void * alloc_devinfo_mem(void)
 627 {
 628         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 629 }
 630
 631 static inline void free_devinfo_mem(void *vaddr)
 632 {
 633         kmem_cache_free(iommu_devinfo_cache, vaddr);
 634 }
 635
 636 static inline int domain_type_is_vm(struct dmar_domain *domain)
 637 {
 638         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 639 }
 640
 641 static inline int domain_type_is_si(struct dmar_domain *domain)
 642 {
 643         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 644 }
 645
 646 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 647 {
 648         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 649                                 DOMAIN_FLAG_STATIC_IDENTITY);
 650 }
 651
 652 static inline int domain_pfn_supported(struct dmar_domain *domain,
 653                                        unsigned long pfn)
 654 {
 655         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 656
 657         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 658 }
 659
 660 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 661 {
 662         unsigned long sagaw;
 663         int agaw = -1;
 664
 665         sagaw = cap_sagaw(iommu->cap);
 666         for (agaw = width_to_agaw(max_gaw);
 667              agaw >= 0; agaw--) {
 668                 if (test_bit(agaw, &sagaw))
 669                         break;
 670         }
 671
 672         return agaw;
 673 }
 674
 675 /*
 676  * Calculate max SAGAW for each iommu.
 677  */
 678 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 679 {
 680         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 681 }
 682
 683 /*
 684  * calculate agaw for each iommu.
 685  * "SAGAW" may be different across iommus, use a default agaw, and
 686  * get a supported less agaw for iommus that don't support the default agaw.
 687  */
 688 int iommu_calculate_agaw(struct intel_iommu *iommu)
 689 {
 690         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 691 }
 692
 693 /* This functionin only returns single iommu in a domain */
 694 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 695 {
 696         int iommu_id;
 697
 698         /* si_domain and vm domain should not get here. */
 699         BUG_ON(domain_type_is_vm_or_si(domain));
 700         for_each_domain_iommu(iommu_id, domain)
 701                 break;
 702
 703         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 704                 return NULL;
 705
 706         return g_iommus[iommu_id];
 707 }
 708
 709 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 710 {
 711         struct dmar_drhd_unit *drhd;
 712         struct intel_iommu *iommu;
 713         bool found = false;
 714         int i;
 715
 716         domain->iommu_coherency = 1;
 717
 718         for_each_domain_iommu(i, domain) {
 719                 found = true;
 720                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 721                         domain->iommu_coherency = 0;
 722                         break;
 723                 }
 724         }
 725         if (found)
 726                 return;
 727
 728         /* No hardware attached; use lowest common denominator */
 729         rcu_read_lock();
 730         for_each_active_iommu(iommu, drhd) {
 731                 if (!ecap_coherent(iommu->ecap)) {
 732                         domain->iommu_coherency = 0;
 733                         break;
 734                 }
 735         }
 736         rcu_read_unlock();
 737 }
 738
 739 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 740 {
 741         struct dmar_drhd_unit *drhd;
 742         struct intel_iommu *iommu;
 743         int ret = 1;
 744
 745         rcu_read_lock();
 746         for_each_active_iommu(iommu, drhd) {
 747                 if (iommu != skip) {
 748                         if (!ecap_sc_support(iommu->ecap)) {
 749                                 ret = 0;
 750                                 break;
 751                         }
 752                 }
 753         }
 754         rcu_read_unlock();
 755
 756         return ret;
 757 }
 758
 759 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 760 {
 761         struct dmar_drhd_unit *drhd;
 762         struct intel_iommu *iommu;
 763         int mask = 0xf;
 764
 765         if (!intel_iommu_superpage) {
 766                 return 0;
 767         }
 768
 769         /* set iommu_superpage to the smallest common denominator */
 770         rcu_read_lock();
 771         for_each_active_iommu(iommu, drhd) {
 772                 if (iommu != skip) {
 773                         mask &= cap_super_page_val(iommu->cap);
 774                         if (!mask)
 775                                 break;
 776                 }
 777         }
 778         rcu_read_unlock();
 779
 780         return fls(mask);
 781 }
 782
 783 /* Some capabilities may be different across iommus */
 784 static void domain_update_iommu_cap(struct dmar_domain *domain)
 785 {
 786         domain_update_iommu_coherency(domain);
 787         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 788         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 789 }
 790
 791 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 792                                                        u8 bus, u8 devfn, int alloc)
 793 {
 794         struct root_entry *root = &iommu->root_entry[bus];
 795         struct context_entry *context;
 796         u64 *entry;
 797
 798         entry = &root->lo;
 799         if (ecs_enabled(iommu)) {
 800                 if (devfn >= 0x80) {
 801                         devfn -= 0x80;
 802                         entry = &root->hi;
 803                 }
 804                 devfn *= 2;
 805         }
 806         if (*entry & 1)
 807                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 808         else {
 809                 unsigned long phy_addr;
 810                 if (!alloc)
 811                         return NULL;
 812
 813                 context = alloc_pgtable_page(iommu->node);
 814                 if (!context)
 815                         return NULL;
 816
 817                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 818                 phy_addr = virt_to_phys((void *)context);
 819                 *entry = phy_addr | 1;
 820                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 821         }
 822         return &context[devfn];
 823 }
 824
 825 static int iommu_dummy(struct device *dev)
 826 {
 827         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 828 }
 829
 830 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 831 {
 832         struct dmar_drhd_unit *drhd = NULL;
 833         struct intel_iommu *iommu;
 834         struct device *tmp;
 835         struct pci_dev *ptmp, *pdev = NULL;
 836         u16 segment = 0;
 837         int i;
 838
 839         if (iommu_dummy(dev))
 840                 return NULL;
 841
 842         if (dev_is_pci(dev)) {
 843                 struct pci_dev *pf_pdev;
 844
 845                 pdev = to_pci_dev(dev);
 846
 847 #ifdef CONFIG_X86
 848                 /* VMD child devices currently cannot be handled individually */
 849                 if (is_vmd(pdev->bus))
 850                         return NULL;
 851 #endif
 852
 853                 /* VFs aren't listed in scope tables; we need to look up
 854                  * the PF instead to find the IOMMU. */
 855                 pf_pdev = pci_physfn(pdev);
 856                 dev = &pf_pdev->dev;
 857                 segment = pci_domain_nr(pdev->bus);
 858         } else if (has_acpi_companion(dev))
 859                 dev = &ACPI_COMPANION(dev)->dev;
 860
 861         rcu_read_lock();
 862         for_each_active_iommu(iommu, drhd) {
 863                 if (pdev && segment != drhd->segment)
 864                         continue;
 865
 866                 for_each_active_dev_scope(drhd->devices,
 867                                           drhd->devices_cnt, i, tmp) {
 868                         if (tmp == dev) {
 869                                 /* For a VF use its original BDF# not that of the PF
 870                                  * which we used for the IOMMU lookup. Strictly speaking
 871                                  * we could do this for all PCI devices; we only need to
 872                                  * get the BDF# from the scope table for ACPI matches. */
 873                                 if (pdev && pdev->is_virtfn)
 874                                         goto got_pdev;
 875
 876                                 *bus = drhd->devices[i].bus;
 877                                 *devfn = drhd->devices[i].devfn;
 878                                 goto out;
 879                         }
 880
 881                         if (!pdev || !dev_is_pci(tmp))
 882                                 continue;
 883
 884                         ptmp = to_pci_dev(tmp);
 885                         if (ptmp->subordinate &&
 886                             ptmp->subordinate->number <= pdev->bus->number &&
 887                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 888                                 goto got_pdev;
 889                 }
 890
 891                 if (pdev && drhd->include_all) {
 892                 got_pdev:
 893                         *bus = pdev->bus->number;
 894                         *devfn = pdev->devfn;
 895                         goto out;
 896                 }
 897         }
 898         iommu = NULL;
 899  out:
 900         rcu_read_unlock();
 901
 902         return iommu;
 903 }
 904
 905 static void domain_flush_cache(struct dmar_domain *domain,
 906                                void *addr, int size)
 907 {
 908         if (!domain->iommu_coherency)
 909                 clflush_cache_range(addr, size);
 910 }
 911
 912 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 913 {
 914         struct context_entry *context;
 915         int ret = 0;
 916         unsigned long flags;
 917
 918         spin_lock_irqsave(&iommu->lock, flags);
 919         context = iommu_context_addr(iommu, bus, devfn, 0);
 920         if (context)
 921                 ret = context_present(context);
 922         spin_unlock_irqrestore(&iommu->lock, flags);
 923         return ret;
 924 }
 925
 926 static void free_context_table(struct intel_iommu *iommu)
 927 {
 928         int i;
 929         unsigned long flags;
 930         struct context_entry *context;
 931
 932         spin_lock_irqsave(&iommu->lock, flags);
 933         if (!iommu->root_entry) {
 934                 goto out;
 935         }
 936         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 937                 context = iommu_context_addr(iommu, i, 0, 0);
 938                 if (context)
 939                         free_pgtable_page(context);
 940
 941                 if (!ecs_enabled(iommu))
 942                         continue;
 943
 944                 context = iommu_context_addr(iommu, i, 0x80, 0);
 945                 if (context)
 946                         free_pgtable_page(context);
 947
 948         }
 949         free_pgtable_page(iommu->root_entry);
 950         iommu->root_entry = NULL;
 951 out:
 952         spin_unlock_irqrestore(&iommu->lock, flags);
 953 }
 954
 955 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 956                                       unsigned long pfn, int *target_level)
 957 {
 958         struct dma_pte *parent, *pte = NULL;
 959         int level = agaw_to_level(domain->agaw);
 960         int offset;
 961
 962         BUG_ON(!domain->pgd);
 963
 964         if (!domain_pfn_supported(domain, pfn))
 965                 /* Address beyond IOMMU's addressing capabilities. */
 966                 return NULL;
 967
 968         parent = domain->pgd;
 969
 970         while (1) {
 971                 void *tmp_page;
 972
 973                 offset = pfn_level_offset(pfn, level);
 974                 pte = &parent[offset];
 975                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 976                         break;
 977                 if (level == *target_level)
 978                         break;
 979
 980                 if (!dma_pte_present(pte)) {
 981                         uint64_t pteval;
 982
 983                         tmp_page = alloc_pgtable_page(domain->nid);
 984
 985                         if (!tmp_page)
 986                                 return NULL;
 987
 988                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 989                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 990                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 991                                 /* Someone else set it while we were thinking; use theirs. */
 992                                 free_pgtable_page(tmp_page);
 993                         else
 994                                 domain_flush_cache(domain, pte, sizeof(*pte));
 995                 }
 996                 if (level == 1)
 997                         break;
 998
 999                 parent = phys_to_virt(dma_pte_addr(pte));
1000                 level--;
1001         }
1002
1003         if (!*target_level)
1004                 *target_level = level;
1005
1006         return pte;
1007 }
1008
1009
1010 /* return address's pte at specific level */
1011 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1012                                          unsigned long pfn,
1013                                          int level, int *large_page)
1014 {
1015         struct dma_pte *parent, *pte = NULL;
1016         int total = agaw_to_level(domain->agaw);
1017         int offset;
1018
1019         parent = domain->pgd;
1020         while (level <= total) {
1021                 offset = pfn_level_offset(pfn, total);
1022                 pte = &parent[offset];
1023                 if (level == total)
1024                         return pte;
1025
1026                 if (!dma_pte_present(pte)) {
1027                         *large_page = total;
1028                         break;
1029                 }
1030
1031                 if (dma_pte_superpage(pte)) {
1032                         *large_page = total;
1033                         return pte;
1034                 }
1035
1036                 parent = phys_to_virt(dma_pte_addr(pte));
1037                 total--;
1038         }
1039         return NULL;
1040 }
1041
1042 /* clear last level pte, a tlb flush should be followed */
1043 static void dma_pte_clear_range(struct dmar_domain *domain,
1044                                 unsigned long start_pfn,
1045                                 unsigned long last_pfn)
1046 {
1047         unsigned int large_page = 1;
1048         struct dma_pte *first_pte, *pte;
1049
1050         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1051         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1052         BUG_ON(start_pfn > last_pfn);
1053
1054         /* we don't need lock here; nobody else touches the iova range */
1055         do {
1056                 large_page = 1;
1057                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1058                 if (!pte) {
1059                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1060                         continue;
1061                 }
1062                 do {
1063                         dma_clear_pte(pte);
1064                         start_pfn += lvl_to_nr_pages(large_page);
1065                         pte++;
1066                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1067
1068                 domain_flush_cache(domain, first_pte,
1069                                    (void *)pte - (void *)first_pte);
1070
1071         } while (start_pfn && start_pfn <= last_pfn);
1072 }
1073
1074 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1075                                int retain_level, struct dma_pte *pte,
1076                                unsigned long pfn, unsigned long start_pfn,
1077                                unsigned long last_pfn)
1078 {
1079         pfn = max(start_pfn, pfn);
1080         pte = &pte[pfn_level_offset(pfn, level)];
1081
1082         do {
1083                 unsigned long level_pfn;
1084                 struct dma_pte *level_pte;
1085
1086                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1087                         goto next;
1088
1089                 level_pfn = pfn & level_mask(level);
1090                 level_pte = phys_to_virt(dma_pte_addr(pte));
1091
1092                 if (level > 2) {
1093                         dma_pte_free_level(domain, level - 1, retain_level,
1094                                            level_pte, level_pfn, start_pfn,
1095                                            last_pfn);
1096                 }
1097
1098                 /*
1099                  * Free the page table if we're below the level we want to
1100                  * retain and the range covers the entire table.
1101                  */
1102                 if (level < retain_level && !(start_pfn > level_pfn ||
1103                       last_pfn < level_pfn + level_size(level) - 1)) {
1104                         dma_clear_pte(pte);
1105                         domain_flush_cache(domain, pte, sizeof(*pte));
1106                         free_pgtable_page(level_pte);
1107                 }
1108 next:
1109                 pfn += level_size(level);
1110         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1111 }
1112
1113 /*
1114  * clear last level (leaf) ptes and free page table pages below the
1115  * level we wish to keep intact.
1116  */
1117 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1118                                    unsigned long start_pfn,
1119                                    unsigned long last_pfn,
1120                                    int retain_level)
1121 {
1122         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1123         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1124         BUG_ON(start_pfn > last_pfn);
1125
1126         dma_pte_clear_range(domain, start_pfn, last_pfn);
1127
1128         /* We don't need lock here; nobody else touches the iova range */
1129         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1130                            domain->pgd, 0, start_pfn, last_pfn);
1131
1132         /* free pgd */
1133         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1134                 free_pgtable_page(domain->pgd);
1135                 domain->pgd = NULL;
1136         }
1137 }
1138
1139 /* When a page at a given level is being unlinked from its parent, we don't
1140    need to *modify* it at all. All we need to do is make a list of all the
1141    pages which can be freed just as soon as we've flushed the IOTLB and we
1142    know the hardware page-walk will no longer touch them.
1143    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1144    be freed. */
1145 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1146                                             int level, struct dma_pte *pte,
1147                                             struct page *freelist)
1148 {
1149         struct page *pg;
1150
1151         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1152         pg->freelist = freelist;
1153         freelist = pg;
1154
1155         if (level == 1)
1156                 return freelist;
1157
1158         pte = page_address(pg);
1159         do {
1160                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1161                         freelist = dma_pte_list_pagetables(domain, level - 1,
1162                                                            pte, freelist);
1163                 pte++;
1164         } while (!first_pte_in_page(pte));
1165
1166         return freelist;
1167 }
1168
1169 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1170                                         struct dma_pte *pte, unsigned long pfn,
1171                                         unsigned long start_pfn,
1172                                         unsigned long last_pfn,
1173                                         struct page *freelist)
1174 {
1175         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1176
1177         pfn = max(start_pfn, pfn);
1178         pte = &pte[pfn_level_offset(pfn, level)];
1179
1180         do {
1181                 unsigned long level_pfn;
1182
1183                 if (!dma_pte_present(pte))
1184                         goto next;
1185
1186                 level_pfn = pfn & level_mask(level);
1187
1188                 /* If range covers entire pagetable, free it */
1189                 if (start_pfn <= level_pfn &&
1190                     last_pfn >= level_pfn + level_size(level) - 1) {
1191                         /* These suborbinate page tables are going away entirely. Don't
1192                            bother to clear them; we're just going to *free* them. */
1193                         if (level > 1 && !dma_pte_superpage(pte))
1194                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1195
1196                         dma_clear_pte(pte);
1197                         if (!first_pte)
1198                                 first_pte = pte;
1199                         last_pte = pte;
1200                 } else if (level > 1) {
1201                         /* Recurse down into a level that isn't *entirely* obsolete */
1202                         freelist = dma_pte_clear_level(domain, level - 1,
1203                                                        phys_to_virt(dma_pte_addr(pte)),
1204                                                        level_pfn, start_pfn, last_pfn,
1205                                                        freelist);
1206                 }
1207 next:
1208                 pfn += level_size(level);
1209         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1210
1211         if (first_pte)
1212                 domain_flush_cache(domain, first_pte,
1213                                    (void *)++last_pte - (void *)first_pte);
1214
1215         return freelist;
1216 }
1217
1218 /* We can't just free the pages because the IOMMU may still be walking
1219    the page tables, and may have cached the intermediate levels. The
1220    pages can only be freed after the IOTLB flush has been done. */
1221 static struct page *domain_unmap(struct dmar_domain *domain,
1222                                  unsigned long start_pfn,
1223                                  unsigned long last_pfn)
1224 {
1225         struct page *freelist = NULL;
1226
1227         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1228         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1229         BUG_ON(start_pfn > last_pfn);
1230
1231         /* we don't need lock here; nobody else touches the iova range */
1232         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1233                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1234
1235         /* free pgd */
1236         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1237                 struct page *pgd_page = virt_to_page(domain->pgd);
1238                 pgd_page->freelist = freelist;
1239                 freelist = pgd_page;
1240
1241                 domain->pgd = NULL;
1242         }
1243
1244         return freelist;
1245 }
1246
1247 static void dma_free_pagelist(struct page *freelist)
1248 {
1249         struct page *pg;
1250
1251         while ((pg = freelist)) {
1252                 freelist = pg->freelist;
1253                 free_pgtable_page(page_address(pg));
1254         }
1255 }
1256
1257 static void iova_entry_free(unsigned long data)
1258 {
1259         struct page *freelist = (struct page *)data;
1260
1261         dma_free_pagelist(freelist);
1262 }
1263
1264 /* iommu handling */
1265 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1266 {
1267         struct root_entry *root;
1268         unsigned long flags;
1269
1270         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1271         if (!root) {
1272                 pr_err("Allocating root entry for %s failed\n",
1273                         iommu->name);
1274                 return -ENOMEM;
1275         }
1276
1277         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1278
1279         spin_lock_irqsave(&iommu->lock, flags);
1280         iommu->root_entry = root;
1281         spin_unlock_irqrestore(&iommu->lock, flags);
1282
1283         return 0;
1284 }
1285
1286 static void iommu_set_root_entry(struct intel_iommu *iommu)
1287 {
1288         u64 addr;
1289         u32 sts;
1290         unsigned long flag;
1291
1292         addr = virt_to_phys(iommu->root_entry);
1293         if (ecs_enabled(iommu))
1294                 addr |= DMA_RTADDR_RTT;
1295
1296         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1297         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1298
1299         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1300
1301         /* Make sure hardware complete it */
1302         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1303                       readl, (sts & DMA_GSTS_RTPS), sts);
1304
1305         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1306 }
1307
1308 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1309 {
1310         u32 val;
1311         unsigned long flag;
1312
1313         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1314                 return;
1315
1316         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1317         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1318
1319         /* Make sure hardware complete it */
1320         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1321                       readl, (!(val & DMA_GSTS_WBFS)), val);
1322
1323         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1324 }
1325
1326 /* return value determine if we need a write buffer flush */
1327 static void __iommu_flush_context(struct intel_iommu *iommu,
1328                                   u16 did, u16 source_id, u8 function_mask,
1329                                   u64 type)
1330 {
1331         u64 val = 0;
1332         unsigned long flag;
1333
1334         switch (type) {
1335         case DMA_CCMD_GLOBAL_INVL:
1336                 val = DMA_CCMD_GLOBAL_INVL;
1337                 break;
1338         case DMA_CCMD_DOMAIN_INVL:
1339                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1340                 break;
1341         case DMA_CCMD_DEVICE_INVL:
1342                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1343                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1344                 break;
1345         default:
1346                 BUG();
1347         }
1348         val |= DMA_CCMD_ICC;
1349
1350         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1351         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1352
1353         /* Make sure hardware complete it */
1354         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1355                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1356
1357         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1358 }
1359
1360 /* return value determine if we need a write buffer flush */
1361 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1362                                 u64 addr, unsigned int size_order, u64 type)
1363 {
1364         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1365         u64 val = 0, val_iva = 0;
1366         unsigned long flag;
1367
1368         switch (type) {
1369         case DMA_TLB_GLOBAL_FLUSH:
1370                 /* global flush doesn't need set IVA_REG */
1371                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1372                 break;
1373         case DMA_TLB_DSI_FLUSH:
1374                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1375                 break;
1376         case DMA_TLB_PSI_FLUSH:
1377                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1378                 /* IH bit is passed in as part of address */
1379                 val_iva = size_order | addr;
1380                 break;
1381         default:
1382                 BUG();
1383         }
1384         /* Note: set drain read/write */
1385 #if 0
1386         /*
1387          * This is probably to be super secure.. Looks like we can
1388          * ignore it without any impact.
1389          */
1390         if (cap_read_drain(iommu->cap))
1391                 val |= DMA_TLB_READ_DRAIN;
1392 #endif
1393         if (cap_write_drain(iommu->cap))
1394                 val |= DMA_TLB_WRITE_DRAIN;
1395
1396         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1397         /* Note: Only uses first TLB reg currently */
1398         if (val_iva)
1399                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1400         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1401
1402         /* Make sure hardware complete it */
1403         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1404                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1405
1406         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1407
1408         /* check IOTLB invalidation granularity */
1409         if (DMA_TLB_IAIG(val) == 0)
1410                 pr_err("Flush IOTLB failed\n");
1411         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1412                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1413                         (unsigned long long)DMA_TLB_IIRG(type),
1414                         (unsigned long long)DMA_TLB_IAIG(val));
1415 }
1416
1417 static struct device_domain_info *
1418 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1419                          u8 bus, u8 devfn)
1420 {
1421         struct device_domain_info *info;
1422
1423         assert_spin_locked(&device_domain_lock);
1424
1425         if (!iommu->qi)
1426                 return NULL;
1427
1428         list_for_each_entry(info, &domain->devices, link)
1429                 if (info->iommu == iommu && info->bus == bus &&
1430                     info->devfn == devfn) {
1431                         if (info->ats_supported && info->dev)
1432                                 return info;
1433                         break;
1434                 }
1435
1436         return NULL;
1437 }
1438
1439 static void domain_update_iotlb(struct dmar_domain *domain)
1440 {
1441         struct device_domain_info *info;
1442         bool has_iotlb_device = false;
1443
1444         assert_spin_locked(&device_domain_lock);
1445
1446         list_for_each_entry(info, &domain->devices, link) {
1447                 struct pci_dev *pdev;
1448
1449                 if (!info->dev || !dev_is_pci(info->dev))
1450                         continue;
1451
1452                 pdev = to_pci_dev(info->dev);
1453                 if (pdev->ats_enabled) {
1454                         has_iotlb_device = true;
1455                         break;
1456                 }
1457         }
1458
1459         domain->has_iotlb_device = has_iotlb_device;
1460 }
1461
1462 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1463 {
1464         struct pci_dev *pdev;
1465
1466         assert_spin_locked(&device_domain_lock);
1467
1468         if (!info || !dev_is_pci(info->dev))
1469                 return;
1470
1471         pdev = to_pci_dev(info->dev);
1472         /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1473          * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1474          * queue depth at PF level. If DIT is not set, PFSID will be treated as
1475          * reserved, which should be set to 0.
1476          */
1477         if (!ecap_dit(info->iommu->ecap))
1478                 info->pfsid = 0;
1479         else {
1480                 struct pci_dev *pf_pdev;
1481
1482                 /* pdev will be returned if device is not a vf */
1483                 pf_pdev = pci_physfn(pdev);
1484                 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1485         }
1486
1487 #ifdef CONFIG_INTEL_IOMMU_SVM
1488         /* The PCIe spec, in its wisdom, declares that the behaviour of
1489            the device if you enable PASID support after ATS support is
1490            undefined. So always enable PASID support on devices which
1491            have it, even if we can't yet know if we're ever going to
1492            use it. */
1493         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1494                 info->pasid_enabled = 1;
1495
1496         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1497                 info->pri_enabled = 1;
1498 #endif
1499         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1500                 info->ats_enabled = 1;
1501                 domain_update_iotlb(info->domain);
1502                 info->ats_qdep = pci_ats_queue_depth(pdev);
1503         }
1504 }
1505
1506 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1507 {
1508         struct pci_dev *pdev;
1509
1510         assert_spin_locked(&device_domain_lock);
1511
1512         if (!dev_is_pci(info->dev))
1513                 return;
1514
1515         pdev = to_pci_dev(info->dev);
1516
1517         if (info->ats_enabled) {
1518                 pci_disable_ats(pdev);
1519                 info->ats_enabled = 0;
1520                 domain_update_iotlb(info->domain);
1521         }
1522 #ifdef CONFIG_INTEL_IOMMU_SVM
1523         if (info->pri_enabled) {
1524                 pci_disable_pri(pdev);
1525                 info->pri_enabled = 0;
1526         }
1527         if (info->pasid_enabled) {
1528                 pci_disable_pasid(pdev);
1529                 info->pasid_enabled = 0;
1530         }
1531 #endif
1532 }
1533
1534 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1535                                   u64 addr, unsigned mask)
1536 {
1537         u16 sid, qdep;
1538         unsigned long flags;
1539         struct device_domain_info *info;
1540
1541         if (!domain->has_iotlb_device)
1542                 return;
1543
1544         spin_lock_irqsave(&device_domain_lock, flags);
1545         list_for_each_entry(info, &domain->devices, link) {
1546                 if (!info->ats_enabled)
1547                         continue;
1548
1549                 sid = info->bus << 8 | info->devfn;
1550                 qdep = info->ats_qdep;
1551                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1552                                 qdep, addr, mask);
1553         }
1554         spin_unlock_irqrestore(&device_domain_lock, flags);
1555 }
1556
1557 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1558                                   struct dmar_domain *domain,
1559                                   unsigned long pfn, unsigned int pages,
1560                                   int ih, int map)
1561 {
1562         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1563         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1564         u16 did = domain->iommu_did[iommu->seq_id];
1565
1566         BUG_ON(pages == 0);
1567
1568         if (ih)
1569                 ih = 1 << 6;
1570         /*
1571          * Fallback to domain selective flush if no PSI support or the size is
1572          * too big.
1573          * PSI requires page size to be 2 ^ x, and the base address is naturally
1574          * aligned to the size
1575          */
1576         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1577                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1578                                                 DMA_TLB_DSI_FLUSH);
1579         else
1580                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1581                                                 DMA_TLB_PSI_FLUSH);
1582
1583         /*
1584          * In caching mode, changes of pages from non-present to present require
1585          * flush. However, device IOTLB doesn't need to be flushed in this case.
1586          */
1587         if (!cap_caching_mode(iommu->cap) || !map)
1588                 iommu_flush_dev_iotlb(domain, addr, mask);
1589 }
1590
1591 /* Notification for newly created mappings */
1592 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1593                                         struct dmar_domain *domain,
1594                                         unsigned long pfn, unsigned int pages)
1595 {
1596         /* It's a non-present to present mapping. Only flush if caching mode */
1597         if (cap_caching_mode(iommu->cap))
1598                 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1599         else
1600                 iommu_flush_write_buffer(iommu);
1601 }
1602
1603 static void iommu_flush_iova(struct iova_domain *iovad)
1604 {
1605         struct dmar_domain *domain;
1606         int idx;
1607
1608         domain = container_of(iovad, struct dmar_domain, iovad);
1609
1610         for_each_domain_iommu(idx, domain) {
1611                 struct intel_iommu *iommu = g_iommus[idx];
1612                 u16 did = domain->iommu_did[iommu->seq_id];
1613
1614                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1615
1616                 if (!cap_caching_mode(iommu->cap))
1617                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1618                                               0, MAX_AGAW_PFN_WIDTH);
1619         }
1620 }
1621
1622 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1623 {
1624         u32 pmen;
1625         unsigned long flags;
1626
1627         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1628         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1629         pmen &= ~DMA_PMEN_EPM;
1630         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1631
1632         /* wait for the protected region status bit to clear */
1633         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1634                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1635
1636         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637 }
1638
1639 static void iommu_enable_translation(struct intel_iommu *iommu)
1640 {
1641         u32 sts;
1642         unsigned long flags;
1643
1644         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1645         iommu->gcmd |= DMA_GCMD_TE;
1646         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648         /* Make sure hardware complete it */
1649         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1650                       readl, (sts & DMA_GSTS_TES), sts);
1651
1652         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1653 }
1654
1655 static void iommu_disable_translation(struct intel_iommu *iommu)
1656 {
1657         u32 sts;
1658         unsigned long flag;
1659
1660         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1661         iommu->gcmd &= ~DMA_GCMD_TE;
1662         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1663
1664         /* Make sure hardware complete it */
1665         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1666                       readl, (!(sts & DMA_GSTS_TES)), sts);
1667
1668         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1669 }
1670
1671
1672 static int iommu_init_domains(struct intel_iommu *iommu)
1673 {
1674         u32 ndomains, nlongs;
1675         size_t size;
1676
1677         ndomains = cap_ndoms(iommu->cap);
1678         pr_debug("%s: Number of Domains supported <%d>\n",
1679                  iommu->name, ndomains);
1680         nlongs = BITS_TO_LONGS(ndomains);
1681
1682         spin_lock_init(&iommu->lock);
1683
1684         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1685         if (!iommu->domain_ids) {
1686                 pr_err("%s: Allocating domain id array failed\n",
1687                        iommu->name);
1688                 return -ENOMEM;
1689         }
1690
1691         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1692         iommu->domains = kzalloc(size, GFP_KERNEL);
1693
1694         if (iommu->domains) {
1695                 size = 256 * sizeof(struct dmar_domain *);
1696                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1697         }
1698
1699         if (!iommu->domains || !iommu->domains[0]) {
1700                 pr_err("%s: Allocating domain array failed\n",
1701                        iommu->name);
1702                 kfree(iommu->domain_ids);
1703                 kfree(iommu->domains);
1704                 iommu->domain_ids = NULL;
1705                 iommu->domains    = NULL;
1706                 return -ENOMEM;
1707         }
1708
1709
1710
1711         /*
1712          * If Caching mode is set, then invalid translations are tagged
1713          * with domain-id 0, hence we need to pre-allocate it. We also
1714          * use domain-id 0 as a marker for non-allocated domain-id, so
1715          * make sure it is not used for a real domain.
1716          */
1717         set_bit(0, iommu->domain_ids);
1718
1719         return 0;
1720 }
1721
1722 static void disable_dmar_iommu(struct intel_iommu *iommu)
1723 {
1724         struct device_domain_info *info, *tmp;
1725         unsigned long flags;
1726
1727         if (!iommu->domains || !iommu->domain_ids)
1728                 return;
1729
1730 again:
1731         spin_lock_irqsave(&device_domain_lock, flags);
1732         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1733                 struct dmar_domain *domain;
1734
1735                 if (info->iommu != iommu)
1736                         continue;
1737
1738                 if (!info->dev || !info->domain)
1739                         continue;
1740
1741                 domain = info->domain;
1742
1743                 __dmar_remove_one_dev_info(info);
1744
1745                 if (!domain_type_is_vm_or_si(domain)) {
1746                         /*
1747                          * The domain_exit() function  can't be called under
1748                          * device_domain_lock, as it takes this lock itself.
1749                          * So release the lock here and re-run the loop
1750                          * afterwards.
1751                          */
1752                         spin_unlock_irqrestore(&device_domain_lock, flags);
1753                         domain_exit(domain);
1754                         goto again;
1755                 }
1756         }
1757         spin_unlock_irqrestore(&device_domain_lock, flags);
1758
1759         if (iommu->gcmd & DMA_GCMD_TE)
1760                 iommu_disable_translation(iommu);
1761 }
1762
1763 static void free_dmar_iommu(struct intel_iommu *iommu)
1764 {
1765         if ((iommu->domains) && (iommu->domain_ids)) {
1766                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1767                 int i;
1768
1769                 for (i = 0; i < elems; i++)
1770                         kfree(iommu->domains[i]);
1771                 kfree(iommu->domains);
1772                 kfree(iommu->domain_ids);
1773                 iommu->domains = NULL;
1774                 iommu->domain_ids = NULL;
1775         }
1776
1777         g_iommus[iommu->seq_id] = NULL;
1778
1779         /* free context mapping */
1780         free_context_table(iommu);
1781
1782 #ifdef CONFIG_INTEL_IOMMU_SVM
1783         if (pasid_enabled(iommu)) {
1784                 if (ecap_prs(iommu->ecap))
1785                         intel_svm_finish_prq(iommu);
1786                 intel_svm_exit(iommu);
1787         }
1788 #endif
1789 }
1790
1791 static struct dmar_domain *alloc_domain(int flags)
1792 {
1793         struct dmar_domain *domain;
1794
1795         domain = alloc_domain_mem();
1796         if (!domain)
1797                 return NULL;
1798
1799         memset(domain, 0, sizeof(*domain));
1800         domain->nid = -1;
1801         domain->flags = flags;
1802         domain->has_iotlb_device = false;
1803         INIT_LIST_HEAD(&domain->devices);
1804
1805         return domain;
1806 }
1807
1808 /* Must be called with iommu->lock */
1809 static int domain_attach_iommu(struct dmar_domain *domain,
1810                                struct intel_iommu *iommu)
1811 {
1812         unsigned long ndomains;
1813         int num;
1814
1815         assert_spin_locked(&device_domain_lock);
1816         assert_spin_locked(&iommu->lock);
1817
1818         domain->iommu_refcnt[iommu->seq_id] += 1;
1819         domain->iommu_count += 1;
1820         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1821                 ndomains = cap_ndoms(iommu->cap);
1822                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1823
1824                 if (num >= ndomains) {
1825                         pr_err("%s: No free domain ids\n", iommu->name);
1826                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1827                         domain->iommu_count -= 1;
1828                         return -ENOSPC;
1829                 }
1830
1831                 set_bit(num, iommu->domain_ids);
1832                 set_iommu_domain(iommu, num, domain);
1833
1834                 domain->iommu_did[iommu->seq_id] = num;
1835                 domain->nid                      = iommu->node;
1836
1837                 domain_update_iommu_cap(domain);
1838         }
1839
1840         return 0;
1841 }
1842
1843 static int domain_detach_iommu(struct dmar_domain *domain,
1844                                struct intel_iommu *iommu)
1845 {
1846         int num, count = INT_MAX;
1847
1848         assert_spin_locked(&device_domain_lock);
1849         assert_spin_locked(&iommu->lock);
1850
1851         domain->iommu_refcnt[iommu->seq_id] -= 1;
1852         count = --domain->iommu_count;
1853         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1854                 num = domain->iommu_did[iommu->seq_id];
1855                 clear_bit(num, iommu->domain_ids);
1856                 set_iommu_domain(iommu, num, NULL);
1857
1858                 domain_update_iommu_cap(domain);
1859                 domain->iommu_did[iommu->seq_id] = 0;
1860         }
1861
1862         return count;
1863 }
1864
1865 static struct iova_domain reserved_iova_list;
1866 static struct lock_class_key reserved_rbtree_key;
1867
1868 static int dmar_init_reserved_ranges(void)
1869 {
1870         struct pci_dev *pdev = NULL;
1871         struct iova *iova;
1872         int i;
1873
1874         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1875
1876         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1877                 &reserved_rbtree_key);
1878
1879         /* IOAPIC ranges shouldn't be accessed by DMA */
1880         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1881                 IOVA_PFN(IOAPIC_RANGE_END));
1882         if (!iova) {
1883                 pr_err("Reserve IOAPIC range failed\n");
1884                 return -ENODEV;
1885         }
1886
1887         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1888         for_each_pci_dev(pdev) {
1889                 struct resource *r;
1890
1891                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1892                         r = &pdev->resource[i];
1893                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1894                                 continue;
1895                         iova = reserve_iova(&reserved_iova_list,
1896                                             IOVA_PFN(r->start),
1897                                             IOVA_PFN(r->end));
1898                         if (!iova) {
1899                                 pr_err("Reserve iova failed\n");
1900                                 return -ENODEV;
1901                         }
1902                 }
1903         }
1904         return 0;
1905 }
1906
1907 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1908 {
1909         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1910 }
1911
1912 static inline int guestwidth_to_adjustwidth(int gaw)
1913 {
1914         int agaw;
1915         int r = (gaw - 12) % 9;
1916
1917         if (r == 0)
1918                 agaw = gaw;
1919         else
1920                 agaw = gaw + 9 - r;
1921         if (agaw > 64)
1922                 agaw = 64;
1923         return agaw;
1924 }
1925
1926 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1927                        int guest_width)
1928 {
1929         int adjust_width, agaw;
1930         unsigned long sagaw;
1931         int err;
1932
1933         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1934
1935         err = init_iova_flush_queue(&domain->iovad,
1936                                     iommu_flush_iova, iova_entry_free);
1937         if (err)
1938                 return err;
1939
1940         domain_reserve_special_ranges(domain);
1941
1942         /* calculate AGAW */
1943         if (guest_width > cap_mgaw(iommu->cap))
1944                 guest_width = cap_mgaw(iommu->cap);
1945         domain->gaw = guest_width;
1946         adjust_width = guestwidth_to_adjustwidth(guest_width);
1947         agaw = width_to_agaw(adjust_width);
1948         sagaw = cap_sagaw(iommu->cap);
1949         if (!test_bit(agaw, &sagaw)) {
1950                 /* hardware doesn't support it, choose a bigger one */
1951                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1952                 agaw = find_next_bit(&sagaw, 5, agaw);
1953                 if (agaw >= 5)
1954                         return -ENODEV;
1955         }
1956         domain->agaw = agaw;
1957
1958         if (ecap_coherent(iommu->ecap))
1959                 domain->iommu_coherency = 1;
1960         else
1961                 domain->iommu_coherency = 0;
1962
1963         if (ecap_sc_support(iommu->ecap))
1964                 domain->iommu_snooping = 1;
1965         else
1966                 domain->iommu_snooping = 0;
1967
1968         if (intel_iommu_superpage)
1969                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1970         else
1971                 domain->iommu_superpage = 0;
1972
1973         domain->nid = iommu->node;
1974
1975         /* always allocate the top pgd */
1976         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1977         if (!domain->pgd)
1978                 return -ENOMEM;
1979         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1980         return 0;
1981 }
1982
1983 static void domain_exit(struct dmar_domain *domain)
1984 {
1985         struct page *freelist = NULL;
1986
1987         /* Domain 0 is reserved, so dont process it */
1988         if (!domain)
1989                 return;
1990
1991         /* Remove associated devices and clear attached or cached domains */
1992         rcu_read_lock();
1993         domain_remove_dev_info(domain);
1994         rcu_read_unlock();
1995
1996         /* destroy iovas */
1997         put_iova_domain(&domain->iovad);
1998
1999         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2000
2001         dma_free_pagelist(freelist);
2002
2003         free_domain_mem(domain);
2004 }
2005
2006 static int domain_context_mapping_one(struct dmar_domain *domain,
2007                                       struct intel_iommu *iommu,
2008                                       u8 bus, u8 devfn)
2009 {
2010         u16 did = domain->iommu_did[iommu->seq_id];
2011         int translation = CONTEXT_TT_MULTI_LEVEL;
2012         struct device_domain_info *info = NULL;
2013         struct context_entry *context;
2014         unsigned long flags;
2015         struct dma_pte *pgd;
2016         int ret, agaw;
2017
2018         WARN_ON(did == 0);
2019
2020         if (hw_pass_through && domain_type_is_si(domain))
2021                 translation = CONTEXT_TT_PASS_THROUGH;
2022
2023         pr_debug("Set context mapping for %02x:%02x.%d\n",
2024                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2025
2026         BUG_ON(!domain->pgd);
2027
2028         spin_lock_irqsave(&device_domain_lock, flags);
2029         spin_lock(&iommu->lock);
2030
2031         ret = -ENOMEM;
2032         context = iommu_context_addr(iommu, bus, devfn, 1);
2033         if (!context)
2034                 goto out_unlock;
2035
2036         ret = 0;
2037         if (context_present(context))
2038                 goto out_unlock;
2039
2040         /*
2041          * For kdump cases, old valid entries may be cached due to the
2042          * in-flight DMA and copied pgtable, but there is no unmapping
2043          * behaviour for them, thus we need an explicit cache flush for
2044          * the newly-mapped device. For kdump, at this point, the device
2045          * is supposed to finish reset at its driver probe stage, so no
2046          * in-flight DMA will exist, and we don't need to worry anymore
2047          * hereafter.
2048          */
2049         if (context_copied(context)) {
2050                 u16 did_old = context_domain_id(context);
2051
2052                 if (did_old < cap_ndoms(iommu->cap)) {
2053                         iommu->flush.flush_context(iommu, did_old,
2054                                                    (((u16)bus) << 8) | devfn,
2055                                                    DMA_CCMD_MASK_NOBIT,
2056                                                    DMA_CCMD_DEVICE_INVL);
2057                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2058                                                  DMA_TLB_DSI_FLUSH);
2059                 }
2060         }
2061
2062         pgd = domain->pgd;
2063
2064         context_clear_entry(context);
2065         context_set_domain_id(context, did);
2066
2067         /*
2068          * Skip top levels of page tables for iommu which has less agaw
2069          * than default.  Unnecessary for PT mode.
2070          */
2071         if (translation != CONTEXT_TT_PASS_THROUGH) {
2072                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2073                         ret = -ENOMEM;
2074                         pgd = phys_to_virt(dma_pte_addr(pgd));
2075                         if (!dma_pte_present(pgd))
2076                                 goto out_unlock;
2077                 }
2078
2079                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2080                 if (info && info->ats_supported)
2081                         translation = CONTEXT_TT_DEV_IOTLB;
2082                 else
2083                         translation = CONTEXT_TT_MULTI_LEVEL;
2084
2085                 context_set_address_root(context, virt_to_phys(pgd));
2086                 context_set_address_width(context, iommu->agaw);
2087         } else {
2088                 /*
2089                  * In pass through mode, AW must be programmed to
2090                  * indicate the largest AGAW value supported by
2091                  * hardware. And ASR is ignored by hardware.
2092                  */
2093                 context_set_address_width(context, iommu->msagaw);
2094         }
2095
2096         context_set_translation_type(context, translation);
2097         context_set_fault_enable(context);
2098         context_set_present(context);
2099         domain_flush_cache(domain, context, sizeof(*context));
2100
2101         /*
2102          * It's a non-present to present mapping. If hardware doesn't cache
2103          * non-present entry we only need to flush the write-buffer. If the
2104          * _does_ cache non-present entries, then it does so in the special
2105          * domain #0, which we have to flush:
2106          */
2107         if (cap_caching_mode(iommu->cap)) {
2108                 iommu->flush.flush_context(iommu, 0,
2109                                            (((u16)bus) << 8) | devfn,
2110                                            DMA_CCMD_MASK_NOBIT,
2111                                            DMA_CCMD_DEVICE_INVL);
2112                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2113         } else {
2114                 iommu_flush_write_buffer(iommu);
2115         }
2116         iommu_enable_dev_iotlb(info);
2117
2118         ret = 0;
2119
2120 out_unlock:
2121         spin_unlock(&iommu->lock);
2122         spin_unlock_irqrestore(&device_domain_lock, flags);
2123
2124         return ret;
2125 }
2126
2127 struct domain_context_mapping_data {
2128         struct dmar_domain *domain;
2129         struct intel_iommu *iommu;
2130 };
2131
2132 static int domain_context_mapping_cb(struct pci_dev *pdev,
2133                                      u16 alias, void *opaque)
2134 {
2135         struct domain_context_mapping_data *data = opaque;
2136
2137         return domain_context_mapping_one(data->domain, data->iommu,
2138                                           PCI_BUS_NUM(alias), alias & 0xff);
2139 }
2140
2141 static int
2142 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2143 {
2144         struct intel_iommu *iommu;
2145         u8 bus, devfn;
2146         struct domain_context_mapping_data data;
2147
2148         iommu = device_to_iommu(dev, &bus, &devfn);
2149         if (!iommu)
2150                 return -ENODEV;
2151
2152         if (!dev_is_pci(dev))
2153                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2154
2155         data.domain = domain;
2156         data.iommu = iommu;
2157
2158         return pci_for_each_dma_alias(to_pci_dev(dev),
2159                                       &domain_context_mapping_cb, &data);
2160 }
2161
2162 static int domain_context_mapped_cb(struct pci_dev *pdev,
2163                                     u16 alias, void *opaque)
2164 {
2165         struct intel_iommu *iommu = opaque;
2166
2167         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2168 }
2169
2170 static int domain_context_mapped(struct device *dev)
2171 {
2172         struct intel_iommu *iommu;
2173         u8 bus, devfn;
2174
2175         iommu = device_to_iommu(dev, &bus, &devfn);
2176         if (!iommu)
2177                 return -ENODEV;
2178
2179         if (!dev_is_pci(dev))
2180                 return device_context_mapped(iommu, bus, devfn);
2181
2182         return !pci_for_each_dma_alias(to_pci_dev(dev),
2183                                        domain_context_mapped_cb, iommu);
2184 }
2185
2186 /* Returns a number of VTD pages, but aligned to MM page size */
2187 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2188                                             size_t size)
2189 {
2190         host_addr &= ~PAGE_MASK;
2191         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2192 }
2193
2194 /* Return largest possible superpage level for a given mapping */
2195 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2196                                           unsigned long iov_pfn,
2197                                           unsigned long phy_pfn,
2198                                           unsigned long pages)
2199 {
2200         int support, level = 1;
2201         unsigned long pfnmerge;
2202
2203         support = domain->iommu_superpage;
2204
2205         /* To use a large page, the virtual *and* physical addresses
2206            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2207            of them will mean we have to use smaller pages. So just
2208            merge them and check both at once. */
2209         pfnmerge = iov_pfn | phy_pfn;
2210
2211         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2212                 pages >>= VTD_STRIDE_SHIFT;
2213                 if (!pages)
2214                         break;
2215                 pfnmerge >>= VTD_STRIDE_SHIFT;
2216                 level++;
2217                 support--;
2218         }
2219         return level;
2220 }
2221
2222 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2223                             struct scatterlist *sg, unsigned long phys_pfn,
2224                             unsigned long nr_pages, int prot)
2225 {
2226         struct dma_pte *first_pte = NULL, *pte = NULL;
2227         phys_addr_t uninitialized_var(pteval);
2228         unsigned long sg_res = 0;
2229         unsigned int largepage_lvl = 0;
2230         unsigned long lvl_pages = 0;
2231
2232         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2233
2234         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2235                 return -EINVAL;
2236
2237         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2238
2239         if (!sg) {
2240                 sg_res = nr_pages;
2241                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2242         }
2243
2244         while (nr_pages > 0) {
2245                 uint64_t tmp;
2246
2247                 if (!sg_res) {
2248                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2249
2250                         sg_res = aligned_nrpages(sg->offset, sg->length);
2251                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2252                         sg->dma_length = sg->length;
2253                         pteval = (sg_phys(sg) - pgoff) | prot;
2254                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2255                 }
2256
2257                 if (!pte) {
2258                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2259
2260                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2261                         if (!pte)
2262                                 return -ENOMEM;
2263                         /* It is large page*/
2264                         if (largepage_lvl > 1) {
2265                                 unsigned long nr_superpages, end_pfn;
2266
2267                                 pteval |= DMA_PTE_LARGE_PAGE;
2268                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2269
2270                                 nr_superpages = sg_res / lvl_pages;
2271                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2272
2273                                 /*
2274                                  * Ensure that old small page tables are
2275                                  * removed to make room for superpage(s).
2276                                  * We're adding new large pages, so make sure
2277                                  * we don't remove their parent tables.
2278                                  */
2279                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2280                                                        largepage_lvl + 1);
2281                         } else {
2282                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2283                         }
2284
2285                 }
2286                 /* We don't need lock here, nobody else
2287                  * touches the iova range
2288                  */
2289                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2290                 if (tmp) {
2291                         static int dumps = 5;
2292                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2293                                 iov_pfn, tmp, (unsigned long long)pteval);
2294                         if (dumps) {
2295                                 dumps--;
2296                                 debug_dma_dump_mappings(NULL);
2297                         }
2298                         WARN_ON(1);
2299                 }
2300
2301                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2302
2303                 BUG_ON(nr_pages < lvl_pages);
2304                 BUG_ON(sg_res < lvl_pages);
2305
2306                 nr_pages -= lvl_pages;
2307                 iov_pfn += lvl_pages;
2308                 phys_pfn += lvl_pages;
2309                 pteval += lvl_pages * VTD_PAGE_SIZE;
2310                 sg_res -= lvl_pages;
2311
2312                 /* If the next PTE would be the first in a new page, then we
2313                    need to flush the cache on the entries we've just written.
2314                    And then we'll need to recalculate 'pte', so clear it and
2315                    let it get set again in the if (!pte) block above.
2316
2317                    If we're done (!nr_pages) we need to flush the cache too.
2318
2319                    Also if we've been setting superpages, we may need to
2320                    recalculate 'pte' and switch back to smaller pages for the
2321                    end of the mapping, if the trailing size is not enough to
2322                    use another superpage (i.e. sg_res < lvl_pages). */
2323                 pte++;
2324                 if (!nr_pages || first_pte_in_page(pte) ||
2325                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2326                         domain_flush_cache(domain, first_pte,
2327                                            (void *)pte - (void *)first_pte);
2328                         pte = NULL;
2329                 }
2330
2331                 if (!sg_res && nr_pages)
2332                         sg = sg_next(sg);
2333         }
2334         return 0;
2335 }
2336
2337 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2338                          struct scatterlist *sg, unsigned long phys_pfn,
2339                          unsigned long nr_pages, int prot)
2340 {
2341        int ret;
2342        struct intel_iommu *iommu;
2343
2344        /* Do the real mapping first */
2345        ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2346        if (ret)
2347                return ret;
2348
2349        /* Notify about the new mapping */
2350        if (domain_type_is_vm(domain)) {
2351                /* VM typed domains can have more than one IOMMUs */
2352                int iommu_id;
2353                for_each_domain_iommu(iommu_id, domain) {
2354                        iommu = g_iommus[iommu_id];
2355                        __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2356                }
2357        } else {
2358                /* General domains only have one IOMMU */
2359                iommu = domain_get_iommu(domain);
2360                __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2361        }
2362
2363        return 0;
2364 }
2365
2366 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2367                                     struct scatterlist *sg, unsigned long nr_pages,
2368                                     int prot)
2369 {
2370         return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2371 }
2372
2373 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2374                                      unsigned long phys_pfn, unsigned long nr_pages,
2375                                      int prot)
2376 {
2377         return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2378 }
2379
2380 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2381 {
2382         unsigned long flags;
2383         struct context_entry *context;
2384         u16 did_old;
2385
2386         if (!iommu)
2387                 return;
2388
2389         spin_lock_irqsave(&iommu->lock, flags);
2390         context = iommu_context_addr(iommu, bus, devfn, 0);
2391         if (!context) {
2392                 spin_unlock_irqrestore(&iommu->lock, flags);
2393                 return;
2394         }
2395         did_old = context_domain_id(context);
2396         context_clear_entry(context);
2397         __iommu_flush_cache(iommu, context, sizeof(*context));
2398         spin_unlock_irqrestore(&iommu->lock, flags);
2399         iommu->flush.flush_context(iommu,
2400                                    did_old,
2401                                    (((u16)bus) << 8) | devfn,
2402                                    DMA_CCMD_MASK_NOBIT,
2403                                    DMA_CCMD_DEVICE_INVL);
2404         iommu->flush.flush_iotlb(iommu,
2405                                  did_old,
2406                                  0,
2407                                  0,
2408                                  DMA_TLB_DSI_FLUSH);
2409 }
2410
2411 static inline void unlink_domain_info(struct device_domain_info *info)
2412 {
2413         assert_spin_locked(&device_domain_lock);
2414         list_del(&info->link);
2415         list_del(&info->global);
2416         if (info->dev)
2417                 info->dev->archdata.iommu = NULL;
2418 }
2419
2420 static void domain_remove_dev_info(struct dmar_domain *domain)
2421 {
2422         struct device_domain_info *info, *tmp;
2423         unsigned long flags;
2424
2425         spin_lock_irqsave(&device_domain_lock, flags);
2426         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2427                 __dmar_remove_one_dev_info(info);
2428         spin_unlock_irqrestore(&device_domain_lock, flags);
2429 }
2430
2431 /*
2432  * find_domain
2433  * Note: we use struct device->archdata.iommu stores the info
2434  */
2435 static struct dmar_domain *find_domain(struct device *dev)
2436 {
2437         struct device_domain_info *info;
2438
2439         /* No lock here, assumes no domain exit in normal case */
2440         info = dev->archdata.iommu;
2441         if (likely(info))
2442                 return info->domain;
2443         return NULL;
2444 }
2445
2446 static inline struct device_domain_info *
2447 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2448 {
2449         struct device_domain_info *info;
2450
2451         list_for_each_entry(info, &device_domain_list, global)
2452                 if (info->iommu->segment == segment && info->bus == bus &&
2453                     info->devfn == devfn)
2454                         return info;
2455
2456         return NULL;
2457 }
2458
2459 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2460                                                     int bus, int devfn,
2461                                                     struct device *dev,
2462                                                     struct dmar_domain *domain)
2463 {
2464         struct dmar_domain *found = NULL;
2465         struct device_domain_info *info;
2466         unsigned long flags;
2467         int ret;
2468
2469         info = alloc_devinfo_mem();
2470         if (!info)
2471                 return NULL;
2472
2473         info->bus = bus;
2474         info->devfn = devfn;
2475         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2476         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2477         info->ats_qdep = 0;
2478         info->dev = dev;
2479         info->domain = domain;
2480         info->iommu = iommu;
2481         info->pasid_table = NULL;
2482
2483         if (dev && dev_is_pci(dev)) {
2484                 struct pci_dev *pdev = to_pci_dev(info->dev);
2485
2486                 if (!pci_ats_disabled() &&
2487                     ecap_dev_iotlb_support(iommu->ecap) &&
2488                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2489                     dmar_find_matched_atsr_unit(pdev))
2490                         info->ats_supported = 1;
2491
2492                 if (ecs_enabled(iommu)) {
2493                         if (pasid_enabled(iommu)) {
2494                                 int features = pci_pasid_features(pdev);
2495                                 if (features >= 0)
2496                                         info->pasid_supported = features | 1;
2497                         }
2498
2499                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2500                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2501                                 info->pri_supported = 1;
2502                 }
2503         }
2504
2505         spin_lock_irqsave(&device_domain_lock, flags);
2506         if (dev)
2507                 found = find_domain(dev);
2508
2509         if (!found) {
2510                 struct device_domain_info *info2;
2511                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2512                 if (info2) {
2513                         found      = info2->domain;
2514                         info2->dev = dev;
2515                 }
2516         }
2517
2518         if (found) {
2519                 spin_unlock_irqrestore(&device_domain_lock, flags);
2520                 free_devinfo_mem(info);
2521                 /* Caller must free the original domain */
2522                 return found;
2523         }
2524
2525         spin_lock(&iommu->lock);
2526         ret = domain_attach_iommu(domain, iommu);
2527         spin_unlock(&iommu->lock);
2528
2529         if (ret) {
2530                 spin_unlock_irqrestore(&device_domain_lock, flags);
2531                 free_devinfo_mem(info);
2532                 return NULL;
2533         }
2534
2535         list_add(&info->link, &domain->devices);
2536         list_add(&info->global, &device_domain_list);
2537         if (dev)
2538                 dev->archdata.iommu = info;
2539
2540         if (dev && dev_is_pci(dev) && info->pasid_supported) {
2541                 ret = intel_pasid_alloc_table(dev);
2542                 if (ret) {
2543                         __dmar_remove_one_dev_info(info);
2544                         spin_unlock_irqrestore(&device_domain_lock, flags);
2545                         return NULL;
2546                 }
2547         }
2548         spin_unlock_irqrestore(&device_domain_lock, flags);
2549
2550         if (dev && domain_context_mapping(domain, dev)) {
2551                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2552                 dmar_remove_one_dev_info(domain, dev);
2553                 return NULL;
2554         }
2555
2556         return domain;
2557 }
2558
2559 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2560 {
2561         *(u16 *)opaque = alias;
2562         return 0;
2563 }
2564
2565 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2566 {
2567         struct device_domain_info *info = NULL;
2568         struct dmar_domain *domain = NULL;
2569         struct intel_iommu *iommu;
2570         u16 dma_alias;
2571         unsigned long flags;
2572         u8 bus, devfn;
2573
2574         iommu = device_to_iommu(dev, &bus, &devfn);
2575         if (!iommu)
2576                 return NULL;
2577
2578         if (dev_is_pci(dev)) {
2579                 struct pci_dev *pdev = to_pci_dev(dev);
2580
2581                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2582
2583                 spin_lock_irqsave(&device_domain_lock, flags);
2584                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2585                                                       PCI_BUS_NUM(dma_alias),
2586                                                       dma_alias & 0xff);
2587                 if (info) {
2588                         iommu = info->iommu;
2589                         domain = info->domain;
2590                 }
2591                 spin_unlock_irqrestore(&device_domain_lock, flags);
2592
2593                 /* DMA alias already has a domain, use it */
2594                 if (info)
2595                         goto out;
2596         }
2597
2598         /* Allocate and initialize new domain for the device */
2599         domain = alloc_domain(0);
2600         if (!domain)
2601                 return NULL;
2602         if (domain_init(domain, iommu, gaw)) {
2603                 domain_exit(domain);
2604                 return NULL;
2605         }
2606
2607 out:
2608
2609         return domain;
2610 }
2611
2612 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2613                                               struct dmar_domain *domain)
2614 {
2615         struct intel_iommu *iommu;
2616         struct dmar_domain *tmp;
2617         u16 req_id, dma_alias;
2618         u8 bus, devfn;
2619
2620         iommu = device_to_iommu(dev, &bus, &devfn);
2621         if (!iommu)
2622                 return NULL;
2623
2624         req_id = ((u16)bus << 8) | devfn;
2625
2626         if (dev_is_pci(dev)) {
2627                 struct pci_dev *pdev = to_pci_dev(dev);
2628
2629                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2630
2631                 /* register PCI DMA alias device */
2632                 if (req_id != dma_alias) {
2633                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2634                                         dma_alias & 0xff, NULL, domain);
2635
2636                         if (!tmp || tmp != domain)
2637                                 return tmp;
2638                 }
2639         }
2640
2641         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2642         if (!tmp || tmp != domain)
2643                 return tmp;
2644
2645         return domain;
2646 }
2647
2648 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2649 {
2650         struct dmar_domain *domain, *tmp;
2651
2652         domain = find_domain(dev);
2653         if (domain)
2654                 goto out;
2655
2656         domain = find_or_alloc_domain(dev, gaw);
2657         if (!domain)
2658                 goto out;
2659
2660         tmp = set_domain_for_dev(dev, domain);
2661         if (!tmp || domain != tmp) {
2662                 domain_exit(domain);
2663                 domain = tmp;
2664         }
2665
2666 out:
2667
2668         return domain;
2669 }
2670
2671 static int iommu_domain_identity_map(struct dmar_domain *domain,
2672                                      unsigned long long start,
2673                                      unsigned long long end)
2674 {
2675         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2676         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2677
2678         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2679                           dma_to_mm_pfn(last_vpfn))) {
2680                 pr_err("Reserving iova failed\n");
2681                 return -ENOMEM;
2682         }
2683
2684         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2685         /*
2686          * RMRR range might have overlap with physical memory range,
2687          * clear it first
2688          */
2689         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2690
2691         return __domain_mapping(domain, first_vpfn, NULL,
2692                                 first_vpfn, last_vpfn - first_vpfn + 1,
2693                                 DMA_PTE_READ|DMA_PTE_WRITE);
2694 }
2695
2696 static int domain_prepare_identity_map(struct device *dev,
2697                                        struct dmar_domain *domain,
2698                                        unsigned long long start,
2699                                        unsigned long long end)
2700 {
2701         /* For _hardware_ passthrough, don't bother. But for software
2702            passthrough, we do it anyway -- it may indicate a memory
2703            range which is reserved in E820, so which didn't get set
2704            up to start with in si_domain */
2705         if (domain == si_domain && hw_pass_through) {
2706                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2707                         dev_name(dev), start, end);
2708                 return 0;
2709         }
2710
2711         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2712                 dev_name(dev), start, end);
2713
2714         if (end < start) {
2715                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2716                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2717                         dmi_get_system_info(DMI_BIOS_VENDOR),
2718                         dmi_get_system_info(DMI_BIOS_VERSION),
2719                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2720                 return -EIO;
2721         }
2722
2723         if (end >> agaw_to_width(domain->agaw)) {
2724                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2725                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2726                      agaw_to_width(domain->agaw),
2727                      dmi_get_system_info(DMI_BIOS_VENDOR),
2728                      dmi_get_system_info(DMI_BIOS_VERSION),
2729                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2730                 return -EIO;
2731         }
2732
2733         return iommu_domain_identity_map(domain, start, end);
2734 }
2735
2736 static int iommu_prepare_identity_map(struct device *dev,
2737                                       unsigned long long start,
2738                                       unsigned long long end)
2739 {
2740         struct dmar_domain *domain;
2741         int ret;
2742
2743         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2744         if (!domain)
2745                 return -ENOMEM;
2746
2747         ret = domain_prepare_identity_map(dev, domain, start, end);
2748         if (ret)
2749                 domain_exit(domain);
2750
2751         return ret;
2752 }
2753
2754 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2755                                          struct device *dev)
2756 {
2757         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2758                 return 0;
2759         return iommu_prepare_identity_map(dev, rmrr->base_address,
2760                                           rmrr->end_address);
2761 }
2762
2763 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2764 static inline void iommu_prepare_isa(void)
2765 {
2766         struct pci_dev *pdev;
2767         int ret;
2768
2769         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2770         if (!pdev)
2771                 return;
2772
2773         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2774         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2775
2776         if (ret)
2777                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2778
2779         pci_dev_put(pdev);
2780 }
2781 #else
2782 static inline void iommu_prepare_isa(void)
2783 {
2784         return;
2785 }
2786 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2787
2788 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2789
2790 static int __init si_domain_init(int hw)
2791 {
2792         int nid, ret = 0;
2793
2794         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2795         if (!si_domain)
2796                 return -EFAULT;
2797
2798         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2799                 domain_exit(si_domain);
2800                 return -EFAULT;
2801         }
2802
2803         pr_debug("Identity mapping domain allocated\n");
2804
2805         if (hw)
2806                 return 0;
2807
2808         for_each_online_node(nid) {
2809                 unsigned long start_pfn, end_pfn;
2810                 int i;
2811
2812                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2813                         ret = iommu_domain_identity_map(si_domain,
2814                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2815                         if (ret)
2816                                 return ret;
2817                 }
2818         }
2819
2820         return 0;
2821 }
2822
2823 static int identity_mapping(struct device *dev)
2824 {
2825         struct device_domain_info *info;
2826
2827         if (likely(!iommu_identity_mapping))
2828                 return 0;
2829
2830         info = dev->archdata.iommu;
2831         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2832                 return (info->domain == si_domain);
2833
2834         return 0;
2835 }
2836
2837 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2838 {
2839         struct dmar_domain *ndomain;
2840         struct intel_iommu *iommu;
2841         u8 bus, devfn;
2842
2843         iommu = device_to_iommu(dev, &bus, &devfn);
2844         if (!iommu)
2845                 return -ENODEV;
2846
2847         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2848         if (ndomain != domain)
2849                 return -EBUSY;
2850
2851         return 0;
2852 }
2853
2854 static bool device_has_rmrr(struct device *dev)
2855 {
2856         struct dmar_rmrr_unit *rmrr;
2857         struct device *tmp;
2858         int i;
2859
2860         rcu_read_lock();
2861         for_each_rmrr_units(rmrr) {
2862                 /*
2863                  * Return TRUE if this RMRR contains the device that
2864                  * is passed in.
2865                  */
2866                 for_each_active_dev_scope(rmrr->devices,
2867                                           rmrr->devices_cnt, i, tmp)
2868                         if (tmp == dev) {
2869                                 rcu_read_unlock();
2870                                 return true;
2871                         }
2872         }
2873         rcu_read_unlock();
2874         return false;
2875 }
2876
2877 /*
2878  * There are a couple cases where we need to restrict the functionality of
2879  * devices associated with RMRRs.  The first is when evaluating a device for
2880  * identity mapping because problems exist when devices are moved in and out
2881  * of domains and their respective RMRR information is lost.  This means that
2882  * a device with associated RMRRs will never be in a "passthrough" domain.
2883  * The second is use of the device through the IOMMU API.  This interface
2884  * expects to have full control of the IOVA space for the device.  We cannot
2885  * satisfy both the requirement that RMRR access is maintained and have an
2886  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2887  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2888  * We therefore prevent devices associated with an RMRR from participating in
2889  * the IOMMU API, which eliminates them from device assignment.
2890  *
2891  * In both cases we assume that PCI USB devices with RMRRs have them largely
2892  * for historical reasons and that the RMRR space is not actively used post
2893  * boot.  This exclusion may change if vendors begin to abuse it.
2894  *
2895  * The same exception is made for graphics devices, with the requirement that
2896  * any use of the RMRR regions will be torn down before assigning the device
2897  * to a guest.
2898  */
2899 static bool device_is_rmrr_locked(struct device *dev)
2900 {
2901         if (!device_has_rmrr(dev))
2902                 return false;
2903
2904         if (dev_is_pci(dev)) {
2905                 struct pci_dev *pdev = to_pci_dev(dev);
2906
2907                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2908                         return false;
2909         }
2910
2911         return true;
2912 }
2913
2914 static int iommu_should_identity_map(struct device *dev, int startup)
2915 {
2916
2917         if (dev_is_pci(dev)) {
2918                 struct pci_dev *pdev = to_pci_dev(dev);
2919
2920                 if (device_is_rmrr_locked(dev))
2921                         return 0;
2922
2923                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2924                         return 1;
2925
2926                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2927                         return 1;
2928
2929                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2930                         return 0;
2931
2932                 /*
2933                  * We want to start off with all devices in the 1:1 domain, and
2934                  * take them out later if we find they can't access all of memory.
2935                  *
2936                  * However, we can't do this for PCI devices behind bridges,
2937                  * because all PCI devices behind the same bridge will end up
2938                  * with the same source-id on their transactions.
2939                  *
2940                  * Practically speaking, we can't change things around for these
2941                  * devices at run-time, because we can't be sure there'll be no
2942                  * DMA transactions in flight for any of their siblings.
2943                  *
2944                  * So PCI devices (unless they're on the root bus) as well as
2945                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2946                  * the 1:1 domain, just in _case_ one of their siblings turns out
2947                  * not to be able to map all of memory.
2948                  */
2949                 if (!pci_is_pcie(pdev)) {
2950                         if (!pci_is_root_bus(pdev->bus))
2951                                 return 0;
2952                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2953                                 return 0;
2954                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2955                         return 0;
2956         } else {
2957                 if (device_has_rmrr(dev))
2958                         return 0;
2959         }
2960
2961         /*
2962          * At boot time, we don't yet know if devices will be 64-bit capable.
2963          * Assume that they will — if they turn out not to be, then we can
2964          * take them out of the 1:1 domain later.
2965          */
2966         if (!startup) {
2967                 /*
2968                  * If the device's dma_mask is less than the system's memory
2969                  * size then this is not a candidate for identity mapping.
2970                  */
2971                 u64 dma_mask = *dev->dma_mask;
2972
2973                 if (dev->coherent_dma_mask &&
2974                     dev->coherent_dma_mask < dma_mask)
2975                         dma_mask = dev->coherent_dma_mask;
2976
2977                 return dma_mask >= dma_get_required_mask(dev);
2978         }
2979
2980         return 1;
2981 }
2982
2983 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2984 {
2985         int ret;
2986
2987         if (!iommu_should_identity_map(dev, 1))
2988                 return 0;
2989
2990         ret = domain_add_dev_info(si_domain, dev);
2991         if (!ret)
2992                 pr_info("%s identity mapping for device %s\n",
2993                         hw ? "Hardware" : "Software", dev_name(dev));
2994         else if (ret == -ENODEV)
2995                 /* device not associated with an iommu */
2996                 ret = 0;
2997
2998         return ret;
2999 }
3000
3001
3002 static int __init iommu_prepare_static_identity_mapping(int hw)
3003 {
3004         struct pci_dev *pdev = NULL;
3005         struct dmar_drhd_unit *drhd;
3006         struct intel_iommu *iommu;
3007         struct device *dev;
3008         int i;
3009         int ret = 0;
3010
3011         for_each_pci_dev(pdev) {
3012                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3013                 if (ret)
3014                         return ret;
3015         }
3016
3017         for_each_active_iommu(iommu, drhd)
3018                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3019                         struct acpi_device_physical_node *pn;
3020                         struct acpi_device *adev;
3021
3022                         if (dev->bus != &acpi_bus_type)
3023                                 continue;
3024
3025                         adev= to_acpi_device(dev);
3026                         mutex_lock(&adev->physical_node_lock);
3027                         list_for_each_entry(pn, &adev->physical_node_list, node) {
3028                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3029                                 if (ret)
3030                                         break;
3031                         }
3032                         mutex_unlock(&adev->physical_node_lock);
3033                         if (ret)
3034                                 return ret;
3035                 }
3036
3037         return 0;
3038 }
3039
3040 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3041 {
3042         /*
3043          * Start from the sane iommu hardware state.
3044          * If the queued invalidation is already initialized by us
3045          * (for example, while enabling interrupt-remapping) then
3046          * we got the things already rolling from a sane state.
3047          */
3048         if (!iommu->qi) {
3049                 /*
3050                  * Clear any previous faults.
3051                  */
3052                 dmar_fault(-1, iommu);
3053                 /*
3054                  * Disable queued invalidation if supported and already enabled
3055                  * before OS handover.
3056                  */
3057                 dmar_disable_qi(iommu);
3058         }
3059
3060         if (dmar_enable_qi(iommu)) {
3061                 /*
3062                  * Queued Invalidate not enabled, use Register Based Invalidate
3063                  */
3064                 iommu->flush.flush_context = __iommu_flush_context;
3065                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3066                 pr_info("%s: Using Register based invalidation\n",
3067                         iommu->name);
3068         } else {
3069                 iommu->flush.flush_context = qi_flush_context;
3070                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3071                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3072         }
3073 }
3074
3075 static int copy_context_table(struct intel_iommu *iommu,
3076                               struct root_entry *old_re,
3077                               struct context_entry **tbl,
3078                               int bus, bool ext)
3079 {
3080         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3081         struct context_entry *new_ce = NULL, ce;
3082         struct context_entry *old_ce = NULL;
3083         struct root_entry re;
3084         phys_addr_t old_ce_phys;
3085
3086         tbl_idx = ext ? bus * 2 : bus;
3087         memcpy(&re, old_re, sizeof(re));
3088
3089         for (devfn = 0; devfn < 256; devfn++) {
3090                 /* First calculate the correct index */
3091                 idx = (ext ? devfn * 2 : devfn) % 256;
3092
3093                 if (idx == 0) {
3094                         /* First save what we may have and clean up */
3095                         if (new_ce) {
3096                                 tbl[tbl_idx] = new_ce;
3097                                 __iommu_flush_cache(iommu, new_ce,
3098                                                     VTD_PAGE_SIZE);
3099                                 pos = 1;
3100                         }
3101
3102                         if (old_ce)
3103                                 iounmap(old_ce);
3104
3105                         ret = 0;
3106                         if (devfn < 0x80)
3107                                 old_ce_phys = root_entry_lctp(&re);
3108                         else
3109                                 old_ce_phys = root_entry_uctp(&re);
3110
3111                         if (!old_ce_phys) {
3112                                 if (ext && devfn == 0) {
3113                                         /* No LCTP, try UCTP */
3114                                         devfn = 0x7f;
3115                                         continue;
3116                                 } else {
3117                                         goto out;
3118                                 }
3119                         }
3120
3121                         ret = -ENOMEM;
3122                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3123                                         MEMREMAP_WB);
3124                         if (!old_ce)
3125                                 goto out;
3126
3127                         new_ce = alloc_pgtable_page(iommu->node);
3128                         if (!new_ce)
3129                                 goto out_unmap;
3130
3131                         ret = 0;
3132                 }
3133
3134                 /* Now copy the context entry */
3135                 memcpy(&ce, old_ce + idx, sizeof(ce));
3136
3137                 if (!__context_present(&ce))
3138                         continue;
3139
3140                 did = context_domain_id(&ce);
3141                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3142                         set_bit(did, iommu->domain_ids);
3143
3144                 /*
3145                  * We need a marker for copied context entries. This
3146                  * marker needs to work for the old format as well as
3147                  * for extended context entries.
3148                  *
3149                  * Bit 67 of the context entry is used. In the old
3150                  * format this bit is available to software, in the
3151                  * extended format it is the PGE bit, but PGE is ignored
3152                  * by HW if PASIDs are disabled (and thus still
3153                  * available).
3154                  *
3155                  * So disable PASIDs first and then mark the entry
3156                  * copied. This means that we don't copy PASID
3157                  * translations from the old kernel, but this is fine as
3158                  * faults there are not fatal.
3159                  */
3160                 context_clear_pasid_enable(&ce);
3161                 context_set_copied(&ce);
3162
3163                 new_ce[idx] = ce;
3164         }
3165
3166         tbl[tbl_idx + pos] = new_ce;
3167
3168         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3169
3170 out_unmap:
3171         memunmap(old_ce);
3172
3173 out:
3174         return ret;
3175 }
3176
3177 static int copy_translation_tables(struct intel_iommu *iommu)
3178 {
3179         struct context_entry **ctxt_tbls;
3180         struct root_entry *old_rt;
3181         phys_addr_t old_rt_phys;
3182         int ctxt_table_entries;
3183         unsigned long flags;
3184         u64 rtaddr_reg;
3185         int bus, ret;
3186         bool new_ext, ext;
3187
3188         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3189         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3190         new_ext    = !!ecap_ecs(iommu->ecap);
3191
3192         /*
3193          * The RTT bit can only be changed when translation is disabled,
3194          * but disabling translation means to open a window for data
3195          * corruption. So bail out and don't copy anything if we would
3196          * have to change the bit.
3197          */
3198         if (new_ext != ext)
3199                 return -EINVAL;
3200
3201         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3202         if (!old_rt_phys)
3203                 return -EINVAL;
3204
3205         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3206         if (!old_rt)
3207                 return -ENOMEM;
3208
3209         /* This is too big for the stack - allocate it from slab */
3210         ctxt_table_entries = ext ? 512 : 256;
3211         ret = -ENOMEM;
3212         ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3213         if (!ctxt_tbls)
3214                 goto out_unmap;
3215
3216         for (bus = 0; bus < 256; bus++) {
3217                 ret = copy_context_table(iommu, &old_rt[bus],
3218                                          ctxt_tbls, bus, ext);
3219                 if (ret) {
3220                         pr_err("%s: Failed to copy context table for bus %d\n",
3221                                 iommu->name, bus);
3222                         continue;
3223                 }
3224         }
3225
3226         spin_lock_irqsave(&iommu->lock, flags);
3227
3228         /* Context tables are copied, now write them to the root_entry table */
3229         for (bus = 0; bus < 256; bus++) {
3230                 int idx = ext ? bus * 2 : bus;
3231                 u64 val;
3232
3233                 if (ctxt_tbls[idx]) {
3234                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3235                         iommu->root_entry[bus].lo = val;
3236                 }
3237
3238                 if (!ext || !ctxt_tbls[idx + 1])
3239                         continue;
3240
3241                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3242                 iommu->root_entry[bus].hi = val;
3243         }
3244
3245         spin_unlock_irqrestore(&iommu->lock, flags);
3246
3247         kfree(ctxt_tbls);
3248
3249         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3250
3251         ret = 0;
3252
3253 out_unmap:
3254         memunmap(old_rt);
3255
3256         return ret;
3257 }
3258
3259 static int __init init_dmars(void)
3260 {
3261         struct dmar_drhd_unit *drhd;
3262         struct dmar_rmrr_unit *rmrr;
3263         bool copied_tables = false;
3264         struct device *dev;
3265         struct intel_iommu *iommu;
3266         int i, ret;
3267
3268         /*
3269          * for each drhd
3270          *    allocate root
3271          *    initialize and program root entry to not present
3272          * endfor
3273          */
3274         for_each_drhd_unit(drhd) {
3275                 /*
3276                  * lock not needed as this is only incremented in the single
3277                  * threaded kernel __init code path all other access are read
3278                  * only
3279                  */
3280                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3281                         g_num_of_iommus++;
3282                         continue;
3283                 }
3284                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3285         }
3286
3287         /* Preallocate enough resources for IOMMU hot-addition */
3288         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3289                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3290
3291         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3292                         GFP_KERNEL);
3293         if (!g_iommus) {
3294                 pr_err("Allocating global iommu array failed\n");
3295                 ret = -ENOMEM;
3296                 goto error;
3297         }
3298
3299         for_each_active_iommu(iommu, drhd) {
3300                 /*
3301                  * Find the max pasid size of all IOMMU's in the system.
3302                  * We need to ensure the system pasid table is no bigger
3303                  * than the smallest supported.
3304                  */
3305                 if (pasid_enabled(iommu)) {
3306                         u32 temp = 2 << ecap_pss(iommu->ecap);
3307
3308                         intel_pasid_max_id = min_t(u32, temp,
3309                                                    intel_pasid_max_id);
3310                 }
3311
3312                 g_iommus[iommu->seq_id] = iommu;
3313
3314                 intel_iommu_init_qi(iommu);
3315
3316                 ret = iommu_init_domains(iommu);
3317                 if (ret)
3318                         goto free_iommu;
3319
3320                 init_translation_status(iommu);
3321
3322                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3323                         iommu_disable_translation(iommu);
3324                         clear_translation_pre_enabled(iommu);
3325                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3326                                 iommu->name);
3327                 }
3328
3329                 /*
3330                  * TBD:
3331                  * we could share the same root & context tables
3332                  * among all IOMMU's. Need to Split it later.
3333                  */
3334                 ret = iommu_alloc_root_entry(iommu);
3335                 if (ret)
3336                         goto free_iommu;
3337
3338                 if (translation_pre_enabled(iommu)) {
3339                         pr_info("Translation already enabled - trying to copy translation structures\n");
3340
3341                         ret = copy_translation_tables(iommu);
3342                         if (ret) {
3343                                 /*
3344                                  * We found the IOMMU with translation
3345                                  * enabled - but failed to copy over the
3346                                  * old root-entry table. Try to proceed
3347                                  * by disabling translation now and
3348                                  * allocating a clean root-entry table.
3349                                  * This might cause DMAR faults, but
3350                                  * probably the dump will still succeed.
3351                                  */
3352                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3353                                        iommu->name);
3354                                 iommu_disable_translation(iommu);
3355                                 clear_translation_pre_enabled(iommu);
3356                         } else {
3357                                 pr_info("Copied translation tables from previous kernel for %s\n",
3358                                         iommu->name);
3359                                 copied_tables = true;
3360                         }
3361                 }
3362
3363                 if (!ecap_pass_through(iommu->ecap))
3364                         hw_pass_through = 0;
3365 #ifdef CONFIG_INTEL_IOMMU_SVM
3366                 if (pasid_enabled(iommu))
3367                         intel_svm_init(iommu);
3368 #endif
3369         }
3370
3371         /*
3372          * Now that qi is enabled on all iommus, set the root entry and flush
3373          * caches. This is required on some Intel X58 chipsets, otherwise the
3374          * flush_context function will loop forever and the boot hangs.
3375          */
3376         for_each_active_iommu(iommu, drhd) {
3377                 iommu_flush_write_buffer(iommu);
3378                 iommu_set_root_entry(iommu);
3379                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3380                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3381         }
3382
3383         if (iommu_pass_through)
3384                 iommu_identity_mapping |= IDENTMAP_ALL;
3385
3386 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3387         iommu_identity_mapping |= IDENTMAP_GFX;
3388 #endif
3389
3390         check_tylersburg_isoch();
3391
3392         if (iommu_identity_mapping) {
3393                 ret = si_domain_init(hw_pass_through);
3394                 if (ret)
3395                         goto free_iommu;
3396         }
3397
3398
3399         /*
3400          * If we copied translations from a previous kernel in the kdump
3401          * case, we can not assign the devices to domains now, as that
3402          * would eliminate the old mappings. So skip this part and defer
3403          * the assignment to device driver initialization time.
3404          */
3405         if (copied_tables)
3406                 goto domains_done;
3407
3408         /*
3409          * If pass through is not set or not enabled, setup context entries for
3410          * identity mappings for rmrr, gfx, and isa and may fall back to static
3411          * identity mapping if iommu_identity_mapping is set.
3412          */
3413         if (iommu_identity_mapping) {
3414                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3415                 if (ret) {
3416                         pr_crit("Failed to setup IOMMU pass-through\n");
3417                         goto free_iommu;
3418                 }
3419         }
3420         /*
3421          * For each rmrr
3422          *   for each dev attached to rmrr
3423          *   do
3424          *     locate drhd for dev, alloc domain for dev
3425          *     allocate free domain
3426          *     allocate page table entries for rmrr
3427          *     if context not allocated for bus
3428          *           allocate and init context
3429          *           set present in root table for this bus
3430          *     init context with domain, translation etc
3431          *    endfor
3432          * endfor
3433          */
3434         pr_info("Setting RMRR:\n");
3435         for_each_rmrr_units(rmrr) {
3436                 /* some BIOS lists non-exist devices in DMAR table. */
3437                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3438                                           i, dev) {
3439                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3440                         if (ret)
3441                                 pr_err("Mapping reserved region failed\n");
3442                 }
3443         }
3444
3445         iommu_prepare_isa();
3446
3447 domains_done:
3448
3449         /*
3450          * for each drhd
3451          *   enable fault log
3452          *   global invalidate context cache
3453          *   global invalidate iotlb
3454          *   enable translation
3455          */
3456         for_each_iommu(iommu, drhd) {
3457                 if (drhd->ignored) {
3458                         /*
3459                          * we always have to disable PMRs or DMA may fail on
3460                          * this device
3461                          */
3462                         if (force_on)
3463                                 iommu_disable_protect_mem_regions(iommu);
3464                         continue;
3465                 }
3466
3467                 iommu_flush_write_buffer(iommu);
3468
3469 #ifdef CONFIG_INTEL_IOMMU_SVM
3470                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3471                         ret = intel_svm_enable_prq(iommu);
3472                         if (ret)
3473                                 goto free_iommu;
3474                 }
3475 #endif
3476                 ret = dmar_set_interrupt(iommu);
3477                 if (ret)
3478                         goto free_iommu;
3479
3480                 if (!translation_pre_enabled(iommu))
3481                         iommu_enable_translation(iommu);
3482
3483                 iommu_disable_protect_mem_regions(iommu);
3484         }
3485
3486         return 0;
3487
3488 free_iommu:
3489         for_each_active_iommu(iommu, drhd) {
3490                 disable_dmar_iommu(iommu);
3491                 free_dmar_iommu(iommu);
3492         }
3493
3494         kfree(g_iommus);
3495
3496 error:
3497         return ret;
3498 }
3499
3500 /* This takes a number of _MM_ pages, not VTD pages */
3501 static unsigned long intel_alloc_iova(struct device *dev,
3502                                      struct dmar_domain *domain,
3503                                      unsigned long nrpages, uint64_t dma_mask)
3504 {
3505         unsigned long iova_pfn = 0;
3506
3507         /* Restrict dma_mask to the width that the iommu can handle */
3508         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3509         /* Ensure we reserve the whole size-aligned region */
3510         nrpages = __roundup_pow_of_two(nrpages);
3511
3512         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3513                 /*
3514                  * First try to allocate an io virtual address in
3515                  * DMA_BIT_MASK(32) and if that fails then try allocating
3516                  * from higher range
3517                  */
3518                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3519                                            IOVA_PFN(DMA_BIT_MASK(32)), false);
3520                 if (iova_pfn)
3521                         return iova_pfn;
3522         }
3523         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3524                                    IOVA_PFN(dma_mask), true);
3525         if (unlikely(!iova_pfn)) {
3526                 pr_err("Allocating %ld-page iova for %s failed",
3527                        nrpages, dev_name(dev));
3528                 return 0;
3529         }
3530
3531         return iova_pfn;
3532 }
3533
3534 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3535 {
3536         struct dmar_domain *domain, *tmp;
3537         struct dmar_rmrr_unit *rmrr;
3538         struct device *i_dev;
3539         int i, ret;
3540
3541         domain = find_domain(dev);
3542         if (domain)
3543                 goto out;
3544
3545         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3546         if (!domain)
3547                 goto out;
3548
3549         /* We have a new domain - setup possible RMRRs for the device */
3550         rcu_read_lock();
3551         for_each_rmrr_units(rmrr) {
3552                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3553                                           i, i_dev) {
3554                         if (i_dev != dev)
3555                                 continue;
3556
3557                         ret = domain_prepare_identity_map(dev, domain,
3558                                                           rmrr->base_address,
3559                                                           rmrr->end_address);
3560                         if (ret)
3561                                 dev_err(dev, "Mapping reserved region failed\n");
3562                 }
3563         }
3564         rcu_read_unlock();
3565
3566         tmp = set_domain_for_dev(dev, domain);
3567         if (!tmp || domain != tmp) {
3568                 domain_exit(domain);
3569                 domain = tmp;
3570         }
3571
3572 out:
3573
3574         if (!domain)
3575                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3576
3577
3578         return domain;
3579 }
3580
3581 /* Check if the dev needs to go through non-identity map and unmap process.*/
3582 static int iommu_no_mapping(struct device *dev)
3583 {
3584         int found;
3585
3586         if (iommu_dummy(dev))
3587                 return 1;
3588
3589         if (!iommu_identity_mapping)
3590                 return 0;
3591
3592         found = identity_mapping(dev);
3593         if (found) {
3594                 if (iommu_should_identity_map(dev, 0))
3595                         return 1;
3596                 else {
3597                         /*
3598                          * 32 bit DMA is removed from si_domain and fall back
3599                          * to non-identity mapping.
3600                          */
3601                         dmar_remove_one_dev_info(si_domain, dev);
3602                         pr_info("32bit %s uses non-identity mapping\n",
3603                                 dev_name(dev));
3604                         return 0;
3605                 }
3606         } else {
3607                 /*
3608                  * In case of a detached 64 bit DMA device from vm, the device
3609                  * is put into si_domain for identity mapping.
3610                  */
3611                 if (iommu_should_identity_map(dev, 0)) {
3612                         int ret;
3613                         ret = domain_add_dev_info(si_domain, dev);
3614                         if (!ret) {
3615                                 pr_info("64bit %s uses identity mapping\n",
3616                                         dev_name(dev));
3617                                 return 1;
3618                         }
3619                 }
3620         }
3621
3622         return 0;
3623 }
3624
3625 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3626                                      size_t size, int dir, u64 dma_mask)
3627 {
3628         struct dmar_domain *domain;
3629         phys_addr_t start_paddr;
3630         unsigned long iova_pfn;
3631         int prot = 0;
3632         int ret;
3633         struct intel_iommu *iommu;
3634         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3635
3636         BUG_ON(dir == DMA_NONE);
3637
3638         if (iommu_no_mapping(dev))
3639                 return paddr;
3640
3641         domain = get_valid_domain_for_dev(dev);
3642         if (!domain)
3643                 return 0;
3644
3645         iommu = domain_get_iommu(domain);
3646         size = aligned_nrpages(paddr, size);
3647
3648         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3649         if (!iova_pfn)
3650                 goto error;
3651
3652         /*
3653          * Check if DMAR supports zero-length reads on write only
3654          * mappings..
3655          */
3656         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3657                         !cap_zlr(iommu->cap))
3658                 prot |= DMA_PTE_READ;
3659         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3660                 prot |= DMA_PTE_WRITE;
3661         /*
3662          * paddr - (paddr + size) might be partial page, we should map the whole
3663          * page.  Note: if two part of one page are separately mapped, we
3664          * might have two guest_addr mapping to the same host paddr, but this
3665          * is not a big problem
3666          */
3667         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3668                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3669         if (ret)
3670                 goto error;
3671
3672         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3673         start_paddr += paddr & ~PAGE_MASK;
3674         return start_paddr;
3675
3676 error:
3677         if (iova_pfn)
3678                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3679         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3680                 dev_name(dev), size, (unsigned long long)paddr, dir);
3681         return 0;
3682 }
3683
3684 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3685                                  unsigned long offset, size_t size,
3686                                  enum dma_data_direction dir,
3687                                  unsigned long attrs)
3688 {
3689         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3690                                   dir, *dev->dma_mask);
3691 }
3692
3693 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3694 {
3695         struct dmar_domain *domain;
3696         unsigned long start_pfn, last_pfn;
3697         unsigned long nrpages;
3698         unsigned long iova_pfn;
3699         struct intel_iommu *iommu;
3700         struct page *freelist;
3701
3702         if (iommu_no_mapping(dev))
3703                 return;
3704
3705         domain = find_domain(dev);
3706         BUG_ON(!domain);
3707
3708         iommu = domain_get_iommu(domain);
3709
3710         iova_pfn = IOVA_PFN(dev_addr);
3711
3712         nrpages = aligned_nrpages(dev_addr, size);
3713         start_pfn = mm_to_dma_pfn(iova_pfn);
3714         last_pfn = start_pfn + nrpages - 1;
3715
3716         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3717                  dev_name(dev), start_pfn, last_pfn);
3718
3719         freelist = domain_unmap(domain, start_pfn, last_pfn);
3720
3721         if (intel_iommu_strict) {
3722                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3723                                       nrpages, !freelist, 0);
3724                 /* free iova */
3725                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3726                 dma_free_pagelist(freelist);
3727         } else {
3728                 queue_iova(&domain->iovad, iova_pfn, nrpages,
3729                            (unsigned long)freelist);
3730                 /*
3731                  * queue up the release of the unmap to save the 1/6th of the
3732                  * cpu used up by the iotlb flush operation...
3733                  */
3734         }
3735 }
3736
3737 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3738                              size_t size, enum dma_data_direction dir,
3739                              unsigned long attrs)
3740 {
3741         intel_unmap(dev, dev_addr, size);
3742 }
3743
3744 static void *intel_alloc_coherent(struct device *dev, size_t size,
3745                                   dma_addr_t *dma_handle, gfp_t flags,
3746                                   unsigned long attrs)
3747 {
3748         struct page *page = NULL;
3749         int order;
3750
3751         size = PAGE_ALIGN(size);
3752         order = get_order(size);
3753
3754         if (!iommu_no_mapping(dev))
3755                 flags &= ~(GFP_DMA | GFP_DMA32);
3756         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3757                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3758                         flags |= GFP_DMA;
3759                 else
3760                         flags |= GFP_DMA32;
3761         }
3762
3763         if (gfpflags_allow_blocking(flags)) {
3764                 unsigned int count = size >> PAGE_SHIFT;
3765
3766                 page = dma_alloc_from_contiguous(dev, count, order,
3767                                                  flags & __GFP_NOWARN);
3768                 if (page && iommu_no_mapping(dev) &&
3769                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3770                         dma_release_from_contiguous(dev, page, count);
3771                         page = NULL;
3772                 }
3773         }
3774
3775         if (!page)
3776                 page = alloc_pages(flags, order);
3777         if (!page)
3778                 return NULL;
3779         memset(page_address(page), 0, size);
3780
3781         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3782                                          DMA_BIDIRECTIONAL,
3783                                          dev->coherent_dma_mask);
3784         if (*dma_handle)
3785                 return page_address(page);
3786         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3787                 __free_pages(page, order);
3788
3789         return NULL;
3790 }
3791
3792 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3793                                 dma_addr_t dma_handle, unsigned long attrs)
3794 {
3795         int order;
3796         struct page *page = virt_to_page(vaddr);
3797
3798         size = PAGE_ALIGN(size);
3799         order = get_order(size);
3800
3801         intel_unmap(dev, dma_handle, size);
3802         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3803                 __free_pages(page, order);
3804 }
3805
3806 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3807                            int nelems, enum dma_data_direction dir,
3808                            unsigned long attrs)
3809 {
3810         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3811         unsigned long nrpages = 0;
3812         struct scatterlist *sg;
3813         int i;
3814
3815         for_each_sg(sglist, sg, nelems, i) {
3816                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3817         }
3818
3819         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3820 }
3821
3822 static int intel_nontranslate_map_sg(struct device *hddev,
3823         struct scatterlist *sglist, int nelems, int dir)
3824 {
3825         int i;
3826         struct scatterlist *sg;
3827
3828         for_each_sg(sglist, sg, nelems, i) {
3829                 BUG_ON(!sg_page(sg));
3830                 sg->dma_address = sg_phys(sg);
3831                 sg->dma_length = sg->length;
3832         }
3833         return nelems;
3834 }
3835
3836 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3837                         enum dma_data_direction dir, unsigned long attrs)
3838 {
3839         int i;
3840         struct dmar_domain *domain;
3841         size_t size = 0;
3842         int prot = 0;
3843         unsigned long iova_pfn;
3844         int ret;
3845         struct scatterlist *sg;
3846         unsigned long start_vpfn;
3847         struct intel_iommu *iommu;
3848
3849         BUG_ON(dir == DMA_NONE);
3850         if (iommu_no_mapping(dev))
3851                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3852
3853         domain = get_valid_domain_for_dev(dev);
3854         if (!domain)
3855                 return 0;
3856
3857         iommu = domain_get_iommu(domain);
3858
3859         for_each_sg(sglist, sg, nelems, i)
3860                 size += aligned_nrpages(sg->offset, sg->length);
3861
3862         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3863                                 *dev->dma_mask);
3864         if (!iova_pfn) {
3865                 sglist->dma_length = 0;
3866                 return 0;
3867         }
3868
3869         /*
3870          * Check if DMAR supports zero-length reads on write only
3871          * mappings..
3872          */
3873         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3874                         !cap_zlr(iommu->cap))
3875                 prot |= DMA_PTE_READ;
3876         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3877                 prot |= DMA_PTE_WRITE;
3878
3879         start_vpfn = mm_to_dma_pfn(iova_pfn);
3880
3881         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3882         if (unlikely(ret)) {
3883                 dma_pte_free_pagetable(domain, start_vpfn,
3884                                        start_vpfn + size - 1,
3885                                        agaw_to_level(domain->agaw) + 1);
3886                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3887                 return 0;
3888         }
3889
3890         return nelems;
3891 }
3892
3893 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3894 {
3895         return !dma_addr;
3896 }
3897
3898 const struct dma_map_ops intel_dma_ops = {
3899         .alloc = intel_alloc_coherent,
3900         .free = intel_free_coherent,
3901         .map_sg = intel_map_sg,
3902         .unmap_sg = intel_unmap_sg,
3903         .map_page = intel_map_page,
3904         .unmap_page = intel_unmap_page,
3905         .mapping_error = intel_mapping_error,
3906 #ifdef CONFIG_X86
3907         .dma_supported = dma_direct_supported,
3908 #endif
3909 };
3910
3911 static inline int iommu_domain_cache_init(void)
3912 {
3913         int ret = 0;
3914
3915         iommu_domain_cache = kmem_cache_create("iommu_domain",
3916                                          sizeof(struct dmar_domain),
3917                                          0,
3918                                          SLAB_HWCACHE_ALIGN,
3919
3920                                          NULL);
3921         if (!iommu_domain_cache) {
3922                 pr_err("Couldn't create iommu_domain cache\n");
3923                 ret = -ENOMEM;
3924         }
3925
3926         return ret;
3927 }
3928
3929 static inline int iommu_devinfo_cache_init(void)
3930 {
3931         int ret = 0;
3932
3933         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3934                                          sizeof(struct device_domain_info),
3935                                          0,
3936                                          SLAB_HWCACHE_ALIGN,
3937                                          NULL);
3938         if (!iommu_devinfo_cache) {
3939                 pr_err("Couldn't create devinfo cache\n");
3940                 ret = -ENOMEM;
3941         }
3942
3943         return ret;
3944 }
3945
3946 static int __init iommu_init_mempool(void)
3947 {
3948         int ret;
3949         ret = iova_cache_get();
3950         if (ret)
3951                 return ret;
3952
3953         ret = iommu_domain_cache_init();
3954         if (ret)
3955                 goto domain_error;
3956
3957         ret = iommu_devinfo_cache_init();
3958         if (!ret)
3959                 return ret;
3960
3961         kmem_cache_destroy(iommu_domain_cache);
3962 domain_error:
3963         iova_cache_put();
3964
3965         return -ENOMEM;
3966 }
3967
3968 static void __init iommu_exit_mempool(void)
3969 {
3970         kmem_cache_destroy(iommu_devinfo_cache);
3971         kmem_cache_destroy(iommu_domain_cache);
3972         iova_cache_put();
3973 }
3974
3975 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3976 {
3977         struct dmar_drhd_unit *drhd;
3978         u32 vtbar;
3979         int rc;
3980
3981         /* We know that this device on this chipset has its own IOMMU.
3982          * If we find it under a different IOMMU, then the BIOS is lying
3983          * to us. Hope that the IOMMU for this device is actually
3984          * disabled, and it needs no translation...
3985          */
3986         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3987         if (rc) {
3988                 /* "can't" happen */
3989                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3990                 return;
3991         }
3992         vtbar &= 0xffff0000;
3993
3994         /* we know that the this iommu should be at offset 0xa000 from vtbar */
3995         drhd = dmar_find_matched_drhd_unit(pdev);
3996         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3997                             TAINT_FIRMWARE_WORKAROUND,
3998                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3999                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4000 }
4001 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4002
4003 static void __init init_no_remapping_devices(void)
4004 {
4005         struct dmar_drhd_unit *drhd;
4006         struct device *dev;
4007         int i;
4008
4009         for_each_drhd_unit(drhd) {
4010                 if (!drhd->include_all) {
4011                         for_each_active_dev_scope(drhd->devices,
4012                                                   drhd->devices_cnt, i, dev)
4013                                 break;
4014                         /* ignore DMAR unit if no devices exist */
4015                         if (i == drhd->devices_cnt)
4016                                 drhd->ignored = 1;
4017                 }
4018         }
4019
4020         for_each_active_drhd_unit(drhd) {
4021                 if (drhd->include_all)
4022                         continue;
4023
4024                 for_each_active_dev_scope(drhd->devices,
4025                                           drhd->devices_cnt, i, dev)
4026                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4027                                 break;
4028                 if (i < drhd->devices_cnt)
4029                         continue;
4030
4031                 /* This IOMMU has *only* gfx devices. Either bypass it or
4032                    set the gfx_mapped flag, as appropriate */
4033                 if (dmar_map_gfx) {
4034                         intel_iommu_gfx_mapped = 1;
4035                 } else {
4036                         drhd->ignored = 1;
4037                         for_each_active_dev_scope(drhd->devices,
4038                                                   drhd->devices_cnt, i, dev)
4039                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4040                 }
4041         }
4042 }
4043
4044 #ifdef CONFIG_SUSPEND
4045 static int init_iommu_hw(void)
4046 {
4047         struct dmar_drhd_unit *drhd;
4048         struct intel_iommu *iommu = NULL;
4049
4050         for_each_active_iommu(iommu, drhd)
4051                 if (iommu->qi)
4052                         dmar_reenable_qi(iommu);
4053
4054         for_each_iommu(iommu, drhd) {
4055                 if (drhd->ignored) {
4056                         /*
4057                          * we always have to disable PMRs or DMA may fail on
4058                          * this device
4059                          */
4060                         if (force_on)
4061                                 iommu_disable_protect_mem_regions(iommu);
4062                         continue;
4063                 }
4064
4065                 iommu_flush_write_buffer(iommu);
4066
4067                 iommu_set_root_entry(iommu);
4068
4069                 iommu->flush.flush_context(iommu, 0, 0, 0,
4070                                            DMA_CCMD_GLOBAL_INVL);
4071                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4072                 iommu_enable_translation(iommu);
4073                 iommu_disable_protect_mem_regions(iommu);
4074         }
4075
4076         return 0;
4077 }
4078
4079 static void iommu_flush_all(void)
4080 {
4081         struct dmar_drhd_unit *drhd;
4082         struct intel_iommu *iommu;
4083
4084         for_each_active_iommu(iommu, drhd) {
4085                 iommu->flush.flush_context(iommu, 0, 0, 0,
4086                                            DMA_CCMD_GLOBAL_INVL);
4087                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4088                                          DMA_TLB_GLOBAL_FLUSH);
4089         }
4090 }
4091
4092 static int iommu_suspend(void)
4093 {
4094         struct dmar_drhd_unit *drhd;
4095         struct intel_iommu *iommu = NULL;
4096         unsigned long flag;
4097
4098         for_each_active_iommu(iommu, drhd) {
4099                 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4100                                                  GFP_ATOMIC);
4101                 if (!iommu->iommu_state)
4102                         goto nomem;
4103         }
4104
4105         iommu_flush_all();
4106
4107         for_each_active_iommu(iommu, drhd) {
4108                 iommu_disable_translation(iommu);
4109
4110                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4111
4112                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4113                         readl(iommu->reg + DMAR_FECTL_REG);
4114                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4115                         readl(iommu->reg + DMAR_FEDATA_REG);
4116                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4117                         readl(iommu->reg + DMAR_FEADDR_REG);
4118                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4119                         readl(iommu->reg + DMAR_FEUADDR_REG);
4120
4121                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4122         }
4123         return 0;
4124
4125 nomem:
4126         for_each_active_iommu(iommu, drhd)
4127                 kfree(iommu->iommu_state);
4128
4129         return -ENOMEM;
4130 }
4131
4132 static void iommu_resume(void)
4133 {
4134         struct dmar_drhd_unit *drhd;
4135         struct intel_iommu *iommu = NULL;
4136         unsigned long flag;
4137
4138         if (init_iommu_hw()) {
4139                 if (force_on)
4140                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4141                 else
4142                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4143                 return;
4144         }
4145
4146         for_each_active_iommu(iommu, drhd) {
4147
4148                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4149
4150                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4151                         iommu->reg + DMAR_FECTL_REG);
4152                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4153                         iommu->reg + DMAR_FEDATA_REG);
4154                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4155                         iommu->reg + DMAR_FEADDR_REG);
4156                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4157                         iommu->reg + DMAR_FEUADDR_REG);
4158
4159                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4160         }
4161
4162         for_each_active_iommu(iommu, drhd)
4163                 kfree(iommu->iommu_state);
4164 }
4165
4166 static struct syscore_ops iommu_syscore_ops = {
4167         .resume         = iommu_resume,
4168         .suspend        = iommu_suspend,
4169 };
4170
4171 static void __init init_iommu_pm_ops(void)
4172 {
4173         register_syscore_ops(&iommu_syscore_ops);
4174 }
4175
4176 #else
4177 static inline void init_iommu_pm_ops(void) {}
4178 #endif  /* CONFIG_PM */
4179
4180
4181 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4182 {
4183         struct acpi_dmar_reserved_memory *rmrr;
4184         int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4185         struct dmar_rmrr_unit *rmrru;
4186         size_t length;
4187
4188         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4189         if (!rmrru)
4190                 goto out;
4191
4192         rmrru->hdr = header;
4193         rmrr = (struct acpi_dmar_reserved_memory *)header;
4194         rmrru->base_address = rmrr->base_address;
4195         rmrru->end_address = rmrr->end_address;
4196
4197         length = rmrr->end_address - rmrr->base_address + 1;
4198         rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4199                                               IOMMU_RESV_DIRECT);
4200         if (!rmrru->resv)
4201                 goto free_rmrru;
4202
4203         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4204                                 ((void *)rmrr) + rmrr->header.length,
4205                                 &rmrru->devices_cnt);
4206         if (rmrru->devices_cnt && rmrru->devices == NULL)
4207                 goto free_all;
4208
4209         list_add(&rmrru->list, &dmar_rmrr_units);
4210
4211         return 0;
4212 free_all:
4213         kfree(rmrru->resv);
4214 free_rmrru:
4215         kfree(rmrru);
4216 out:
4217         return -ENOMEM;
4218 }
4219
4220 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4221 {
4222         struct dmar_atsr_unit *atsru;
4223         struct acpi_dmar_atsr *tmp;
4224
4225         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4226                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4227                 if (atsr->segment != tmp->segment)
4228                         continue;
4229                 if (atsr->header.length != tmp->header.length)
4230                         continue;
4231                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4232                         return atsru;
4233         }
4234
4235         return NULL;
4236 }
4237
4238 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4239 {
4240         struct acpi_dmar_atsr *atsr;
4241         struct dmar_atsr_unit *atsru;
4242
4243         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4244                 return 0;
4245
4246         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4247         atsru = dmar_find_atsr(atsr);
4248         if (atsru)
4249                 return 0;
4250
4251         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4252         if (!atsru)
4253                 return -ENOMEM;
4254
4255         /*
4256          * If memory is allocated from slab by ACPI _DSM method, we need to
4257          * copy the memory content because the memory buffer will be freed
4258          * on return.
4259          */
4260         atsru->hdr = (void *)(atsru + 1);
4261         memcpy(atsru->hdr, hdr, hdr->length);
4262         atsru->include_all = atsr->flags & 0x1;
4263         if (!atsru->include_all) {
4264                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4265                                 (void *)atsr + atsr->header.length,
4266                                 &atsru->devices_cnt);
4267                 if (atsru->devices_cnt && atsru->devices == NULL) {
4268                         kfree(atsru);
4269                         return -ENOMEM;
4270                 }
4271         }
4272
4273         list_add_rcu(&atsru->list, &dmar_atsr_units);
4274
4275         return 0;
4276 }
4277
4278 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4279 {
4280         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4281         kfree(atsru);
4282 }
4283
4284 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4285 {
4286         struct acpi_dmar_atsr *atsr;
4287         struct dmar_atsr_unit *atsru;
4288
4289         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4290         atsru = dmar_find_atsr(atsr);
4291         if (atsru) {
4292                 list_del_rcu(&atsru->list);
4293                 synchronize_rcu();
4294                 intel_iommu_free_atsr(atsru);
4295         }
4296
4297         return 0;
4298 }
4299
4300 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4301 {
4302         int i;
4303         struct device *dev;
4304         struct acpi_dmar_atsr *atsr;
4305         struct dmar_atsr_unit *atsru;
4306
4307         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4308         atsru = dmar_find_atsr(atsr);
4309         if (!atsru)
4310                 return 0;
4311
4312         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4313                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4314                                           i, dev)
4315                         return -EBUSY;
4316         }
4317
4318         return 0;
4319 }
4320
4321 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4322 {
4323         int sp, ret = 0;
4324         struct intel_iommu *iommu = dmaru->iommu;
4325
4326         if (g_iommus[iommu->seq_id])
4327                 return 0;
4328
4329         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4330                 pr_warn("%s: Doesn't support hardware pass through.\n",
4331                         iommu->name);
4332                 return -ENXIO;
4333         }
4334         if (!ecap_sc_support(iommu->ecap) &&
4335             domain_update_iommu_snooping(iommu)) {
4336                 pr_warn("%s: Doesn't support snooping.\n",
4337                         iommu->name);
4338                 return -ENXIO;
4339         }
4340         sp = domain_update_iommu_superpage(iommu) - 1;
4341         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4342                 pr_warn("%s: Doesn't support large page.\n",
4343                         iommu->name);
4344                 return -ENXIO;
4345         }
4346
4347         /*
4348          * Disable translation if already enabled prior to OS handover.
4349          */
4350         if (iommu->gcmd & DMA_GCMD_TE)
4351                 iommu_disable_translation(iommu);
4352
4353         g_iommus[iommu->seq_id] = iommu;
4354         ret = iommu_init_domains(iommu);
4355         if (ret == 0)
4356                 ret = iommu_alloc_root_entry(iommu);
4357         if (ret)
4358                 goto out;
4359
4360 #ifdef CONFIG_INTEL_IOMMU_SVM
4361         if (pasid_enabled(iommu))
4362                 intel_svm_init(iommu);
4363 #endif
4364
4365         if (dmaru->ignored) {
4366                 /*
4367                  * we always have to disable PMRs or DMA may fail on this device
4368                  */
4369                 if (force_on)
4370                         iommu_disable_protect_mem_regions(iommu);
4371                 return 0;
4372         }
4373
4374         intel_iommu_init_qi(iommu);
4375         iommu_flush_write_buffer(iommu);
4376
4377 #ifdef CONFIG_INTEL_IOMMU_SVM
4378         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4379                 ret = intel_svm_enable_prq(iommu);
4380                 if (ret)
4381                         goto disable_iommu;
4382         }
4383 #endif
4384         ret = dmar_set_interrupt(iommu);
4385         if (ret)
4386                 goto disable_iommu;
4387
4388         iommu_set_root_entry(iommu);
4389         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4390         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4391         iommu_enable_translation(iommu);
4392
4393         iommu_disable_protect_mem_regions(iommu);
4394         return 0;
4395
4396 disable_iommu:
4397         disable_dmar_iommu(iommu);
4398 out:
4399         free_dmar_iommu(iommu);
4400         return ret;
4401 }
4402
4403 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4404 {
4405         int ret = 0;
4406         struct intel_iommu *iommu = dmaru->iommu;
4407
4408         if (!intel_iommu_enabled)
4409                 return 0;
4410         if (iommu == NULL)
4411                 return -EINVAL;
4412
4413         if (insert) {
4414                 ret = intel_iommu_add(dmaru);
4415         } else {
4416                 disable_dmar_iommu(iommu);
4417                 free_dmar_iommu(iommu);
4418         }
4419
4420         return ret;
4421 }
4422
4423 static void intel_iommu_free_dmars(void)
4424 {
4425         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4426         struct dmar_atsr_unit *atsru, *atsr_n;
4427
4428         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4429                 list_del(&rmrru->list);
4430                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4431                 kfree(rmrru->resv);
4432                 kfree(rmrru);
4433         }
4434
4435         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4436                 list_del(&atsru->list);
4437                 intel_iommu_free_atsr(atsru);
4438         }
4439 }
4440
4441 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4442 {
4443         int i, ret = 1;
4444         struct pci_bus *bus;
4445         struct pci_dev *bridge = NULL;
4446         struct device *tmp;
4447         struct acpi_dmar_atsr *atsr;
4448         struct dmar_atsr_unit *atsru;
4449
4450         dev = pci_physfn(dev);
4451         for (bus = dev->bus; bus; bus = bus->parent) {
4452                 bridge = bus->self;
4453                 /* If it's an integrated device, allow ATS */
4454                 if (!bridge)
4455                         return 1;
4456                 /* Connected via non-PCIe: no ATS */
4457                 if (!pci_is_pcie(bridge) ||
4458                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4459                         return 0;
4460                 /* If we found the root port, look it up in the ATSR */
4461                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4462                         break;
4463         }
4464
4465         rcu_read_lock();
4466         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4467                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4468                 if (atsr->segment != pci_domain_nr(dev->bus))
4469                         continue;
4470
4471                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4472                         if (tmp == &bridge->dev)
4473                                 goto out;
4474
4475                 if (atsru->include_all)
4476                         goto out;
4477         }
4478         ret = 0;
4479 out:
4480         rcu_read_unlock();
4481
4482         return ret;
4483 }
4484
4485 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4486 {
4487         int ret = 0;
4488         struct dmar_rmrr_unit *rmrru;
4489         struct dmar_atsr_unit *atsru;
4490         struct acpi_dmar_atsr *atsr;
4491         struct acpi_dmar_reserved_memory *rmrr;
4492
4493         if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4494                 return 0;
4495
4496         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4497                 rmrr = container_of(rmrru->hdr,
4498                                     struct acpi_dmar_reserved_memory, header);
4499                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4500                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4501                                 ((void *)rmrr) + rmrr->header.length,
4502                                 rmrr->segment, rmrru->devices,
4503                                 rmrru->devices_cnt);
4504                         if(ret < 0)
4505                                 return ret;
4506                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4507                         dmar_remove_dev_scope(info, rmrr->segment,
4508                                 rmrru->devices, rmrru->devices_cnt);
4509                 }
4510         }
4511
4512         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4513                 if (atsru->include_all)
4514                         continue;
4515
4516                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4517                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4518                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4519                                         (void *)atsr + atsr->header.length,
4520                                         atsr->segment, atsru->devices,
4521                                         atsru->devices_cnt);
4522                         if (ret > 0)
4523                                 break;
4524                         else if(ret < 0)
4525                                 return ret;
4526                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4527                         if (dmar_remove_dev_scope(info, atsr->segment,
4528                                         atsru->devices, atsru->devices_cnt))
4529                                 break;
4530                 }
4531         }
4532
4533         return 0;
4534 }
4535
4536 /*
4537  * Here we only respond to action of unbound device from driver.
4538  *
4539  * Added device is not attached to its DMAR domain here yet. That will happen
4540  * when mapping the device to iova.
4541  */
4542 static int device_notifier(struct notifier_block *nb,
4543                                   unsigned long action, void *data)
4544 {
4545         struct device *dev = data;
4546         struct dmar_domain *domain;
4547
4548         if (iommu_dummy(dev))
4549                 return 0;
4550
4551         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4552                 return 0;
4553
4554         domain = find_domain(dev);
4555         if (!domain)
4556                 return 0;
4557
4558         dmar_remove_one_dev_info(domain, dev);
4559         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4560                 domain_exit(domain);
4561
4562         return 0;
4563 }
4564
4565 static struct notifier_block device_nb = {
4566         .notifier_call = device_notifier,
4567 };
4568
4569 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4570                                        unsigned long val, void *v)
4571 {
4572         struct memory_notify *mhp = v;
4573         unsigned long long start, end;
4574         unsigned long start_vpfn, last_vpfn;
4575
4576         switch (val) {
4577         case MEM_GOING_ONLINE:
4578                 start = mhp->start_pfn << PAGE_SHIFT;
4579                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4580                 if (iommu_domain_identity_map(si_domain, start, end)) {
4581                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4582                                 start, end);
4583                         return NOTIFY_BAD;
4584                 }
4585                 break;
4586
4587         case MEM_OFFLINE:
4588         case MEM_CANCEL_ONLINE:
4589                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4590                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4591                 while (start_vpfn <= last_vpfn) {
4592                         struct iova *iova;
4593                         struct dmar_drhd_unit *drhd;
4594                         struct intel_iommu *iommu;
4595                         struct page *freelist;
4596
4597                         iova = find_iova(&si_domain->iovad, start_vpfn);
4598                         if (iova == NULL) {
4599                                 pr_debug("Failed get IOVA for PFN %lx\n",
4600                                          start_vpfn);
4601                                 break;
4602                         }
4603
4604                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4605                                                      start_vpfn, last_vpfn);
4606                         if (iova == NULL) {
4607                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4608                                         start_vpfn, last_vpfn);
4609                                 return NOTIFY_BAD;
4610                         }
4611
4612                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4613                                                iova->pfn_hi);
4614
4615                         rcu_read_lock();
4616                         for_each_active_iommu(iommu, drhd)
4617                                 iommu_flush_iotlb_psi(iommu, si_domain,
4618                                         iova->pfn_lo, iova_size(iova),
4619                                         !freelist, 0);
4620                         rcu_read_unlock();
4621                         dma_free_pagelist(freelist);
4622
4623                         start_vpfn = iova->pfn_hi + 1;
4624                         free_iova_mem(iova);
4625                 }
4626                 break;
4627         }
4628
4629         return NOTIFY_OK;
4630 }
4631
4632 static struct notifier_block intel_iommu_memory_nb = {
4633         .notifier_call = intel_iommu_memory_notifier,
4634         .priority = 0
4635 };
4636
4637 static void free_all_cpu_cached_iovas(unsigned int cpu)
4638 {
4639         int i;
4640
4641         for (i = 0; i < g_num_of_iommus; i++) {
4642                 struct intel_iommu *iommu = g_iommus[i];
4643                 struct dmar_domain *domain;
4644                 int did;
4645
4646                 if (!iommu)
4647                         continue;
4648
4649                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4650                         domain = get_iommu_domain(iommu, (u16)did);
4651
4652                         if (!domain)
4653                                 continue;
4654                         free_cpu_cached_iovas(cpu, &domain->iovad);
4655                 }
4656         }
4657 }
4658
4659 static int intel_iommu_cpu_dead(unsigned int cpu)
4660 {
4661         free_all_cpu_cached_iovas(cpu);
4662         return 0;
4663 }
4664
4665 static void intel_disable_iommus(void)
4666 {
4667         struct intel_iommu *iommu = NULL;
4668         struct dmar_drhd_unit *drhd;
4669
4670         for_each_iommu(iommu, drhd)
4671                 iommu_disable_translation(iommu);
4672 }
4673
4674 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4675 {
4676         struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4677
4678         return container_of(iommu_dev, struct intel_iommu, iommu);
4679 }
4680
4681 static ssize_t intel_iommu_show_version(struct device *dev,
4682                                         struct device_attribute *attr,
4683                                         char *buf)
4684 {
4685         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4686         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4687         return sprintf(buf, "%d:%d\n",
4688                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4689 }
4690 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4691
4692 static ssize_t intel_iommu_show_address(struct device *dev,
4693                                         struct device_attribute *attr,
4694                                         char *buf)
4695 {
4696         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4697         return sprintf(buf, "%llx\n", iommu->reg_phys);
4698 }
4699 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4700
4701 static ssize_t intel_iommu_show_cap(struct device *dev,
4702                                     struct device_attribute *attr,
4703                                     char *buf)
4704 {
4705         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4706         return sprintf(buf, "%llx\n", iommu->cap);
4707 }
4708 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4709
4710 static ssize_t intel_iommu_show_ecap(struct device *dev,
4711                                     struct device_attribute *attr,
4712                                     char *buf)
4713 {
4714         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4715         return sprintf(buf, "%llx\n", iommu->ecap);
4716 }
4717 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4718
4719 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4720                                       struct device_attribute *attr,
4721                                       char *buf)
4722 {
4723         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4724         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4725 }
4726 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4727
4728 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4729                                            struct device_attribute *attr,
4730                                            char *buf)
4731 {
4732         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4733         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4734                                                   cap_ndoms(iommu->cap)));
4735 }
4736 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4737
4738 static struct attribute *intel_iommu_attrs[] = {
4739         &dev_attr_version.attr,
4740         &dev_attr_address.attr,
4741         &dev_attr_cap.attr,
4742         &dev_attr_ecap.attr,
4743         &dev_attr_domains_supported.attr,
4744         &dev_attr_domains_used.attr,
4745         NULL,
4746 };
4747
4748 static struct attribute_group intel_iommu_group = {
4749         .name = "intel-iommu",
4750         .attrs = intel_iommu_attrs,
4751 };
4752
4753 const struct attribute_group *intel_iommu_groups[] = {
4754         &intel_iommu_group,
4755         NULL,
4756 };
4757
4758 int __init intel_iommu_init(void)
4759 {
4760         int ret = -ENODEV;
4761         struct dmar_drhd_unit *drhd;
4762         struct intel_iommu *iommu;
4763
4764         /* VT-d is required for a TXT/tboot launch, so enforce that */
4765         force_on = tboot_force_iommu();
4766
4767         if (iommu_init_mempool()) {
4768                 if (force_on)
4769                         panic("tboot: Failed to initialize iommu memory\n");
4770                 return -ENOMEM;
4771         }
4772
4773         down_write(&dmar_global_lock);
4774         if (dmar_table_init()) {
4775                 if (force_on)
4776                         panic("tboot: Failed to initialize DMAR table\n");
4777                 goto out_free_dmar;
4778         }
4779
4780         if (dmar_dev_scope_init() < 0) {
4781                 if (force_on)
4782                         panic("tboot: Failed to initialize DMAR device scope\n");
4783                 goto out_free_dmar;
4784         }
4785
4786         up_write(&dmar_global_lock);
4787
4788         /*
4789          * The bus notifier takes the dmar_global_lock, so lockdep will
4790          * complain later when we register it under the lock.
4791          */
4792         dmar_register_bus_notifier();
4793
4794         down_write(&dmar_global_lock);
4795
4796         if (no_iommu || dmar_disabled) {
4797                 /*
4798                  * We exit the function here to ensure IOMMU's remapping and
4799                  * mempool aren't setup, which means that the IOMMU's PMRs
4800                  * won't be disabled via the call to init_dmars(). So disable
4801                  * it explicitly here. The PMRs were setup by tboot prior to
4802                  * calling SENTER, but the kernel is expected to reset/tear
4803                  * down the PMRs.
4804                  */
4805                 if (intel_iommu_tboot_noforce) {
4806                         for_each_iommu(iommu, drhd)
4807                                 iommu_disable_protect_mem_regions(iommu);
4808                 }
4809
4810                 /*
4811                  * Make sure the IOMMUs are switched off, even when we
4812                  * boot into a kexec kernel and the previous kernel left
4813                  * them enabled
4814                  */
4815                 intel_disable_iommus();
4816                 goto out_free_dmar;
4817         }
4818
4819         if (list_empty(&dmar_rmrr_units))
4820                 pr_info("No RMRR found\n");
4821
4822         if (list_empty(&dmar_atsr_units))
4823                 pr_info("No ATSR found\n");
4824
4825         if (dmar_init_reserved_ranges()) {
4826                 if (force_on)
4827                         panic("tboot: Failed to reserve iommu ranges\n");
4828                 goto out_free_reserved_range;
4829         }
4830
4831         init_no_remapping_devices();
4832
4833         ret = init_dmars();
4834         if (ret) {
4835                 if (force_on)
4836                         panic("tboot: Failed to initialize DMARs\n");
4837                 pr_err("Initialization failed\n");
4838                 goto out_free_reserved_range;
4839         }
4840         up_write(&dmar_global_lock);
4841         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4842
4843 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4844         swiotlb = 0;
4845 #endif
4846         dma_ops = &intel_dma_ops;
4847
4848         init_iommu_pm_ops();
4849
4850         for_each_active_iommu(iommu, drhd) {
4851                 iommu_device_sysfs_add(&iommu->iommu, NULL,
4852                                        intel_iommu_groups,
4853                                        "%s", iommu->name);
4854                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4855                 iommu_device_register(&iommu->iommu);
4856         }
4857
4858         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4859         bus_register_notifier(&pci_bus_type, &device_nb);
4860         if (si_domain && !hw_pass_through)
4861                 register_memory_notifier(&intel_iommu_memory_nb);
4862         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4863                           intel_iommu_cpu_dead);
4864         intel_iommu_enabled = 1;
4865
4866         return 0;
4867
4868 out_free_reserved_range:
4869         put_iova_domain(&reserved_iova_list);
4870 out_free_dmar:
4871         intel_iommu_free_dmars();
4872         up_write(&dmar_global_lock);
4873         iommu_exit_mempool();
4874         return ret;
4875 }
4876
4877 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4878 {
4879         struct intel_iommu *iommu = opaque;
4880
4881         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4882         return 0;
4883 }
4884
4885 /*
4886  * NB - intel-iommu lacks any sort of reference counting for the users of
4887  * dependent devices.  If multiple endpoints have intersecting dependent
4888  * devices, unbinding the driver from any one of them will possibly leave
4889  * the others unable to operate.
4890  */
4891 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4892 {
4893         if (!iommu || !dev || !dev_is_pci(dev))
4894                 return;
4895
4896         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4897 }
4898
4899 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4900 {
4901         struct intel_iommu *iommu;
4902         unsigned long flags;
4903
4904         assert_spin_locked(&device_domain_lock);
4905
4906         if (WARN_ON(!info))
4907                 return;
4908
4909         iommu = info->iommu;
4910
4911         if (info->dev) {
4912                 iommu_disable_dev_iotlb(info);
4913                 domain_context_clear(iommu, info->dev);
4914                 intel_pasid_free_table(info->dev);
4915         }
4916
4917         unlink_domain_info(info);
4918
4919         spin_lock_irqsave(&iommu->lock, flags);
4920         domain_detach_iommu(info->domain, iommu);
4921         spin_unlock_irqrestore(&iommu->lock, flags);
4922
4923         free_devinfo_mem(info);
4924 }
4925
4926 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4927                                      struct device *dev)
4928 {
4929         struct device_domain_info *info;
4930         unsigned long flags;
4931
4932         spin_lock_irqsave(&device_domain_lock, flags);
4933         info = dev->archdata.iommu;
4934         __dmar_remove_one_dev_info(info);
4935         spin_unlock_irqrestore(&device_domain_lock, flags);
4936 }
4937
4938 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4939 {
4940         int adjust_width;
4941
4942         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4943         domain_reserve_special_ranges(domain);
4944
4945         /* calculate AGAW */
4946         domain->gaw = guest_width;
4947         adjust_width = guestwidth_to_adjustwidth(guest_width);
4948         domain->agaw = width_to_agaw(adjust_width);
4949
4950         domain->iommu_coherency = 0;
4951         domain->iommu_snooping = 0;
4952         domain->iommu_superpage = 0;
4953         domain->max_addr = 0;
4954
4955         /* always allocate the top pgd */
4956         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4957         if (!domain->pgd)
4958                 return -ENOMEM;
4959         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4960         return 0;
4961 }
4962
4963 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4964 {
4965         struct dmar_domain *dmar_domain;
4966         struct iommu_domain *domain;
4967
4968         if (type != IOMMU_DOMAIN_UNMANAGED)
4969                 return NULL;
4970
4971         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4972         if (!dmar_domain) {
4973                 pr_err("Can't allocate dmar_domain\n");
4974                 return NULL;
4975         }
4976         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4977                 pr_err("Domain initialization failed\n");
4978                 domain_exit(dmar_domain);
4979                 return NULL;
4980         }
4981         domain_update_iommu_cap(dmar_domain);
4982
4983         domain = &dmar_domain->domain;
4984         domain->geometry.aperture_start = 0;
4985         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4986         domain->geometry.force_aperture = true;
4987
4988         return domain;
4989 }
4990
4991 static void intel_iommu_domain_free(struct iommu_domain *domain)
4992 {
4993         domain_exit(to_dmar_domain(domain));
4994 }
4995
4996 static int intel_iommu_attach_device(struct iommu_domain *domain,
4997                                      struct device *dev)
4998 {
4999         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5000         struct intel_iommu *iommu;
5001         int addr_width;
5002         u8 bus, devfn;
5003
5004         if (device_is_rmrr_locked(dev)) {
5005                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5006                 return -EPERM;
5007         }
5008
5009         /* normally dev is not mapped */
5010         if (unlikely(domain_context_mapped(dev))) {
5011                 struct dmar_domain *old_domain;
5012
5013                 old_domain = find_domain(dev);
5014                 if (old_domain) {
5015                         rcu_read_lock();
5016                         dmar_remove_one_dev_info(old_domain, dev);
5017                         rcu_read_unlock();
5018
5019                         if (!domain_type_is_vm_or_si(old_domain) &&
5020                              list_empty(&old_domain->devices))
5021                                 domain_exit(old_domain);
5022                 }
5023         }
5024
5025         iommu = device_to_iommu(dev, &bus, &devfn);
5026         if (!iommu)
5027                 return -ENODEV;
5028
5029         /* check if this iommu agaw is sufficient for max mapped address */
5030         addr_width = agaw_to_width(iommu->agaw);
5031         if (addr_width > cap_mgaw(iommu->cap))
5032                 addr_width = cap_mgaw(iommu->cap);
5033
5034         if (dmar_domain->max_addr > (1LL << addr_width)) {
5035                 pr_err("%s: iommu width (%d) is not "
5036                        "sufficient for the mapped address (%llx)\n",
5037                        __func__, addr_width, dmar_domain->max_addr);
5038                 return -EFAULT;
5039         }
5040         dmar_domain->gaw = addr_width;
5041
5042         /*
5043          * Knock out extra levels of page tables if necessary
5044          */
5045         while (iommu->agaw < dmar_domain->agaw) {
5046                 struct dma_pte *pte;
5047
5048                 pte = dmar_domain->pgd;
5049                 if (dma_pte_present(pte)) {
5050                         dmar_domain->pgd = (struct dma_pte *)
5051                                 phys_to_virt(dma_pte_addr(pte));
5052                         free_pgtable_page(pte);
5053                 }
5054                 dmar_domain->agaw--;
5055         }
5056
5057         return domain_add_dev_info(dmar_domain, dev);
5058 }
5059
5060 static void intel_iommu_detach_device(struct iommu_domain *domain,
5061                                       struct device *dev)
5062 {
5063         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5064 }
5065
5066 static int intel_iommu_map(struct iommu_domain *domain,
5067                            unsigned long iova, phys_addr_t hpa,
5068                            size_t size, int iommu_prot)
5069 {
5070         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5071         u64 max_addr;
5072         int prot = 0;
5073         int ret;
5074
5075         if (iommu_prot & IOMMU_READ)
5076                 prot |= DMA_PTE_READ;
5077         if (iommu_prot & IOMMU_WRITE)
5078                 prot |= DMA_PTE_WRITE;
5079         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5080                 prot |= DMA_PTE_SNP;
5081
5082         max_addr = iova + size;
5083         if (dmar_domain->max_addr < max_addr) {
5084                 u64 end;
5085
5086                 /* check if minimum agaw is sufficient for mapped address */
5087                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5088                 if (end < max_addr) {
5089                         pr_err("%s: iommu width (%d) is not "
5090                                "sufficient for the mapped address (%llx)\n",
5091                                __func__, dmar_domain->gaw, max_addr);
5092                         return -EFAULT;
5093                 }
5094                 dmar_domain->max_addr = max_addr;
5095         }
5096         /* Round up size to next multiple of PAGE_SIZE, if it and
5097            the low bits of hpa would take us onto the next page */
5098         size = aligned_nrpages(hpa, size);
5099         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5100                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5101         return ret;
5102 }
5103
5104 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5105                                 unsigned long iova, size_t size)
5106 {
5107         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5108         struct page *freelist = NULL;
5109         unsigned long start_pfn, last_pfn;
5110         unsigned int npages;
5111         int iommu_id, level = 0;
5112
5113         /* Cope with horrid API which requires us to unmap more than the
5114            size argument if it happens to be a large-page mapping. */
5115         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5116
5117         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5118                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5119
5120         start_pfn = iova >> VTD_PAGE_SHIFT;
5121         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5122
5123         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5124
5125         npages = last_pfn - start_pfn + 1;
5126
5127         for_each_domain_iommu(iommu_id, dmar_domain)
5128                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5129                                       start_pfn, npages, !freelist, 0);
5130
5131         dma_free_pagelist(freelist);
5132
5133         if (dmar_domain->max_addr == iova + size)
5134                 dmar_domain->max_addr = iova;
5135
5136         return size;
5137 }
5138
5139 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5140                                             dma_addr_t iova)
5141 {
5142         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5143         struct dma_pte *pte;
5144         int level = 0;
5145         u64 phys = 0;
5146
5147         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5148         if (pte)
5149                 phys = dma_pte_addr(pte);
5150
5151         return phys;
5152 }
5153
5154 static bool intel_iommu_capable(enum iommu_cap cap)
5155 {
5156         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5157                 return domain_update_iommu_snooping(NULL) == 1;
5158         if (cap == IOMMU_CAP_INTR_REMAP)
5159                 return irq_remapping_enabled == 1;
5160
5161         return false;
5162 }
5163
5164 static int intel_iommu_add_device(struct device *dev)
5165 {
5166         struct intel_iommu *iommu;
5167         struct iommu_group *group;
5168         u8 bus, devfn;
5169
5170         iommu = device_to_iommu(dev, &bus, &devfn);
5171         if (!iommu)
5172                 return -ENODEV;
5173
5174         iommu_device_link(&iommu->iommu, dev);
5175
5176         group = iommu_group_get_for_dev(dev);
5177
5178         if (IS_ERR(group))
5179                 return PTR_ERR(group);
5180
5181         iommu_group_put(group);
5182         return 0;
5183 }
5184
5185 static void intel_iommu_remove_device(struct device *dev)
5186 {
5187         struct intel_iommu *iommu;
5188         u8 bus, devfn;
5189
5190         iommu = device_to_iommu(dev, &bus, &devfn);
5191         if (!iommu)
5192                 return;
5193
5194         iommu_group_remove_device(dev);
5195
5196         iommu_device_unlink(&iommu->iommu, dev);
5197 }
5198
5199 static void intel_iommu_get_resv_regions(struct device *device,
5200                                          struct list_head *head)
5201 {
5202         struct iommu_resv_region *reg;
5203         struct dmar_rmrr_unit *rmrr;
5204         struct device *i_dev;
5205         int i;
5206
5207         rcu_read_lock();
5208         for_each_rmrr_units(rmrr) {
5209                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5210                                           i, i_dev) {
5211                         if (i_dev != device)
5212                                 continue;
5213
5214                         list_add_tail(&rmrr->resv->list, head);
5215                 }
5216         }
5217         rcu_read_unlock();
5218
5219         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5220                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5221                                       0, IOMMU_RESV_MSI);
5222         if (!reg)
5223                 return;
5224         list_add_tail(&reg->list, head);
5225 }
5226
5227 static void intel_iommu_put_resv_regions(struct device *dev,
5228                                          struct list_head *head)
5229 {
5230         struct iommu_resv_region *entry, *next;
5231
5232         list_for_each_entry_safe(entry, next, head, list) {
5233                 if (entry->type == IOMMU_RESV_RESERVED)
5234                         kfree(entry);
5235         }
5236 }
5237
5238 #ifdef CONFIG_INTEL_IOMMU_SVM
5239 #define MAX_NR_PASID_BITS (20)
5240 static inline unsigned long intel_iommu_get_pts(struct device *dev)
5241 {
5242         int pts, max_pasid;
5243
5244         max_pasid = intel_pasid_get_dev_max_id(dev);
5245         pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5246         if (pts < 5)
5247                 return 0;
5248
5249         return pts - 5;
5250 }
5251
5252 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5253 {
5254         struct device_domain_info *info;
5255         struct context_entry *context;
5256         struct dmar_domain *domain;
5257         unsigned long flags;
5258         u64 ctx_lo;
5259         int ret;
5260
5261         domain = get_valid_domain_for_dev(sdev->dev);
5262         if (!domain)
5263                 return -EINVAL;
5264
5265         spin_lock_irqsave(&device_domain_lock, flags);
5266         spin_lock(&iommu->lock);
5267
5268         ret = -EINVAL;
5269         info = sdev->dev->archdata.iommu;
5270         if (!info || !info->pasid_supported)
5271                 goto out;
5272
5273         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5274         if (WARN_ON(!context))
5275                 goto out;
5276
5277         ctx_lo = context[0].lo;
5278
5279         sdev->did = domain->iommu_did[iommu->seq_id];
5280         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5281
5282         if (!(ctx_lo & CONTEXT_PASIDE)) {
5283                 if (iommu->pasid_state_table)
5284                         context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5285                 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5286                         intel_iommu_get_pts(sdev->dev);
5287
5288                 wmb();
5289                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5290                  * extended to permit requests-with-PASID if the PASIDE bit
5291                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5292                  * however, the PASIDE bit is ignored and requests-with-PASID
5293                  * are unconditionally blocked. Which makes less sense.
5294                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5295                  * "guest mode" translation types depending on whether ATS
5296                  * is available or not. Annoyingly, we can't use the new
5297                  * modes *unless* PASIDE is set. */
5298                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5299                         ctx_lo &= ~CONTEXT_TT_MASK;
5300                         if (info->ats_supported)
5301                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5302                         else
5303                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5304                 }
5305                 ctx_lo |= CONTEXT_PASIDE;
5306                 if (iommu->pasid_state_table)
5307                         ctx_lo |= CONTEXT_DINVE;
5308                 if (info->pri_supported)
5309                         ctx_lo |= CONTEXT_PRS;
5310                 context[0].lo = ctx_lo;
5311                 wmb();
5312                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5313                                            DMA_CCMD_MASK_NOBIT,
5314                                            DMA_CCMD_DEVICE_INVL);
5315         }
5316
5317         /* Enable PASID support in the device, if it wasn't already */
5318         if (!info->pasid_enabled)
5319                 iommu_enable_dev_iotlb(info);
5320
5321         if (info->ats_enabled) {
5322                 sdev->dev_iotlb = 1;
5323                 sdev->qdep = info->ats_qdep;
5324                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5325                         sdev->qdep = 0;
5326         }
5327         ret = 0;
5328
5329  out:
5330         spin_unlock(&iommu->lock);
5331         spin_unlock_irqrestore(&device_domain_lock, flags);
5332
5333         return ret;
5334 }
5335
5336 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5337 {
5338         struct intel_iommu *iommu;
5339         u8 bus, devfn;
5340
5341         if (iommu_dummy(dev)) {
5342                 dev_warn(dev,
5343                          "No IOMMU translation for device; cannot enable SVM\n");
5344                 return NULL;
5345         }
5346
5347         iommu = device_to_iommu(dev, &bus, &devfn);
5348         if ((!iommu)) {
5349                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5350                 return NULL;
5351         }
5352
5353         return iommu;
5354 }
5355 #endif /* CONFIG_INTEL_IOMMU_SVM */
5356
5357 const struct iommu_ops intel_iommu_ops = {
5358         .capable                = intel_iommu_capable,
5359         .domain_alloc           = intel_iommu_domain_alloc,
5360         .domain_free            = intel_iommu_domain_free,
5361         .attach_dev             = intel_iommu_attach_device,
5362         .detach_dev             = intel_iommu_detach_device,
5363         .map                    = intel_iommu_map,
5364         .unmap                  = intel_iommu_unmap,
5365         .iova_to_phys           = intel_iommu_iova_to_phys,
5366         .add_device             = intel_iommu_add_device,
5367         .remove_device          = intel_iommu_remove_device,
5368         .get_resv_regions       = intel_iommu_get_resv_regions,
5369         .put_resv_regions       = intel_iommu_put_resv_regions,
5370         .device_group           = pci_device_group,
5371         .pgsize_bitmap          = INTEL_IOMMU_PGSIZES,
5372 };
5373
5374 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5375 {
5376         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5377         pr_info("Disabling IOMMU for graphics on this chipset\n");
5378         dmar_map_gfx = 0;
5379 }
5380
5381 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5383 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5384 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5385 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5386 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5387 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5388
5389 static void quirk_iommu_rwbf(struct pci_dev *dev)
5390 {
5391         /*
5392          * Mobile 4 Series Chipset neglects to set RWBF capability,
5393          * but needs it. Same seems to hold for the desktop versions.
5394          */
5395         pr_info("Forcing write-buffer flush capability\n");
5396         rwbf_quirk = 1;
5397 }
5398
5399 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5400 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5401 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5402 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5403 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5404 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5405 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5406
5407 #define GGC 0x52
5408 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5409 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5410 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5411 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5412 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5413 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5414 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5415 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5416
5417 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5418 {
5419         unsigned short ggc;
5420
5421         if (pci_read_config_word(dev, GGC, &ggc))
5422                 return;
5423
5424         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5425                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5426                 dmar_map_gfx = 0;
5427         } else if (dmar_map_gfx) {
5428                 /* we have to ensure the gfx device is idle before we flush */
5429                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5430                 intel_iommu_strict = 1;
5431        }
5432 }
5433 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5434 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5435 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5436 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5437
5438 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5439    ISOCH DMAR unit for the Azalia sound device, but not give it any
5440    TLB entries, which causes it to deadlock. Check for that.  We do
5441    this in a function called from init_dmars(), instead of in a PCI
5442    quirk, because we don't want to print the obnoxious "BIOS broken"
5443    message if VT-d is actually disabled.
5444 */
5445 static void __init check_tylersburg_isoch(void)
5446 {
5447         struct pci_dev *pdev;
5448         uint32_t vtisochctrl;
5449
5450         /* If there's no Azalia in the system anyway, forget it. */
5451         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5452         if (!pdev)
5453                 return;
5454         pci_dev_put(pdev);
5455
5456         /* System Management Registers. Might be hidden, in which case
5457            we can't do the sanity check. But that's OK, because the
5458            known-broken BIOSes _don't_ actually hide it, so far. */
5459         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5460         if (!pdev)
5461                 return;
5462
5463         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5464                 pci_dev_put(pdev);
5465                 return;
5466         }
5467
5468         pci_dev_put(pdev);
5469
5470         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5471         if (vtisochctrl & 1)
5472                 return;
5473
5474         /* Drop all bits other than the number of TLB entries */
5475         vtisochctrl &= 0x1c;
5476
5477         /* If we have the recommended number of TLB entries (16), fine. */
5478         if (vtisochctrl == 0x10)
5479                 return;
5480
5481         /* Zero TLB entries? You get to ride the short bus to school. */
5482         if (!vtisochctrl) {
5483                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5484                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5485                      dmi_get_system_info(DMI_BIOS_VENDOR),
5486                      dmi_get_system_info(DMI_BIOS_VERSION),
5487                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5488                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5489                 return;
5490         }
5491
5492         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5493                vtisochctrl);
5494 }