2 * Copyright © 2006-2014 Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
21 #define pr_fmt(fmt) "DMAR: " fmt
22 #define dev_fmt(fmt) pr_fmt(fmt)
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/export.h>
28 #include <linux/slab.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/memory.h>
37 #include <linux/cpu.h>
38 #include <linux/timer.h>
40 #include <linux/iova.h>
41 #include <linux/iommu.h>
42 #include <linux/intel-iommu.h>
43 #include <linux/syscore_ops.h>
44 #include <linux/tboot.h>
45 #include <linux/dmi.h>
46 #include <linux/pci-ats.h>
47 #include <linux/memblock.h>
48 #include <linux/dma-contiguous.h>
49 #include <linux/dma-direct.h>
50 #include <linux/crash_dump.h>
51 #include <linux/numa.h>
52 #include <asm/irq_remapping.h>
53 #include <asm/cacheflush.h>
54 #include <asm/iommu.h>
56 #include "irq_remapping.h"
57 #include "intel-pasid.h"
59 #define ROOT_SIZE VTD_PAGE_SIZE
60 #define CONTEXT_SIZE VTD_PAGE_SIZE
62 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
63 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
64 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
65 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
67 #define IOAPIC_RANGE_START (0xfee00000)
68 #define IOAPIC_RANGE_END (0xfeefffff)
69 #define IOVA_START_ADDR (0x1000)
71 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
73 #define MAX_AGAW_WIDTH 64
74 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
76 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
77 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
79 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
80 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
81 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
82 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
83 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
85 /* IO virtual address start page frame number */
86 #define IOVA_START_PFN (1)
88 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
90 /* page table handling */
91 #define LEVEL_STRIDE (9)
92 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
95 * This bitmap is used to advertise the page sizes our hardware support
96 * to the IOMMU core, which will then use this information to split
97 * physically contiguous memory regions it is mapping into page sizes
100 * Traditionally the IOMMU core just handed us the mappings directly,
101 * after making sure the size is an order of a 4KiB page and that the
102 * mapping has natural alignment.
104 * To retain this behavior, we currently advertise that we support
105 * all page sizes that are an order of 4KiB.
107 * If at some point we'd like to utilize the IOMMU core's new behavior,
108 * we could change this to advertise the real page sizes we support.
110 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
112 static inline int agaw_to_level(int agaw)
117 static inline int agaw_to_width(int agaw)
119 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
122 static inline int width_to_agaw(int width)
124 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
127 static inline unsigned int level_to_offset_bits(int level)
129 return (level - 1) * LEVEL_STRIDE;
132 static inline int pfn_level_offset(unsigned long pfn, int level)
134 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
137 static inline unsigned long level_mask(int level)
139 return -1UL << level_to_offset_bits(level);
142 static inline unsigned long level_size(int level)
144 return 1UL << level_to_offset_bits(level);
147 static inline unsigned long align_to_level(unsigned long pfn, int level)
149 return (pfn + level_size(level) - 1) & level_mask(level);
152 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
154 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
157 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
158 are never going to work. */
159 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
161 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
164 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
166 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
168 static inline unsigned long page_to_dma_pfn(struct page *pg)
170 return mm_to_dma_pfn(page_to_pfn(pg));
172 static inline unsigned long virt_to_dma_pfn(void *p)
174 return page_to_dma_pfn(virt_to_page(p));
177 /* global iommu list, set NULL for ignored DMAR units */
178 static struct intel_iommu **g_iommus;
180 static void __init check_tylersburg_isoch(void);
181 static int rwbf_quirk;
184 * set to 1 to panic kernel if can't successfully enable VT-d
185 * (used when kernel is launched w/ TXT)
187 static int force_on = 0;
188 int intel_iommu_tboot_noforce;
189 static int no_platform_optin;
191 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
194 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
197 static phys_addr_t root_entry_lctp(struct root_entry *re)
202 return re->lo & VTD_PAGE_MASK;
206 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
209 static phys_addr_t root_entry_uctp(struct root_entry *re)
214 return re->hi & VTD_PAGE_MASK;
217 static inline void context_clear_pasid_enable(struct context_entry *context)
219 context->lo &= ~(1ULL << 11);
222 static inline bool context_pasid_enabled(struct context_entry *context)
224 return !!(context->lo & (1ULL << 11));
227 static inline void context_set_copied(struct context_entry *context)
229 context->hi |= (1ull << 3);
232 static inline bool context_copied(struct context_entry *context)
234 return !!(context->hi & (1ULL << 3));
237 static inline bool __context_present(struct context_entry *context)
239 return (context->lo & 1);
242 bool context_present(struct context_entry *context)
244 return context_pasid_enabled(context) ?
245 __context_present(context) :
246 __context_present(context) && !context_copied(context);
249 static inline void context_set_present(struct context_entry *context)
254 static inline void context_set_fault_enable(struct context_entry *context)
256 context->lo &= (((u64)-1) << 2) | 1;
259 static inline void context_set_translation_type(struct context_entry *context,
262 context->lo &= (((u64)-1) << 4) | 3;
263 context->lo |= (value & 3) << 2;
266 static inline void context_set_address_root(struct context_entry *context,
269 context->lo &= ~VTD_PAGE_MASK;
270 context->lo |= value & VTD_PAGE_MASK;
273 static inline void context_set_address_width(struct context_entry *context,
276 context->hi |= value & 7;
279 static inline void context_set_domain_id(struct context_entry *context,
282 context->hi |= (value & ((1 << 16) - 1)) << 8;
285 static inline int context_domain_id(struct context_entry *c)
287 return((c->hi >> 8) & 0xffff);
290 static inline void context_clear_entry(struct context_entry *context)
297 * This domain is a statically identity mapping domain.
298 * 1. This domain creats a static 1:1 mapping to all usable memory.
299 * 2. It maps to each iommu if successful.
300 * 3. Each iommu mapps to this domain if successful.
302 static struct dmar_domain *si_domain;
303 static int hw_pass_through = 1;
306 * Domain represents a virtual machine, more than one devices
307 * across iommus may be owned in one domain, e.g. kvm guest.
309 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
311 /* si_domain contains mulitple devices */
312 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
314 #define for_each_domain_iommu(idx, domain) \
315 for (idx = 0; idx < g_num_of_iommus; idx++) \
316 if (domain->iommu_refcnt[idx])
318 struct dmar_rmrr_unit {
319 struct list_head list; /* list of rmrr units */
320 struct acpi_dmar_header *hdr; /* ACPI header */
321 u64 base_address; /* reserved base address*/
322 u64 end_address; /* reserved end address */
323 struct dmar_dev_scope *devices; /* target devices */
324 int devices_cnt; /* target device count */
325 struct iommu_resv_region *resv; /* reserved region handle */
328 struct dmar_atsr_unit {
329 struct list_head list; /* list of ATSR units */
330 struct acpi_dmar_header *hdr; /* ACPI header */
331 struct dmar_dev_scope *devices; /* target devices */
332 int devices_cnt; /* target device count */
333 u8 include_all:1; /* include all ports */
336 static LIST_HEAD(dmar_atsr_units);
337 static LIST_HEAD(dmar_rmrr_units);
339 #define for_each_rmrr_units(rmrr) \
340 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
342 /* bitmap for indexing intel_iommus */
343 static int g_num_of_iommus;
345 static void domain_exit(struct dmar_domain *domain);
346 static void domain_remove_dev_info(struct dmar_domain *domain);
347 static void dmar_remove_one_dev_info(struct device *dev);
348 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
349 static void domain_context_clear(struct intel_iommu *iommu,
351 static int domain_detach_iommu(struct dmar_domain *domain,
352 struct intel_iommu *iommu);
354 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
355 int dmar_disabled = 0;
357 int dmar_disabled = 1;
358 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
360 int intel_iommu_enabled = 0;
361 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
363 static int dmar_map_gfx = 1;
364 static int dmar_forcedac;
365 static int intel_iommu_strict;
366 static int intel_iommu_superpage = 1;
367 static int intel_iommu_sm;
368 static int iommu_identity_mapping;
370 #define IDENTMAP_ALL 1
371 #define IDENTMAP_GFX 2
372 #define IDENTMAP_AZALIA 4
374 #define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
375 #define pasid_supported(iommu) (sm_supported(iommu) && \
376 ecap_pasid((iommu)->ecap))
378 int intel_iommu_gfx_mapped;
379 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
381 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
382 static DEFINE_SPINLOCK(device_domain_lock);
383 static LIST_HEAD(device_domain_list);
386 * Iterate over elements in device_domain_list and call the specified
387 * callback @fn against each element.
389 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
390 void *data), void *data)
394 struct device_domain_info *info;
396 spin_lock_irqsave(&device_domain_lock, flags);
397 list_for_each_entry(info, &device_domain_list, global) {
398 ret = fn(info, data);
400 spin_unlock_irqrestore(&device_domain_lock, flags);
404 spin_unlock_irqrestore(&device_domain_lock, flags);
409 const struct iommu_ops intel_iommu_ops;
411 static bool translation_pre_enabled(struct intel_iommu *iommu)
413 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
416 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
418 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
421 static void init_translation_status(struct intel_iommu *iommu)
425 gsts = readl(iommu->reg + DMAR_GSTS_REG);
426 if (gsts & DMA_GSTS_TES)
427 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
430 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
431 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
433 return container_of(dom, struct dmar_domain, domain);
436 static int __init intel_iommu_setup(char *str)
441 if (!strncmp(str, "on", 2)) {
443 pr_info("IOMMU enabled\n");
444 } else if (!strncmp(str, "off", 3)) {
446 no_platform_optin = 1;
447 pr_info("IOMMU disabled\n");
448 } else if (!strncmp(str, "igfx_off", 8)) {
450 pr_info("Disable GFX device mapping\n");
451 } else if (!strncmp(str, "forcedac", 8)) {
452 pr_info("Forcing DAC for PCI devices\n");
454 } else if (!strncmp(str, "strict", 6)) {
455 pr_info("Disable batched IOTLB flush\n");
456 intel_iommu_strict = 1;
457 } else if (!strncmp(str, "sp_off", 6)) {
458 pr_info("Disable supported super page\n");
459 intel_iommu_superpage = 0;
460 } else if (!strncmp(str, "sm_on", 5)) {
461 pr_info("Intel-IOMMU: scalable mode supported\n");
463 } else if (!strncmp(str, "tboot_noforce", 13)) {
465 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
466 intel_iommu_tboot_noforce = 1;
469 str += strcspn(str, ",");
475 __setup("intel_iommu=", intel_iommu_setup);
477 static struct kmem_cache *iommu_domain_cache;
478 static struct kmem_cache *iommu_devinfo_cache;
480 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
482 struct dmar_domain **domains;
485 domains = iommu->domains[idx];
489 return domains[did & 0xff];
492 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
493 struct dmar_domain *domain)
495 struct dmar_domain **domains;
498 if (!iommu->domains[idx]) {
499 size_t size = 256 * sizeof(struct dmar_domain *);
500 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
503 domains = iommu->domains[idx];
504 if (WARN_ON(!domains))
507 domains[did & 0xff] = domain;
510 void *alloc_pgtable_page(int node)
515 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
517 vaddr = page_address(page);
521 void free_pgtable_page(void *vaddr)
523 free_page((unsigned long)vaddr);
526 static inline void *alloc_domain_mem(void)
528 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
531 static void free_domain_mem(void *vaddr)
533 kmem_cache_free(iommu_domain_cache, vaddr);
536 static inline void * alloc_devinfo_mem(void)
538 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
541 static inline void free_devinfo_mem(void *vaddr)
543 kmem_cache_free(iommu_devinfo_cache, vaddr);
546 static inline int domain_type_is_vm(struct dmar_domain *domain)
548 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
551 static inline int domain_type_is_si(struct dmar_domain *domain)
553 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
556 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
558 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
559 DOMAIN_FLAG_STATIC_IDENTITY);
562 static inline int domain_pfn_supported(struct dmar_domain *domain,
565 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
567 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
570 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
575 sagaw = cap_sagaw(iommu->cap);
576 for (agaw = width_to_agaw(max_gaw);
578 if (test_bit(agaw, &sagaw))
586 * Calculate max SAGAW for each iommu.
588 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
590 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
594 * calculate agaw for each iommu.
595 * "SAGAW" may be different across iommus, use a default agaw, and
596 * get a supported less agaw for iommus that don't support the default agaw.
598 int iommu_calculate_agaw(struct intel_iommu *iommu)
600 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
603 /* This functionin only returns single iommu in a domain */
604 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
608 /* si_domain and vm domain should not get here. */
609 BUG_ON(domain_type_is_vm_or_si(domain));
610 for_each_domain_iommu(iommu_id, domain)
613 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
616 return g_iommus[iommu_id];
619 static void domain_update_iommu_coherency(struct dmar_domain *domain)
621 struct dmar_drhd_unit *drhd;
622 struct intel_iommu *iommu;
626 domain->iommu_coherency = 1;
628 for_each_domain_iommu(i, domain) {
630 if (!ecap_coherent(g_iommus[i]->ecap)) {
631 domain->iommu_coherency = 0;
638 /* No hardware attached; use lowest common denominator */
640 for_each_active_iommu(iommu, drhd) {
641 if (!ecap_coherent(iommu->ecap)) {
642 domain->iommu_coherency = 0;
649 static int domain_update_iommu_snooping(struct intel_iommu *skip)
651 struct dmar_drhd_unit *drhd;
652 struct intel_iommu *iommu;
656 for_each_active_iommu(iommu, drhd) {
658 if (!ecap_sc_support(iommu->ecap)) {
669 static int domain_update_iommu_superpage(struct intel_iommu *skip)
671 struct dmar_drhd_unit *drhd;
672 struct intel_iommu *iommu;
675 if (!intel_iommu_superpage) {
679 /* set iommu_superpage to the smallest common denominator */
681 for_each_active_iommu(iommu, drhd) {
683 mask &= cap_super_page_val(iommu->cap);
693 /* Some capabilities may be different across iommus */
694 static void domain_update_iommu_cap(struct dmar_domain *domain)
696 domain_update_iommu_coherency(domain);
697 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
698 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
701 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
704 struct root_entry *root = &iommu->root_entry[bus];
705 struct context_entry *context;
709 if (sm_supported(iommu)) {
717 context = phys_to_virt(*entry & VTD_PAGE_MASK);
719 unsigned long phy_addr;
723 context = alloc_pgtable_page(iommu->node);
727 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
728 phy_addr = virt_to_phys((void *)context);
729 *entry = phy_addr | 1;
730 __iommu_flush_cache(iommu, entry, sizeof(*entry));
732 return &context[devfn];
735 static int iommu_dummy(struct device *dev)
737 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
740 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
742 struct dmar_drhd_unit *drhd = NULL;
743 struct intel_iommu *iommu;
745 struct pci_dev *ptmp, *pdev = NULL;
749 if (iommu_dummy(dev))
752 if (dev_is_pci(dev)) {
753 struct pci_dev *pf_pdev;
755 pdev = to_pci_dev(dev);
758 /* VMD child devices currently cannot be handled individually */
759 if (is_vmd(pdev->bus))
763 /* VFs aren't listed in scope tables; we need to look up
764 * the PF instead to find the IOMMU. */
765 pf_pdev = pci_physfn(pdev);
767 segment = pci_domain_nr(pdev->bus);
768 } else if (has_acpi_companion(dev))
769 dev = &ACPI_COMPANION(dev)->dev;
772 for_each_active_iommu(iommu, drhd) {
773 if (pdev && segment != drhd->segment)
776 for_each_active_dev_scope(drhd->devices,
777 drhd->devices_cnt, i, tmp) {
779 /* For a VF use its original BDF# not that of the PF
780 * which we used for the IOMMU lookup. Strictly speaking
781 * we could do this for all PCI devices; we only need to
782 * get the BDF# from the scope table for ACPI matches. */
783 if (pdev && pdev->is_virtfn)
786 *bus = drhd->devices[i].bus;
787 *devfn = drhd->devices[i].devfn;
791 if (!pdev || !dev_is_pci(tmp))
794 ptmp = to_pci_dev(tmp);
795 if (ptmp->subordinate &&
796 ptmp->subordinate->number <= pdev->bus->number &&
797 ptmp->subordinate->busn_res.end >= pdev->bus->number)
801 if (pdev && drhd->include_all) {
803 *bus = pdev->bus->number;
804 *devfn = pdev->devfn;
815 static void domain_flush_cache(struct dmar_domain *domain,
816 void *addr, int size)
818 if (!domain->iommu_coherency)
819 clflush_cache_range(addr, size);
822 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
824 struct context_entry *context;
828 spin_lock_irqsave(&iommu->lock, flags);
829 context = iommu_context_addr(iommu, bus, devfn, 0);
831 ret = context_present(context);
832 spin_unlock_irqrestore(&iommu->lock, flags);
836 static void free_context_table(struct intel_iommu *iommu)
840 struct context_entry *context;
842 spin_lock_irqsave(&iommu->lock, flags);
843 if (!iommu->root_entry) {
846 for (i = 0; i < ROOT_ENTRY_NR; i++) {
847 context = iommu_context_addr(iommu, i, 0, 0);
849 free_pgtable_page(context);
851 if (!sm_supported(iommu))
854 context = iommu_context_addr(iommu, i, 0x80, 0);
856 free_pgtable_page(context);
859 free_pgtable_page(iommu->root_entry);
860 iommu->root_entry = NULL;
862 spin_unlock_irqrestore(&iommu->lock, flags);
865 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
866 unsigned long pfn, int *target_level)
868 struct dma_pte *parent, *pte;
869 int level = agaw_to_level(domain->agaw);
872 BUG_ON(!domain->pgd);
874 if (!domain_pfn_supported(domain, pfn))
875 /* Address beyond IOMMU's addressing capabilities. */
878 parent = domain->pgd;
883 offset = pfn_level_offset(pfn, level);
884 pte = &parent[offset];
885 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
887 if (level == *target_level)
890 if (!dma_pte_present(pte)) {
893 tmp_page = alloc_pgtable_page(domain->nid);
898 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
899 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
900 if (cmpxchg64(&pte->val, 0ULL, pteval))
901 /* Someone else set it while we were thinking; use theirs. */
902 free_pgtable_page(tmp_page);
904 domain_flush_cache(domain, pte, sizeof(*pte));
909 parent = phys_to_virt(dma_pte_addr(pte));
914 *target_level = level;
920 /* return address's pte at specific level */
921 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
923 int level, int *large_page)
925 struct dma_pte *parent, *pte;
926 int total = agaw_to_level(domain->agaw);
929 parent = domain->pgd;
930 while (level <= total) {
931 offset = pfn_level_offset(pfn, total);
932 pte = &parent[offset];
936 if (!dma_pte_present(pte)) {
941 if (dma_pte_superpage(pte)) {
946 parent = phys_to_virt(dma_pte_addr(pte));
952 /* clear last level pte, a tlb flush should be followed */
953 static void dma_pte_clear_range(struct dmar_domain *domain,
954 unsigned long start_pfn,
955 unsigned long last_pfn)
957 unsigned int large_page;
958 struct dma_pte *first_pte, *pte;
960 BUG_ON(!domain_pfn_supported(domain, start_pfn));
961 BUG_ON(!domain_pfn_supported(domain, last_pfn));
962 BUG_ON(start_pfn > last_pfn);
964 /* we don't need lock here; nobody else touches the iova range */
967 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
969 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
974 start_pfn += lvl_to_nr_pages(large_page);
976 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
978 domain_flush_cache(domain, first_pte,
979 (void *)pte - (void *)first_pte);
981 } while (start_pfn && start_pfn <= last_pfn);
984 static void dma_pte_free_level(struct dmar_domain *domain, int level,
985 int retain_level, struct dma_pte *pte,
986 unsigned long pfn, unsigned long start_pfn,
987 unsigned long last_pfn)
989 pfn = max(start_pfn, pfn);
990 pte = &pte[pfn_level_offset(pfn, level)];
993 unsigned long level_pfn;
994 struct dma_pte *level_pte;
996 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
999 level_pfn = pfn & level_mask(level);
1000 level_pte = phys_to_virt(dma_pte_addr(pte));
1003 dma_pte_free_level(domain, level - 1, retain_level,
1004 level_pte, level_pfn, start_pfn,
1009 * Free the page table if we're below the level we want to
1010 * retain and the range covers the entire table.
1012 if (level < retain_level && !(start_pfn > level_pfn ||
1013 last_pfn < level_pfn + level_size(level) - 1)) {
1015 domain_flush_cache(domain, pte, sizeof(*pte));
1016 free_pgtable_page(level_pte);
1019 pfn += level_size(level);
1020 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1024 * clear last level (leaf) ptes and free page table pages below the
1025 * level we wish to keep intact.
1027 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1028 unsigned long start_pfn,
1029 unsigned long last_pfn,
1032 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1033 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1034 BUG_ON(start_pfn > last_pfn);
1036 dma_pte_clear_range(domain, start_pfn, last_pfn);
1038 /* We don't need lock here; nobody else touches the iova range */
1039 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1040 domain->pgd, 0, start_pfn, last_pfn);
1043 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1044 free_pgtable_page(domain->pgd);
1049 /* When a page at a given level is being unlinked from its parent, we don't
1050 need to *modify* it at all. All we need to do is make a list of all the
1051 pages which can be freed just as soon as we've flushed the IOTLB and we
1052 know the hardware page-walk will no longer touch them.
1053 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1055 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1056 int level, struct dma_pte *pte,
1057 struct page *freelist)
1061 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1062 pg->freelist = freelist;
1068 pte = page_address(pg);
1070 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1071 freelist = dma_pte_list_pagetables(domain, level - 1,
1074 } while (!first_pte_in_page(pte));
1079 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1080 struct dma_pte *pte, unsigned long pfn,
1081 unsigned long start_pfn,
1082 unsigned long last_pfn,
1083 struct page *freelist)
1085 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1087 pfn = max(start_pfn, pfn);
1088 pte = &pte[pfn_level_offset(pfn, level)];
1091 unsigned long level_pfn;
1093 if (!dma_pte_present(pte))
1096 level_pfn = pfn & level_mask(level);
1098 /* If range covers entire pagetable, free it */
1099 if (start_pfn <= level_pfn &&
1100 last_pfn >= level_pfn + level_size(level) - 1) {
1101 /* These suborbinate page tables are going away entirely. Don't
1102 bother to clear them; we're just going to *free* them. */
1103 if (level > 1 && !dma_pte_superpage(pte))
1104 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1110 } else if (level > 1) {
1111 /* Recurse down into a level that isn't *entirely* obsolete */
1112 freelist = dma_pte_clear_level(domain, level - 1,
1113 phys_to_virt(dma_pte_addr(pte)),
1114 level_pfn, start_pfn, last_pfn,
1118 pfn += level_size(level);
1119 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1122 domain_flush_cache(domain, first_pte,
1123 (void *)++last_pte - (void *)first_pte);
1128 /* We can't just free the pages because the IOMMU may still be walking
1129 the page tables, and may have cached the intermediate levels. The
1130 pages can only be freed after the IOTLB flush has been done. */
1131 static struct page *domain_unmap(struct dmar_domain *domain,
1132 unsigned long start_pfn,
1133 unsigned long last_pfn)
1135 struct page *freelist;
1137 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1138 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1139 BUG_ON(start_pfn > last_pfn);
1141 /* we don't need lock here; nobody else touches the iova range */
1142 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1143 domain->pgd, 0, start_pfn, last_pfn, NULL);
1146 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1147 struct page *pgd_page = virt_to_page(domain->pgd);
1148 pgd_page->freelist = freelist;
1149 freelist = pgd_page;
1157 static void dma_free_pagelist(struct page *freelist)
1161 while ((pg = freelist)) {
1162 freelist = pg->freelist;
1163 free_pgtable_page(page_address(pg));
1167 static void iova_entry_free(unsigned long data)
1169 struct page *freelist = (struct page *)data;
1171 dma_free_pagelist(freelist);
1174 /* iommu handling */
1175 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1177 struct root_entry *root;
1178 unsigned long flags;
1180 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1182 pr_err("Allocating root entry for %s failed\n",
1187 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1189 spin_lock_irqsave(&iommu->lock, flags);
1190 iommu->root_entry = root;
1191 spin_unlock_irqrestore(&iommu->lock, flags);
1196 static void iommu_set_root_entry(struct intel_iommu *iommu)
1202 addr = virt_to_phys(iommu->root_entry);
1203 if (sm_supported(iommu))
1204 addr |= DMA_RTADDR_SMT;
1206 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1207 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1209 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1211 /* Make sure hardware complete it */
1212 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1213 readl, (sts & DMA_GSTS_RTPS), sts);
1215 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1218 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1223 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1226 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1227 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1229 /* Make sure hardware complete it */
1230 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1231 readl, (!(val & DMA_GSTS_WBFS)), val);
1233 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1236 /* return value determine if we need a write buffer flush */
1237 static void __iommu_flush_context(struct intel_iommu *iommu,
1238 u16 did, u16 source_id, u8 function_mask,
1245 case DMA_CCMD_GLOBAL_INVL:
1246 val = DMA_CCMD_GLOBAL_INVL;
1248 case DMA_CCMD_DOMAIN_INVL:
1249 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1251 case DMA_CCMD_DEVICE_INVL:
1252 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1253 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1258 val |= DMA_CCMD_ICC;
1260 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1261 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1263 /* Make sure hardware complete it */
1264 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1265 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1267 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1270 /* return value determine if we need a write buffer flush */
1271 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1272 u64 addr, unsigned int size_order, u64 type)
1274 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1275 u64 val = 0, val_iva = 0;
1279 case DMA_TLB_GLOBAL_FLUSH:
1280 /* global flush doesn't need set IVA_REG */
1281 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1283 case DMA_TLB_DSI_FLUSH:
1284 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1286 case DMA_TLB_PSI_FLUSH:
1287 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1288 /* IH bit is passed in as part of address */
1289 val_iva = size_order | addr;
1294 /* Note: set drain read/write */
1297 * This is probably to be super secure.. Looks like we can
1298 * ignore it without any impact.
1300 if (cap_read_drain(iommu->cap))
1301 val |= DMA_TLB_READ_DRAIN;
1303 if (cap_write_drain(iommu->cap))
1304 val |= DMA_TLB_WRITE_DRAIN;
1306 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1307 /* Note: Only uses first TLB reg currently */
1309 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1310 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1312 /* Make sure hardware complete it */
1313 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1314 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1316 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1318 /* check IOTLB invalidation granularity */
1319 if (DMA_TLB_IAIG(val) == 0)
1320 pr_err("Flush IOTLB failed\n");
1321 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1322 pr_debug("TLB flush request %Lx, actual %Lx\n",
1323 (unsigned long long)DMA_TLB_IIRG(type),
1324 (unsigned long long)DMA_TLB_IAIG(val));
1327 static struct device_domain_info *
1328 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1331 struct device_domain_info *info;
1333 assert_spin_locked(&device_domain_lock);
1338 list_for_each_entry(info, &domain->devices, link)
1339 if (info->iommu == iommu && info->bus == bus &&
1340 info->devfn == devfn) {
1341 if (info->ats_supported && info->dev)
1349 static void domain_update_iotlb(struct dmar_domain *domain)
1351 struct device_domain_info *info;
1352 bool has_iotlb_device = false;
1354 assert_spin_locked(&device_domain_lock);
1356 list_for_each_entry(info, &domain->devices, link) {
1357 struct pci_dev *pdev;
1359 if (!info->dev || !dev_is_pci(info->dev))
1362 pdev = to_pci_dev(info->dev);
1363 if (pdev->ats_enabled) {
1364 has_iotlb_device = true;
1369 domain->has_iotlb_device = has_iotlb_device;
1372 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1374 struct pci_dev *pdev;
1376 assert_spin_locked(&device_domain_lock);
1378 if (!info || !dev_is_pci(info->dev))
1381 pdev = to_pci_dev(info->dev);
1382 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1383 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1384 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1385 * reserved, which should be set to 0.
1387 if (!ecap_dit(info->iommu->ecap))
1390 struct pci_dev *pf_pdev;
1392 /* pdev will be returned if device is not a vf */
1393 pf_pdev = pci_physfn(pdev);
1394 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1397 #ifdef CONFIG_INTEL_IOMMU_SVM
1398 /* The PCIe spec, in its wisdom, declares that the behaviour of
1399 the device if you enable PASID support after ATS support is
1400 undefined. So always enable PASID support on devices which
1401 have it, even if we can't yet know if we're ever going to
1403 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1404 info->pasid_enabled = 1;
1406 if (info->pri_supported &&
1407 (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
1408 !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1409 info->pri_enabled = 1;
1411 if (!pdev->untrusted && info->ats_supported &&
1412 pci_ats_page_aligned(pdev) &&
1413 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1414 info->ats_enabled = 1;
1415 domain_update_iotlb(info->domain);
1416 info->ats_qdep = pci_ats_queue_depth(pdev);
1420 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1422 struct pci_dev *pdev;
1424 assert_spin_locked(&device_domain_lock);
1426 if (!dev_is_pci(info->dev))
1429 pdev = to_pci_dev(info->dev);
1431 if (info->ats_enabled) {
1432 pci_disable_ats(pdev);
1433 info->ats_enabled = 0;
1434 domain_update_iotlb(info->domain);
1436 #ifdef CONFIG_INTEL_IOMMU_SVM
1437 if (info->pri_enabled) {
1438 pci_disable_pri(pdev);
1439 info->pri_enabled = 0;
1441 if (info->pasid_enabled) {
1442 pci_disable_pasid(pdev);
1443 info->pasid_enabled = 0;
1448 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1449 u64 addr, unsigned mask)
1452 unsigned long flags;
1453 struct device_domain_info *info;
1455 if (!domain->has_iotlb_device)
1458 spin_lock_irqsave(&device_domain_lock, flags);
1459 list_for_each_entry(info, &domain->devices, link) {
1460 if (!info->ats_enabled)
1463 sid = info->bus << 8 | info->devfn;
1464 qdep = info->ats_qdep;
1465 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1468 spin_unlock_irqrestore(&device_domain_lock, flags);
1471 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1472 struct dmar_domain *domain,
1473 unsigned long pfn, unsigned int pages,
1476 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1477 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1478 u16 did = domain->iommu_did[iommu->seq_id];
1485 * Fallback to domain selective flush if no PSI support or the size is
1487 * PSI requires page size to be 2 ^ x, and the base address is naturally
1488 * aligned to the size
1490 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1491 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1494 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1498 * In caching mode, changes of pages from non-present to present require
1499 * flush. However, device IOTLB doesn't need to be flushed in this case.
1501 if (!cap_caching_mode(iommu->cap) || !map)
1502 iommu_flush_dev_iotlb(domain, addr, mask);
1505 /* Notification for newly created mappings */
1506 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1507 struct dmar_domain *domain,
1508 unsigned long pfn, unsigned int pages)
1510 /* It's a non-present to present mapping. Only flush if caching mode */
1511 if (cap_caching_mode(iommu->cap))
1512 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1514 iommu_flush_write_buffer(iommu);
1517 static void iommu_flush_iova(struct iova_domain *iovad)
1519 struct dmar_domain *domain;
1522 domain = container_of(iovad, struct dmar_domain, iovad);
1524 for_each_domain_iommu(idx, domain) {
1525 struct intel_iommu *iommu = g_iommus[idx];
1526 u16 did = domain->iommu_did[iommu->seq_id];
1528 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1530 if (!cap_caching_mode(iommu->cap))
1531 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1532 0, MAX_AGAW_PFN_WIDTH);
1536 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1539 unsigned long flags;
1541 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1544 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1545 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1546 pmen &= ~DMA_PMEN_EPM;
1547 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1549 /* wait for the protected region status bit to clear */
1550 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1551 readl, !(pmen & DMA_PMEN_PRS), pmen);
1553 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1556 static void iommu_enable_translation(struct intel_iommu *iommu)
1559 unsigned long flags;
1561 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1562 iommu->gcmd |= DMA_GCMD_TE;
1563 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1565 /* Make sure hardware complete it */
1566 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1567 readl, (sts & DMA_GSTS_TES), sts);
1569 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1572 static void iommu_disable_translation(struct intel_iommu *iommu)
1577 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1578 iommu->gcmd &= ~DMA_GCMD_TE;
1579 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1581 /* Make sure hardware complete it */
1582 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1583 readl, (!(sts & DMA_GSTS_TES)), sts);
1585 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1589 static int iommu_init_domains(struct intel_iommu *iommu)
1591 u32 ndomains, nlongs;
1594 ndomains = cap_ndoms(iommu->cap);
1595 pr_debug("%s: Number of Domains supported <%d>\n",
1596 iommu->name, ndomains);
1597 nlongs = BITS_TO_LONGS(ndomains);
1599 spin_lock_init(&iommu->lock);
1601 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1602 if (!iommu->domain_ids) {
1603 pr_err("%s: Allocating domain id array failed\n",
1608 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1609 iommu->domains = kzalloc(size, GFP_KERNEL);
1611 if (iommu->domains) {
1612 size = 256 * sizeof(struct dmar_domain *);
1613 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1616 if (!iommu->domains || !iommu->domains[0]) {
1617 pr_err("%s: Allocating domain array failed\n",
1619 kfree(iommu->domain_ids);
1620 kfree(iommu->domains);
1621 iommu->domain_ids = NULL;
1622 iommu->domains = NULL;
1629 * If Caching mode is set, then invalid translations are tagged
1630 * with domain-id 0, hence we need to pre-allocate it. We also
1631 * use domain-id 0 as a marker for non-allocated domain-id, so
1632 * make sure it is not used for a real domain.
1634 set_bit(0, iommu->domain_ids);
1637 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1638 * entry for first-level or pass-through translation modes should
1639 * be programmed with a domain id different from those used for
1640 * second-level or nested translation. We reserve a domain id for
1643 if (sm_supported(iommu))
1644 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1649 static void disable_dmar_iommu(struct intel_iommu *iommu)
1651 struct device_domain_info *info, *tmp;
1652 unsigned long flags;
1654 if (!iommu->domains || !iommu->domain_ids)
1658 spin_lock_irqsave(&device_domain_lock, flags);
1659 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1660 struct dmar_domain *domain;
1662 if (info->iommu != iommu)
1665 if (!info->dev || !info->domain)
1668 domain = info->domain;
1670 __dmar_remove_one_dev_info(info);
1672 if (!domain_type_is_vm_or_si(domain)) {
1674 * The domain_exit() function can't be called under
1675 * device_domain_lock, as it takes this lock itself.
1676 * So release the lock here and re-run the loop
1679 spin_unlock_irqrestore(&device_domain_lock, flags);
1680 domain_exit(domain);
1684 spin_unlock_irqrestore(&device_domain_lock, flags);
1686 if (iommu->gcmd & DMA_GCMD_TE)
1687 iommu_disable_translation(iommu);
1690 static void free_dmar_iommu(struct intel_iommu *iommu)
1692 if ((iommu->domains) && (iommu->domain_ids)) {
1693 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1696 for (i = 0; i < elems; i++)
1697 kfree(iommu->domains[i]);
1698 kfree(iommu->domains);
1699 kfree(iommu->domain_ids);
1700 iommu->domains = NULL;
1701 iommu->domain_ids = NULL;
1704 g_iommus[iommu->seq_id] = NULL;
1706 /* free context mapping */
1707 free_context_table(iommu);
1709 #ifdef CONFIG_INTEL_IOMMU_SVM
1710 if (pasid_supported(iommu)) {
1711 if (ecap_prs(iommu->ecap))
1712 intel_svm_finish_prq(iommu);
1717 static struct dmar_domain *alloc_domain(int flags)
1719 struct dmar_domain *domain;
1721 domain = alloc_domain_mem();
1725 memset(domain, 0, sizeof(*domain));
1726 domain->nid = NUMA_NO_NODE;
1727 domain->flags = flags;
1728 domain->has_iotlb_device = false;
1729 INIT_LIST_HEAD(&domain->devices);
1734 /* Must be called with iommu->lock */
1735 static int domain_attach_iommu(struct dmar_domain *domain,
1736 struct intel_iommu *iommu)
1738 unsigned long ndomains;
1741 assert_spin_locked(&device_domain_lock);
1742 assert_spin_locked(&iommu->lock);
1744 domain->iommu_refcnt[iommu->seq_id] += 1;
1745 domain->iommu_count += 1;
1746 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1747 ndomains = cap_ndoms(iommu->cap);
1748 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1750 if (num >= ndomains) {
1751 pr_err("%s: No free domain ids\n", iommu->name);
1752 domain->iommu_refcnt[iommu->seq_id] -= 1;
1753 domain->iommu_count -= 1;
1757 set_bit(num, iommu->domain_ids);
1758 set_iommu_domain(iommu, num, domain);
1760 domain->iommu_did[iommu->seq_id] = num;
1761 domain->nid = iommu->node;
1763 domain_update_iommu_cap(domain);
1769 static int domain_detach_iommu(struct dmar_domain *domain,
1770 struct intel_iommu *iommu)
1774 assert_spin_locked(&device_domain_lock);
1775 assert_spin_locked(&iommu->lock);
1777 domain->iommu_refcnt[iommu->seq_id] -= 1;
1778 count = --domain->iommu_count;
1779 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1780 num = domain->iommu_did[iommu->seq_id];
1781 clear_bit(num, iommu->domain_ids);
1782 set_iommu_domain(iommu, num, NULL);
1784 domain_update_iommu_cap(domain);
1785 domain->iommu_did[iommu->seq_id] = 0;
1791 static struct iova_domain reserved_iova_list;
1792 static struct lock_class_key reserved_rbtree_key;
1794 static int dmar_init_reserved_ranges(void)
1796 struct pci_dev *pdev = NULL;
1800 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1802 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1803 &reserved_rbtree_key);
1805 /* IOAPIC ranges shouldn't be accessed by DMA */
1806 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1807 IOVA_PFN(IOAPIC_RANGE_END));
1809 pr_err("Reserve IOAPIC range failed\n");
1813 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1814 for_each_pci_dev(pdev) {
1817 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1818 r = &pdev->resource[i];
1819 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1821 iova = reserve_iova(&reserved_iova_list,
1825 pci_err(pdev, "Reserve iova for %pR failed\n", r);
1833 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1835 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1838 static inline int guestwidth_to_adjustwidth(int gaw)
1841 int r = (gaw - 12) % 9;
1852 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1855 int adjust_width, agaw;
1856 unsigned long sagaw;
1859 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1861 err = init_iova_flush_queue(&domain->iovad,
1862 iommu_flush_iova, iova_entry_free);
1866 domain_reserve_special_ranges(domain);
1868 /* calculate AGAW */
1869 if (guest_width > cap_mgaw(iommu->cap))
1870 guest_width = cap_mgaw(iommu->cap);
1871 domain->gaw = guest_width;
1872 adjust_width = guestwidth_to_adjustwidth(guest_width);
1873 agaw = width_to_agaw(adjust_width);
1874 sagaw = cap_sagaw(iommu->cap);
1875 if (!test_bit(agaw, &sagaw)) {
1876 /* hardware doesn't support it, choose a bigger one */
1877 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1878 agaw = find_next_bit(&sagaw, 5, agaw);
1882 domain->agaw = agaw;
1884 if (ecap_coherent(iommu->ecap))
1885 domain->iommu_coherency = 1;
1887 domain->iommu_coherency = 0;
1889 if (ecap_sc_support(iommu->ecap))
1890 domain->iommu_snooping = 1;
1892 domain->iommu_snooping = 0;
1894 if (intel_iommu_superpage)
1895 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1897 domain->iommu_superpage = 0;
1899 domain->nid = iommu->node;
1901 /* always allocate the top pgd */
1902 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1905 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1909 static void domain_exit(struct dmar_domain *domain)
1911 struct page *freelist;
1913 /* Remove associated devices and clear attached or cached domains */
1915 domain_remove_dev_info(domain);
1919 put_iova_domain(&domain->iovad);
1921 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1923 dma_free_pagelist(freelist);
1925 free_domain_mem(domain);
1929 * Get the PASID directory size for scalable mode context entry.
1930 * Value of X in the PDTS field of a scalable mode context entry
1931 * indicates PASID directory with 2^(X + 7) entries.
1933 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1937 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1938 pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1946 * Set the RID_PASID field of a scalable mode context entry. The
1947 * IOMMU hardware will use the PASID value set in this field for
1948 * DMA translations of DMA requests without PASID.
1951 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1953 context->hi |= pasid & ((1 << 20) - 1);
1954 context->hi |= (1 << 20);
1958 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1961 static inline void context_set_sm_dte(struct context_entry *context)
1963 context->lo |= (1 << 2);
1967 * Set the PRE(Page Request Enable) field of a scalable mode context
1970 static inline void context_set_sm_pre(struct context_entry *context)
1972 context->lo |= (1 << 4);
1975 /* Convert value to context PASID directory size field coding. */
1976 #define context_pdts(pds) (((pds) & 0x7) << 9)
1978 static int domain_context_mapping_one(struct dmar_domain *domain,
1979 struct intel_iommu *iommu,
1980 struct pasid_table *table,
1983 u16 did = domain->iommu_did[iommu->seq_id];
1984 int translation = CONTEXT_TT_MULTI_LEVEL;
1985 struct device_domain_info *info = NULL;
1986 struct context_entry *context;
1987 unsigned long flags;
1992 if (hw_pass_through && domain_type_is_si(domain))
1993 translation = CONTEXT_TT_PASS_THROUGH;
1995 pr_debug("Set context mapping for %02x:%02x.%d\n",
1996 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1998 BUG_ON(!domain->pgd);
2000 spin_lock_irqsave(&device_domain_lock, flags);
2001 spin_lock(&iommu->lock);
2004 context = iommu_context_addr(iommu, bus, devfn, 1);
2009 if (context_present(context))
2013 * For kdump cases, old valid entries may be cached due to the
2014 * in-flight DMA and copied pgtable, but there is no unmapping
2015 * behaviour for them, thus we need an explicit cache flush for
2016 * the newly-mapped device. For kdump, at this point, the device
2017 * is supposed to finish reset at its driver probe stage, so no
2018 * in-flight DMA will exist, and we don't need to worry anymore
2021 if (context_copied(context)) {
2022 u16 did_old = context_domain_id(context);
2024 if (did_old < cap_ndoms(iommu->cap)) {
2025 iommu->flush.flush_context(iommu, did_old,
2026 (((u16)bus) << 8) | devfn,
2027 DMA_CCMD_MASK_NOBIT,
2028 DMA_CCMD_DEVICE_INVL);
2029 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2034 context_clear_entry(context);
2036 if (sm_supported(iommu)) {
2041 /* Setup the PASID DIR pointer: */
2042 pds = context_get_sm_pds(table);
2043 context->lo = (u64)virt_to_phys(table->table) |
2046 /* Setup the RID_PASID field: */
2047 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2050 * Setup the Device-TLB enable bit and Page request
2053 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2054 if (info && info->ats_supported)
2055 context_set_sm_dte(context);
2056 if (info && info->pri_supported)
2057 context_set_sm_pre(context);
2059 struct dma_pte *pgd = domain->pgd;
2062 context_set_domain_id(context, did);
2064 if (translation != CONTEXT_TT_PASS_THROUGH) {
2066 * Skip top levels of page tables for iommu which has
2067 * less agaw than default. Unnecessary for PT mode.
2069 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2071 pgd = phys_to_virt(dma_pte_addr(pgd));
2072 if (!dma_pte_present(pgd))
2076 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2077 if (info && info->ats_supported)
2078 translation = CONTEXT_TT_DEV_IOTLB;
2080 translation = CONTEXT_TT_MULTI_LEVEL;
2082 context_set_address_root(context, virt_to_phys(pgd));
2083 context_set_address_width(context, agaw);
2086 * In pass through mode, AW must be programmed to
2087 * indicate the largest AGAW value supported by
2088 * hardware. And ASR is ignored by hardware.
2090 context_set_address_width(context, iommu->msagaw);
2093 context_set_translation_type(context, translation);
2096 context_set_fault_enable(context);
2097 context_set_present(context);
2098 domain_flush_cache(domain, context, sizeof(*context));
2101 * It's a non-present to present mapping. If hardware doesn't cache
2102 * non-present entry we only need to flush the write-buffer. If the
2103 * _does_ cache non-present entries, then it does so in the special
2104 * domain #0, which we have to flush:
2106 if (cap_caching_mode(iommu->cap)) {
2107 iommu->flush.flush_context(iommu, 0,
2108 (((u16)bus) << 8) | devfn,
2109 DMA_CCMD_MASK_NOBIT,
2110 DMA_CCMD_DEVICE_INVL);
2111 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2113 iommu_flush_write_buffer(iommu);
2115 iommu_enable_dev_iotlb(info);
2120 spin_unlock(&iommu->lock);
2121 spin_unlock_irqrestore(&device_domain_lock, flags);
2126 struct domain_context_mapping_data {
2127 struct dmar_domain *domain;
2128 struct intel_iommu *iommu;
2129 struct pasid_table *table;
2132 static int domain_context_mapping_cb(struct pci_dev *pdev,
2133 u16 alias, void *opaque)
2135 struct domain_context_mapping_data *data = opaque;
2137 return domain_context_mapping_one(data->domain, data->iommu,
2138 data->table, PCI_BUS_NUM(alias),
2143 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2145 struct domain_context_mapping_data data;
2146 struct pasid_table *table;
2147 struct intel_iommu *iommu;
2150 iommu = device_to_iommu(dev, &bus, &devfn);
2154 table = intel_pasid_get_table(dev);
2156 if (!dev_is_pci(dev))
2157 return domain_context_mapping_one(domain, iommu, table,
2160 data.domain = domain;
2164 return pci_for_each_dma_alias(to_pci_dev(dev),
2165 &domain_context_mapping_cb, &data);
2168 static int domain_context_mapped_cb(struct pci_dev *pdev,
2169 u16 alias, void *opaque)
2171 struct intel_iommu *iommu = opaque;
2173 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2176 static int domain_context_mapped(struct device *dev)
2178 struct intel_iommu *iommu;
2181 iommu = device_to_iommu(dev, &bus, &devfn);
2185 if (!dev_is_pci(dev))
2186 return device_context_mapped(iommu, bus, devfn);
2188 return !pci_for_each_dma_alias(to_pci_dev(dev),
2189 domain_context_mapped_cb, iommu);
2192 /* Returns a number of VTD pages, but aligned to MM page size */
2193 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2196 host_addr &= ~PAGE_MASK;
2197 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2200 /* Return largest possible superpage level for a given mapping */
2201 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2202 unsigned long iov_pfn,
2203 unsigned long phy_pfn,
2204 unsigned long pages)
2206 int support, level = 1;
2207 unsigned long pfnmerge;
2209 support = domain->iommu_superpage;
2211 /* To use a large page, the virtual *and* physical addresses
2212 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2213 of them will mean we have to use smaller pages. So just
2214 merge them and check both at once. */
2215 pfnmerge = iov_pfn | phy_pfn;
2217 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2218 pages >>= VTD_STRIDE_SHIFT;
2221 pfnmerge >>= VTD_STRIDE_SHIFT;
2228 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2229 struct scatterlist *sg, unsigned long phys_pfn,
2230 unsigned long nr_pages, int prot)
2232 struct dma_pte *first_pte = NULL, *pte = NULL;
2233 phys_addr_t uninitialized_var(pteval);
2234 unsigned long sg_res = 0;
2235 unsigned int largepage_lvl = 0;
2236 unsigned long lvl_pages = 0;
2238 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2240 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2243 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2247 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2250 while (nr_pages > 0) {
2254 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2256 sg_res = aligned_nrpages(sg->offset, sg->length);
2257 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2258 sg->dma_length = sg->length;
2259 pteval = (sg_phys(sg) - pgoff) | prot;
2260 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2264 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2266 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2269 /* It is large page*/
2270 if (largepage_lvl > 1) {
2271 unsigned long nr_superpages, end_pfn;
2273 pteval |= DMA_PTE_LARGE_PAGE;
2274 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2276 nr_superpages = sg_res / lvl_pages;
2277 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2280 * Ensure that old small page tables are
2281 * removed to make room for superpage(s).
2282 * We're adding new large pages, so make sure
2283 * we don't remove their parent tables.
2285 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2288 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2292 /* We don't need lock here, nobody else
2293 * touches the iova range
2295 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2297 static int dumps = 5;
2298 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2299 iov_pfn, tmp, (unsigned long long)pteval);
2302 debug_dma_dump_mappings(NULL);
2307 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2309 BUG_ON(nr_pages < lvl_pages);
2310 BUG_ON(sg_res < lvl_pages);
2312 nr_pages -= lvl_pages;
2313 iov_pfn += lvl_pages;
2314 phys_pfn += lvl_pages;
2315 pteval += lvl_pages * VTD_PAGE_SIZE;
2316 sg_res -= lvl_pages;
2318 /* If the next PTE would be the first in a new page, then we
2319 need to flush the cache on the entries we've just written.
2320 And then we'll need to recalculate 'pte', so clear it and
2321 let it get set again in the if (!pte) block above.
2323 If we're done (!nr_pages) we need to flush the cache too.
2325 Also if we've been setting superpages, we may need to
2326 recalculate 'pte' and switch back to smaller pages for the
2327 end of the mapping, if the trailing size is not enough to
2328 use another superpage (i.e. sg_res < lvl_pages). */
2330 if (!nr_pages || first_pte_in_page(pte) ||
2331 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2332 domain_flush_cache(domain, first_pte,
2333 (void *)pte - (void *)first_pte);
2337 if (!sg_res && nr_pages)
2343 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2344 struct scatterlist *sg, unsigned long phys_pfn,
2345 unsigned long nr_pages, int prot)
2348 struct intel_iommu *iommu;
2350 /* Do the real mapping first */
2351 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2355 /* Notify about the new mapping */
2356 if (domain_type_is_vm(domain)) {
2357 /* VM typed domains can have more than one IOMMUs */
2359 for_each_domain_iommu(iommu_id, domain) {
2360 iommu = g_iommus[iommu_id];
2361 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2364 /* General domains only have one IOMMU */
2365 iommu = domain_get_iommu(domain);
2366 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2372 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2373 struct scatterlist *sg, unsigned long nr_pages,
2376 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2379 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2380 unsigned long phys_pfn, unsigned long nr_pages,
2383 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2386 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2388 unsigned long flags;
2389 struct context_entry *context;
2395 spin_lock_irqsave(&iommu->lock, flags);
2396 context = iommu_context_addr(iommu, bus, devfn, 0);
2398 spin_unlock_irqrestore(&iommu->lock, flags);
2401 did_old = context_domain_id(context);
2402 context_clear_entry(context);
2403 __iommu_flush_cache(iommu, context, sizeof(*context));
2404 spin_unlock_irqrestore(&iommu->lock, flags);
2405 iommu->flush.flush_context(iommu,
2407 (((u16)bus) << 8) | devfn,
2408 DMA_CCMD_MASK_NOBIT,
2409 DMA_CCMD_DEVICE_INVL);
2410 iommu->flush.flush_iotlb(iommu,
2417 static inline void unlink_domain_info(struct device_domain_info *info)
2419 assert_spin_locked(&device_domain_lock);
2420 list_del(&info->link);
2421 list_del(&info->global);
2423 info->dev->archdata.iommu = NULL;
2426 static void domain_remove_dev_info(struct dmar_domain *domain)
2428 struct device_domain_info *info, *tmp;
2429 unsigned long flags;
2431 spin_lock_irqsave(&device_domain_lock, flags);
2432 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2433 __dmar_remove_one_dev_info(info);
2434 spin_unlock_irqrestore(&device_domain_lock, flags);
2439 * Note: we use struct device->archdata.iommu stores the info
2441 static struct dmar_domain *find_domain(struct device *dev)
2443 struct device_domain_info *info;
2445 /* No lock here, assumes no domain exit in normal case */
2446 info = dev->archdata.iommu;
2448 return info->domain;
2452 static inline struct device_domain_info *
2453 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2455 struct device_domain_info *info;
2457 list_for_each_entry(info, &device_domain_list, global)
2458 if (info->iommu->segment == segment && info->bus == bus &&
2459 info->devfn == devfn)
2465 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2468 struct dmar_domain *domain)
2470 struct dmar_domain *found = NULL;
2471 struct device_domain_info *info;
2472 unsigned long flags;
2475 info = alloc_devinfo_mem();
2480 info->devfn = devfn;
2481 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2482 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2485 info->domain = domain;
2486 info->iommu = iommu;
2487 info->pasid_table = NULL;
2489 if (dev && dev_is_pci(dev)) {
2490 struct pci_dev *pdev = to_pci_dev(info->dev);
2492 if (!pdev->untrusted &&
2493 !pci_ats_disabled() &&
2494 ecap_dev_iotlb_support(iommu->ecap) &&
2495 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2496 dmar_find_matched_atsr_unit(pdev))
2497 info->ats_supported = 1;
2499 if (sm_supported(iommu)) {
2500 if (pasid_supported(iommu)) {
2501 int features = pci_pasid_features(pdev);
2503 info->pasid_supported = features | 1;
2506 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2507 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2508 info->pri_supported = 1;
2512 spin_lock_irqsave(&device_domain_lock, flags);
2514 found = find_domain(dev);
2517 struct device_domain_info *info2;
2518 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2520 found = info2->domain;
2526 spin_unlock_irqrestore(&device_domain_lock, flags);
2527 free_devinfo_mem(info);
2528 /* Caller must free the original domain */
2532 spin_lock(&iommu->lock);
2533 ret = domain_attach_iommu(domain, iommu);
2534 spin_unlock(&iommu->lock);
2537 spin_unlock_irqrestore(&device_domain_lock, flags);
2538 free_devinfo_mem(info);
2542 list_add(&info->link, &domain->devices);
2543 list_add(&info->global, &device_domain_list);
2545 dev->archdata.iommu = info;
2546 spin_unlock_irqrestore(&device_domain_lock, flags);
2548 /* PASID table is mandatory for a PCI device in scalable mode. */
2549 if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2550 ret = intel_pasid_alloc_table(dev);
2552 dev_err(dev, "PASID table allocation failed\n");
2553 dmar_remove_one_dev_info(dev);
2557 /* Setup the PASID entry for requests without PASID: */
2558 spin_lock(&iommu->lock);
2559 if (hw_pass_through && domain_type_is_si(domain))
2560 ret = intel_pasid_setup_pass_through(iommu, domain,
2561 dev, PASID_RID2PASID);
2563 ret = intel_pasid_setup_second_level(iommu, domain,
2564 dev, PASID_RID2PASID);
2565 spin_unlock(&iommu->lock);
2567 dev_err(dev, "Setup RID2PASID failed\n");
2568 dmar_remove_one_dev_info(dev);
2573 if (dev && domain_context_mapping(domain, dev)) {
2574 dev_err(dev, "Domain context map failed\n");
2575 dmar_remove_one_dev_info(dev);
2582 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2584 *(u16 *)opaque = alias;
2588 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2590 struct device_domain_info *info;
2591 struct dmar_domain *domain = NULL;
2592 struct intel_iommu *iommu;
2594 unsigned long flags;
2597 iommu = device_to_iommu(dev, &bus, &devfn);
2601 if (dev_is_pci(dev)) {
2602 struct pci_dev *pdev = to_pci_dev(dev);
2604 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2606 spin_lock_irqsave(&device_domain_lock, flags);
2607 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2608 PCI_BUS_NUM(dma_alias),
2611 iommu = info->iommu;
2612 domain = info->domain;
2614 spin_unlock_irqrestore(&device_domain_lock, flags);
2616 /* DMA alias already has a domain, use it */
2621 /* Allocate and initialize new domain for the device */
2622 domain = alloc_domain(0);
2625 if (domain_init(domain, iommu, gaw)) {
2626 domain_exit(domain);
2635 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2636 struct dmar_domain *domain)
2638 struct intel_iommu *iommu;
2639 struct dmar_domain *tmp;
2640 u16 req_id, dma_alias;
2643 iommu = device_to_iommu(dev, &bus, &devfn);
2647 req_id = ((u16)bus << 8) | devfn;
2649 if (dev_is_pci(dev)) {
2650 struct pci_dev *pdev = to_pci_dev(dev);
2652 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2654 /* register PCI DMA alias device */
2655 if (req_id != dma_alias) {
2656 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2657 dma_alias & 0xff, NULL, domain);
2659 if (!tmp || tmp != domain)
2664 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2665 if (!tmp || tmp != domain)
2671 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2673 struct dmar_domain *domain, *tmp;
2675 domain = find_domain(dev);
2679 domain = find_or_alloc_domain(dev, gaw);
2683 tmp = set_domain_for_dev(dev, domain);
2684 if (!tmp || domain != tmp) {
2685 domain_exit(domain);
2694 static int iommu_domain_identity_map(struct dmar_domain *domain,
2695 unsigned long long start,
2696 unsigned long long end)
2698 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2699 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2701 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2702 dma_to_mm_pfn(last_vpfn))) {
2703 pr_err("Reserving iova failed\n");
2707 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2709 * RMRR range might have overlap with physical memory range,
2712 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2714 return __domain_mapping(domain, first_vpfn, NULL,
2715 first_vpfn, last_vpfn - first_vpfn + 1,
2716 DMA_PTE_READ|DMA_PTE_WRITE);
2719 static int domain_prepare_identity_map(struct device *dev,
2720 struct dmar_domain *domain,
2721 unsigned long long start,
2722 unsigned long long end)
2724 /* For _hardware_ passthrough, don't bother. But for software
2725 passthrough, we do it anyway -- it may indicate a memory
2726 range which is reserved in E820, so which didn't get set
2727 up to start with in si_domain */
2728 if (domain == si_domain && hw_pass_through) {
2729 dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
2734 dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
2737 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2738 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2739 dmi_get_system_info(DMI_BIOS_VENDOR),
2740 dmi_get_system_info(DMI_BIOS_VERSION),
2741 dmi_get_system_info(DMI_PRODUCT_VERSION));
2745 if (end >> agaw_to_width(domain->agaw)) {
2746 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2747 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2748 agaw_to_width(domain->agaw),
2749 dmi_get_system_info(DMI_BIOS_VENDOR),
2750 dmi_get_system_info(DMI_BIOS_VERSION),
2751 dmi_get_system_info(DMI_PRODUCT_VERSION));
2755 return iommu_domain_identity_map(domain, start, end);
2758 static int iommu_prepare_identity_map(struct device *dev,
2759 unsigned long long start,
2760 unsigned long long end)
2762 struct dmar_domain *domain;
2765 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2769 ret = domain_prepare_identity_map(dev, domain, start, end);
2771 domain_exit(domain);
2776 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2779 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2781 return iommu_prepare_identity_map(dev, rmrr->base_address,
2785 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2786 static inline void iommu_prepare_isa(void)
2788 struct pci_dev *pdev;
2791 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2795 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2796 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2799 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2804 static inline void iommu_prepare_isa(void)
2808 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2810 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2812 static int __init si_domain_init(int hw)
2816 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2820 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2821 domain_exit(si_domain);
2825 pr_debug("Identity mapping domain allocated\n");
2830 for_each_online_node(nid) {
2831 unsigned long start_pfn, end_pfn;
2834 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2835 ret = iommu_domain_identity_map(si_domain,
2836 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2845 static int identity_mapping(struct device *dev)
2847 struct device_domain_info *info;
2849 if (likely(!iommu_identity_mapping))
2852 info = dev->archdata.iommu;
2853 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2854 return (info->domain == si_domain);
2859 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2861 struct dmar_domain *ndomain;
2862 struct intel_iommu *iommu;
2865 iommu = device_to_iommu(dev, &bus, &devfn);
2869 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2870 if (ndomain != domain)
2876 static bool device_has_rmrr(struct device *dev)
2878 struct dmar_rmrr_unit *rmrr;
2883 for_each_rmrr_units(rmrr) {
2885 * Return TRUE if this RMRR contains the device that
2888 for_each_active_dev_scope(rmrr->devices,
2889 rmrr->devices_cnt, i, tmp)
2900 * There are a couple cases where we need to restrict the functionality of
2901 * devices associated with RMRRs. The first is when evaluating a device for
2902 * identity mapping because problems exist when devices are moved in and out
2903 * of domains and their respective RMRR information is lost. This means that
2904 * a device with associated RMRRs will never be in a "passthrough" domain.
2905 * The second is use of the device through the IOMMU API. This interface
2906 * expects to have full control of the IOVA space for the device. We cannot
2907 * satisfy both the requirement that RMRR access is maintained and have an
2908 * unencumbered IOVA space. We also have no ability to quiesce the device's
2909 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2910 * We therefore prevent devices associated with an RMRR from participating in
2911 * the IOMMU API, which eliminates them from device assignment.
2913 * In both cases we assume that PCI USB devices with RMRRs have them largely
2914 * for historical reasons and that the RMRR space is not actively used post
2915 * boot. This exclusion may change if vendors begin to abuse it.
2917 * The same exception is made for graphics devices, with the requirement that
2918 * any use of the RMRR regions will be torn down before assigning the device
2921 static bool device_is_rmrr_locked(struct device *dev)
2923 if (!device_has_rmrr(dev))
2926 if (dev_is_pci(dev)) {
2927 struct pci_dev *pdev = to_pci_dev(dev);
2929 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2936 static int iommu_should_identity_map(struct device *dev, int startup)
2938 if (dev_is_pci(dev)) {
2939 struct pci_dev *pdev = to_pci_dev(dev);
2941 if (device_is_rmrr_locked(dev))
2945 * Prevent any device marked as untrusted from getting
2946 * placed into the statically identity mapping domain.
2948 if (pdev->untrusted)
2951 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2954 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2957 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2961 * We want to start off with all devices in the 1:1 domain, and
2962 * take them out later if we find they can't access all of memory.
2964 * However, we can't do this for PCI devices behind bridges,
2965 * because all PCI devices behind the same bridge will end up
2966 * with the same source-id on their transactions.
2968 * Practically speaking, we can't change things around for these
2969 * devices at run-time, because we can't be sure there'll be no
2970 * DMA transactions in flight for any of their siblings.
2972 * So PCI devices (unless they're on the root bus) as well as
2973 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2974 * the 1:1 domain, just in _case_ one of their siblings turns out
2975 * not to be able to map all of memory.
2977 if (!pci_is_pcie(pdev)) {
2978 if (!pci_is_root_bus(pdev->bus))
2980 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2982 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2985 if (device_has_rmrr(dev))
2990 * At boot time, we don't yet know if devices will be 64-bit capable.
2991 * Assume that they will — if they turn out not to be, then we can
2992 * take them out of the 1:1 domain later.
2996 * If the device's dma_mask is less than the system's memory
2997 * size then this is not a candidate for identity mapping.
2999 u64 dma_mask = *dev->dma_mask;
3001 if (dev->coherent_dma_mask &&
3002 dev->coherent_dma_mask < dma_mask)
3003 dma_mask = dev->coherent_dma_mask;
3005 return dma_mask >= dma_get_required_mask(dev);
3011 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
3015 if (!iommu_should_identity_map(dev, 1))
3018 ret = domain_add_dev_info(si_domain, dev);
3020 dev_info(dev, "%s identity mapping\n",
3021 hw ? "Hardware" : "Software");
3022 else if (ret == -ENODEV)
3023 /* device not associated with an iommu */
3030 static int __init iommu_prepare_static_identity_mapping(int hw)
3032 struct pci_dev *pdev = NULL;
3033 struct dmar_drhd_unit *drhd;
3034 struct intel_iommu *iommu;
3039 for_each_pci_dev(pdev) {
3040 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3045 for_each_active_iommu(iommu, drhd)
3046 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3047 struct acpi_device_physical_node *pn;
3048 struct acpi_device *adev;
3050 if (dev->bus != &acpi_bus_type)
3053 adev= to_acpi_device(dev);
3054 mutex_lock(&adev->physical_node_lock);
3055 list_for_each_entry(pn, &adev->physical_node_list, node) {
3056 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3060 mutex_unlock(&adev->physical_node_lock);
3068 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3071 * Start from the sane iommu hardware state.
3072 * If the queued invalidation is already initialized by us
3073 * (for example, while enabling interrupt-remapping) then
3074 * we got the things already rolling from a sane state.
3078 * Clear any previous faults.
3080 dmar_fault(-1, iommu);
3082 * Disable queued invalidation if supported and already enabled
3083 * before OS handover.
3085 dmar_disable_qi(iommu);
3088 if (dmar_enable_qi(iommu)) {
3090 * Queued Invalidate not enabled, use Register Based Invalidate
3092 iommu->flush.flush_context = __iommu_flush_context;
3093 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3094 pr_info("%s: Using Register based invalidation\n",
3097 iommu->flush.flush_context = qi_flush_context;
3098 iommu->flush.flush_iotlb = qi_flush_iotlb;
3099 pr_info("%s: Using Queued invalidation\n", iommu->name);
3103 static int copy_context_table(struct intel_iommu *iommu,
3104 struct root_entry *old_re,
3105 struct context_entry **tbl,
3108 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3109 struct context_entry *new_ce = NULL, ce;
3110 struct context_entry *old_ce = NULL;
3111 struct root_entry re;
3112 phys_addr_t old_ce_phys;
3114 tbl_idx = ext ? bus * 2 : bus;
3115 memcpy(&re, old_re, sizeof(re));
3117 for (devfn = 0; devfn < 256; devfn++) {
3118 /* First calculate the correct index */
3119 idx = (ext ? devfn * 2 : devfn) % 256;
3122 /* First save what we may have and clean up */
3124 tbl[tbl_idx] = new_ce;
3125 __iommu_flush_cache(iommu, new_ce,
3135 old_ce_phys = root_entry_lctp(&re);
3137 old_ce_phys = root_entry_uctp(&re);
3140 if (ext && devfn == 0) {
3141 /* No LCTP, try UCTP */
3150 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3155 new_ce = alloc_pgtable_page(iommu->node);
3162 /* Now copy the context entry */
3163 memcpy(&ce, old_ce + idx, sizeof(ce));
3165 if (!__context_present(&ce))
3168 did = context_domain_id(&ce);
3169 if (did >= 0 && did < cap_ndoms(iommu->cap))
3170 set_bit(did, iommu->domain_ids);
3173 * We need a marker for copied context entries. This
3174 * marker needs to work for the old format as well as
3175 * for extended context entries.
3177 * Bit 67 of the context entry is used. In the old
3178 * format this bit is available to software, in the
3179 * extended format it is the PGE bit, but PGE is ignored
3180 * by HW if PASIDs are disabled (and thus still
3183 * So disable PASIDs first and then mark the entry
3184 * copied. This means that we don't copy PASID
3185 * translations from the old kernel, but this is fine as
3186 * faults there are not fatal.
3188 context_clear_pasid_enable(&ce);
3189 context_set_copied(&ce);
3194 tbl[tbl_idx + pos] = new_ce;
3196 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3205 static int copy_translation_tables(struct intel_iommu *iommu)
3207 struct context_entry **ctxt_tbls;
3208 struct root_entry *old_rt;
3209 phys_addr_t old_rt_phys;
3210 int ctxt_table_entries;
3211 unsigned long flags;
3216 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3217 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3218 new_ext = !!ecap_ecs(iommu->ecap);
3221 * The RTT bit can only be changed when translation is disabled,
3222 * but disabling translation means to open a window for data
3223 * corruption. So bail out and don't copy anything if we would
3224 * have to change the bit.
3229 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3233 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3237 /* This is too big for the stack - allocate it from slab */
3238 ctxt_table_entries = ext ? 512 : 256;
3240 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3244 for (bus = 0; bus < 256; bus++) {
3245 ret = copy_context_table(iommu, &old_rt[bus],
3246 ctxt_tbls, bus, ext);
3248 pr_err("%s: Failed to copy context table for bus %d\n",
3254 spin_lock_irqsave(&iommu->lock, flags);
3256 /* Context tables are copied, now write them to the root_entry table */
3257 for (bus = 0; bus < 256; bus++) {
3258 int idx = ext ? bus * 2 : bus;
3261 if (ctxt_tbls[idx]) {
3262 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3263 iommu->root_entry[bus].lo = val;
3266 if (!ext || !ctxt_tbls[idx + 1])
3269 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3270 iommu->root_entry[bus].hi = val;
3273 spin_unlock_irqrestore(&iommu->lock, flags);
3277 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3287 static int __init init_dmars(void)
3289 struct dmar_drhd_unit *drhd;
3290 struct dmar_rmrr_unit *rmrr;
3291 bool copied_tables = false;
3293 struct intel_iommu *iommu;
3299 * initialize and program root entry to not present
3302 for_each_drhd_unit(drhd) {
3304 * lock not needed as this is only incremented in the single
3305 * threaded kernel __init code path all other access are read
3308 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3312 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3315 /* Preallocate enough resources for IOMMU hot-addition */
3316 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3317 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3319 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3322 pr_err("Allocating global iommu array failed\n");
3327 for_each_active_iommu(iommu, drhd) {
3329 * Find the max pasid size of all IOMMU's in the system.
3330 * We need to ensure the system pasid table is no bigger
3331 * than the smallest supported.
3333 if (pasid_supported(iommu)) {
3334 u32 temp = 2 << ecap_pss(iommu->ecap);
3336 intel_pasid_max_id = min_t(u32, temp,
3337 intel_pasid_max_id);
3340 g_iommus[iommu->seq_id] = iommu;
3342 intel_iommu_init_qi(iommu);
3344 ret = iommu_init_domains(iommu);
3348 init_translation_status(iommu);
3350 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3351 iommu_disable_translation(iommu);
3352 clear_translation_pre_enabled(iommu);
3353 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3359 * we could share the same root & context tables
3360 * among all IOMMU's. Need to Split it later.
3362 ret = iommu_alloc_root_entry(iommu);
3366 if (translation_pre_enabled(iommu)) {
3367 pr_info("Translation already enabled - trying to copy translation structures\n");
3369 ret = copy_translation_tables(iommu);
3372 * We found the IOMMU with translation
3373 * enabled - but failed to copy over the
3374 * old root-entry table. Try to proceed
3375 * by disabling translation now and
3376 * allocating a clean root-entry table.
3377 * This might cause DMAR faults, but
3378 * probably the dump will still succeed.
3380 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3382 iommu_disable_translation(iommu);
3383 clear_translation_pre_enabled(iommu);
3385 pr_info("Copied translation tables from previous kernel for %s\n",
3387 copied_tables = true;
3391 if (!ecap_pass_through(iommu->ecap))
3392 hw_pass_through = 0;
3393 #ifdef CONFIG_INTEL_IOMMU_SVM
3394 if (pasid_supported(iommu))
3395 intel_svm_init(iommu);
3400 * Now that qi is enabled on all iommus, set the root entry and flush
3401 * caches. This is required on some Intel X58 chipsets, otherwise the
3402 * flush_context function will loop forever and the boot hangs.
3404 for_each_active_iommu(iommu, drhd) {
3405 iommu_flush_write_buffer(iommu);
3406 iommu_set_root_entry(iommu);
3407 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3408 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3411 if (iommu_pass_through)
3412 iommu_identity_mapping |= IDENTMAP_ALL;
3414 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3415 iommu_identity_mapping |= IDENTMAP_GFX;
3418 check_tylersburg_isoch();
3420 if (iommu_identity_mapping) {
3421 ret = si_domain_init(hw_pass_through);
3428 * If we copied translations from a previous kernel in the kdump
3429 * case, we can not assign the devices to domains now, as that
3430 * would eliminate the old mappings. So skip this part and defer
3431 * the assignment to device driver initialization time.
3437 * If pass through is not set or not enabled, setup context entries for
3438 * identity mappings for rmrr, gfx, and isa and may fall back to static
3439 * identity mapping if iommu_identity_mapping is set.
3441 if (iommu_identity_mapping) {
3442 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3444 pr_crit("Failed to setup IOMMU pass-through\n");
3450 * for each dev attached to rmrr
3452 * locate drhd for dev, alloc domain for dev
3453 * allocate free domain
3454 * allocate page table entries for rmrr
3455 * if context not allocated for bus
3456 * allocate and init context
3457 * set present in root table for this bus
3458 * init context with domain, translation etc
3462 pr_info("Setting RMRR:\n");
3463 for_each_rmrr_units(rmrr) {
3464 /* some BIOS lists non-exist devices in DMAR table. */
3465 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3467 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3469 pr_err("Mapping reserved region failed\n");
3473 iommu_prepare_isa();
3480 * global invalidate context cache
3481 * global invalidate iotlb
3482 * enable translation
3484 for_each_iommu(iommu, drhd) {
3485 if (drhd->ignored) {
3487 * we always have to disable PMRs or DMA may fail on
3491 iommu_disable_protect_mem_regions(iommu);
3495 iommu_flush_write_buffer(iommu);
3497 #ifdef CONFIG_INTEL_IOMMU_SVM
3498 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3499 ret = intel_svm_enable_prq(iommu);
3504 ret = dmar_set_interrupt(iommu);
3508 if (!translation_pre_enabled(iommu))
3509 iommu_enable_translation(iommu);
3511 iommu_disable_protect_mem_regions(iommu);
3517 for_each_active_iommu(iommu, drhd) {
3518 disable_dmar_iommu(iommu);
3519 free_dmar_iommu(iommu);
3528 /* This takes a number of _MM_ pages, not VTD pages */
3529 static unsigned long intel_alloc_iova(struct device *dev,
3530 struct dmar_domain *domain,
3531 unsigned long nrpages, uint64_t dma_mask)
3533 unsigned long iova_pfn;
3535 /* Restrict dma_mask to the width that the iommu can handle */
3536 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3537 /* Ensure we reserve the whole size-aligned region */
3538 nrpages = __roundup_pow_of_two(nrpages);
3540 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3542 * First try to allocate an io virtual address in
3543 * DMA_BIT_MASK(32) and if that fails then try allocating
3546 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3547 IOVA_PFN(DMA_BIT_MASK(32)), false);
3551 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3552 IOVA_PFN(dma_mask), true);
3553 if (unlikely(!iova_pfn)) {
3554 dev_err(dev, "Allocating %ld-page iova failed", nrpages);
3561 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3563 struct dmar_domain *domain, *tmp;
3564 struct dmar_rmrr_unit *rmrr;
3565 struct device *i_dev;
3568 domain = find_domain(dev);
3572 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3576 /* We have a new domain - setup possible RMRRs for the device */
3578 for_each_rmrr_units(rmrr) {
3579 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3584 ret = domain_prepare_identity_map(dev, domain,
3588 dev_err(dev, "Mapping reserved region failed\n");
3593 tmp = set_domain_for_dev(dev, domain);
3594 if (!tmp || domain != tmp) {
3595 domain_exit(domain);
3602 dev_err(dev, "Allocating domain failed\n");
3608 /* Check if the dev needs to go through non-identity map and unmap process.*/
3609 static int iommu_no_mapping(struct device *dev)
3613 if (iommu_dummy(dev))
3616 if (!iommu_identity_mapping)
3619 found = identity_mapping(dev);
3621 if (iommu_should_identity_map(dev, 0))
3625 * 32 bit DMA is removed from si_domain and fall back
3626 * to non-identity mapping.
3628 dmar_remove_one_dev_info(dev);
3629 dev_info(dev, "32bit DMA uses non-identity mapping\n");
3634 * In case of a detached 64 bit DMA device from vm, the device
3635 * is put into si_domain for identity mapping.
3637 if (iommu_should_identity_map(dev, 0)) {
3639 ret = domain_add_dev_info(si_domain, dev);
3641 dev_info(dev, "64bit DMA uses identity mapping\n");
3650 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3651 size_t size, int dir, u64 dma_mask)
3653 struct dmar_domain *domain;
3654 phys_addr_t start_paddr;
3655 unsigned long iova_pfn;
3658 struct intel_iommu *iommu;
3659 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3661 BUG_ON(dir == DMA_NONE);
3663 if (iommu_no_mapping(dev))
3666 domain = get_valid_domain_for_dev(dev);
3668 return DMA_MAPPING_ERROR;
3670 iommu = domain_get_iommu(domain);
3671 size = aligned_nrpages(paddr, size);
3673 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3678 * Check if DMAR supports zero-length reads on write only
3681 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3682 !cap_zlr(iommu->cap))
3683 prot |= DMA_PTE_READ;
3684 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3685 prot |= DMA_PTE_WRITE;
3687 * paddr - (paddr + size) might be partial page, we should map the whole
3688 * page. Note: if two part of one page are separately mapped, we
3689 * might have two guest_addr mapping to the same host paddr, but this
3690 * is not a big problem
3692 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3693 mm_to_dma_pfn(paddr_pfn), size, prot);
3697 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3698 start_paddr += paddr & ~PAGE_MASK;
3703 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3704 dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
3705 size, (unsigned long long)paddr, dir);
3706 return DMA_MAPPING_ERROR;
3709 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3710 unsigned long offset, size_t size,
3711 enum dma_data_direction dir,
3712 unsigned long attrs)
3714 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3715 dir, *dev->dma_mask);
3718 static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
3719 size_t size, enum dma_data_direction dir,
3720 unsigned long attrs)
3722 return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
3725 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3727 struct dmar_domain *domain;
3728 unsigned long start_pfn, last_pfn;
3729 unsigned long nrpages;
3730 unsigned long iova_pfn;
3731 struct intel_iommu *iommu;
3732 struct page *freelist;
3734 if (iommu_no_mapping(dev))
3737 domain = find_domain(dev);
3740 iommu = domain_get_iommu(domain);
3742 iova_pfn = IOVA_PFN(dev_addr);
3744 nrpages = aligned_nrpages(dev_addr, size);
3745 start_pfn = mm_to_dma_pfn(iova_pfn);
3746 last_pfn = start_pfn + nrpages - 1;
3748 dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn);
3750 freelist = domain_unmap(domain, start_pfn, last_pfn);
3752 if (intel_iommu_strict) {
3753 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3754 nrpages, !freelist, 0);
3756 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3757 dma_free_pagelist(freelist);
3759 queue_iova(&domain->iovad, iova_pfn, nrpages,
3760 (unsigned long)freelist);
3762 * queue up the release of the unmap to save the 1/6th of the
3763 * cpu used up by the iotlb flush operation...
3768 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3769 size_t size, enum dma_data_direction dir,
3770 unsigned long attrs)
3772 intel_unmap(dev, dev_addr, size);
3775 static void *intel_alloc_coherent(struct device *dev, size_t size,
3776 dma_addr_t *dma_handle, gfp_t flags,
3777 unsigned long attrs)
3779 struct page *page = NULL;
3782 size = PAGE_ALIGN(size);
3783 order = get_order(size);
3785 if (!iommu_no_mapping(dev))
3786 flags &= ~(GFP_DMA | GFP_DMA32);
3787 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3788 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3794 if (gfpflags_allow_blocking(flags)) {
3795 unsigned int count = size >> PAGE_SHIFT;
3797 page = dma_alloc_from_contiguous(dev, count, order,
3798 flags & __GFP_NOWARN);
3799 if (page && iommu_no_mapping(dev) &&
3800 page_to_phys(page) + size > dev->coherent_dma_mask) {
3801 dma_release_from_contiguous(dev, page, count);
3807 page = alloc_pages(flags, order);
3810 memset(page_address(page), 0, size);
3812 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3814 dev->coherent_dma_mask);
3815 if (*dma_handle != DMA_MAPPING_ERROR)
3816 return page_address(page);
3817 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3818 __free_pages(page, order);
3823 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3824 dma_addr_t dma_handle, unsigned long attrs)
3827 struct page *page = virt_to_page(vaddr);
3829 size = PAGE_ALIGN(size);
3830 order = get_order(size);
3832 intel_unmap(dev, dma_handle, size);
3833 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3834 __free_pages(page, order);
3837 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3838 int nelems, enum dma_data_direction dir,
3839 unsigned long attrs)
3841 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3842 unsigned long nrpages = 0;
3843 struct scatterlist *sg;
3846 for_each_sg(sglist, sg, nelems, i) {
3847 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3850 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3853 static int intel_nontranslate_map_sg(struct device *hddev,
3854 struct scatterlist *sglist, int nelems, int dir)
3857 struct scatterlist *sg;
3859 for_each_sg(sglist, sg, nelems, i) {
3860 BUG_ON(!sg_page(sg));
3861 sg->dma_address = sg_phys(sg);
3862 sg->dma_length = sg->length;
3867 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3868 enum dma_data_direction dir, unsigned long attrs)
3871 struct dmar_domain *domain;
3874 unsigned long iova_pfn;
3876 struct scatterlist *sg;
3877 unsigned long start_vpfn;
3878 struct intel_iommu *iommu;
3880 BUG_ON(dir == DMA_NONE);
3881 if (iommu_no_mapping(dev))
3882 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3884 domain = get_valid_domain_for_dev(dev);
3888 iommu = domain_get_iommu(domain);
3890 for_each_sg(sglist, sg, nelems, i)
3891 size += aligned_nrpages(sg->offset, sg->length);
3893 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3896 sglist->dma_length = 0;
3901 * Check if DMAR supports zero-length reads on write only
3904 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3905 !cap_zlr(iommu->cap))
3906 prot |= DMA_PTE_READ;
3907 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3908 prot |= DMA_PTE_WRITE;
3910 start_vpfn = mm_to_dma_pfn(iova_pfn);
3912 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3913 if (unlikely(ret)) {
3914 dma_pte_free_pagetable(domain, start_vpfn,
3915 start_vpfn + size - 1,
3916 agaw_to_level(domain->agaw) + 1);
3917 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3924 static const struct dma_map_ops intel_dma_ops = {
3925 .alloc = intel_alloc_coherent,
3926 .free = intel_free_coherent,
3927 .map_sg = intel_map_sg,
3928 .unmap_sg = intel_unmap_sg,
3929 .map_page = intel_map_page,
3930 .unmap_page = intel_unmap_page,
3931 .map_resource = intel_map_resource,
3932 .unmap_resource = intel_unmap_page,
3933 .dma_supported = dma_direct_supported,
3936 static inline int iommu_domain_cache_init(void)
3940 iommu_domain_cache = kmem_cache_create("iommu_domain",
3941 sizeof(struct dmar_domain),
3946 if (!iommu_domain_cache) {
3947 pr_err("Couldn't create iommu_domain cache\n");
3954 static inline int iommu_devinfo_cache_init(void)
3958 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3959 sizeof(struct device_domain_info),
3963 if (!iommu_devinfo_cache) {
3964 pr_err("Couldn't create devinfo cache\n");
3971 static int __init iommu_init_mempool(void)
3974 ret = iova_cache_get();
3978 ret = iommu_domain_cache_init();
3982 ret = iommu_devinfo_cache_init();
3986 kmem_cache_destroy(iommu_domain_cache);
3993 static void __init iommu_exit_mempool(void)
3995 kmem_cache_destroy(iommu_devinfo_cache);
3996 kmem_cache_destroy(iommu_domain_cache);
4000 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4002 struct dmar_drhd_unit *drhd;
4006 /* We know that this device on this chipset has its own IOMMU.
4007 * If we find it under a different IOMMU, then the BIOS is lying
4008 * to us. Hope that the IOMMU for this device is actually
4009 * disabled, and it needs no translation...
4011 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4013 /* "can't" happen */
4014 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4017 vtbar &= 0xffff0000;
4019 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4020 drhd = dmar_find_matched_drhd_unit(pdev);
4021 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4022 TAINT_FIRMWARE_WORKAROUND,
4023 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4024 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4026 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4028 static void __init init_no_remapping_devices(void)
4030 struct dmar_drhd_unit *drhd;
4034 for_each_drhd_unit(drhd) {
4035 if (!drhd->include_all) {
4036 for_each_active_dev_scope(drhd->devices,
4037 drhd->devices_cnt, i, dev)
4039 /* ignore DMAR unit if no devices exist */
4040 if (i == drhd->devices_cnt)
4045 for_each_active_drhd_unit(drhd) {
4046 if (drhd->include_all)
4049 for_each_active_dev_scope(drhd->devices,
4050 drhd->devices_cnt, i, dev)
4051 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4053 if (i < drhd->devices_cnt)
4056 /* This IOMMU has *only* gfx devices. Either bypass it or
4057 set the gfx_mapped flag, as appropriate */
4059 intel_iommu_gfx_mapped = 1;
4062 for_each_active_dev_scope(drhd->devices,
4063 drhd->devices_cnt, i, dev)
4064 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4069 #ifdef CONFIG_SUSPEND
4070 static int init_iommu_hw(void)
4072 struct dmar_drhd_unit *drhd;
4073 struct intel_iommu *iommu = NULL;
4075 for_each_active_iommu(iommu, drhd)
4077 dmar_reenable_qi(iommu);
4079 for_each_iommu(iommu, drhd) {
4080 if (drhd->ignored) {
4082 * we always have to disable PMRs or DMA may fail on
4086 iommu_disable_protect_mem_regions(iommu);
4090 iommu_flush_write_buffer(iommu);
4092 iommu_set_root_entry(iommu);
4094 iommu->flush.flush_context(iommu, 0, 0, 0,
4095 DMA_CCMD_GLOBAL_INVL);
4096 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4097 iommu_enable_translation(iommu);
4098 iommu_disable_protect_mem_regions(iommu);
4104 static void iommu_flush_all(void)
4106 struct dmar_drhd_unit *drhd;
4107 struct intel_iommu *iommu;
4109 for_each_active_iommu(iommu, drhd) {
4110 iommu->flush.flush_context(iommu, 0, 0, 0,
4111 DMA_CCMD_GLOBAL_INVL);
4112 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4113 DMA_TLB_GLOBAL_FLUSH);
4117 static int iommu_suspend(void)
4119 struct dmar_drhd_unit *drhd;
4120 struct intel_iommu *iommu = NULL;
4123 for_each_active_iommu(iommu, drhd) {
4124 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4126 if (!iommu->iommu_state)
4132 for_each_active_iommu(iommu, drhd) {
4133 iommu_disable_translation(iommu);
4135 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4137 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4138 readl(iommu->reg + DMAR_FECTL_REG);
4139 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4140 readl(iommu->reg + DMAR_FEDATA_REG);
4141 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4142 readl(iommu->reg + DMAR_FEADDR_REG);
4143 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4144 readl(iommu->reg + DMAR_FEUADDR_REG);
4146 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4151 for_each_active_iommu(iommu, drhd)
4152 kfree(iommu->iommu_state);
4157 static void iommu_resume(void)
4159 struct dmar_drhd_unit *drhd;
4160 struct intel_iommu *iommu = NULL;
4163 if (init_iommu_hw()) {
4165 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4167 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4171 for_each_active_iommu(iommu, drhd) {
4173 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4175 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4176 iommu->reg + DMAR_FECTL_REG);
4177 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4178 iommu->reg + DMAR_FEDATA_REG);
4179 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4180 iommu->reg + DMAR_FEADDR_REG);
4181 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4182 iommu->reg + DMAR_FEUADDR_REG);
4184 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4187 for_each_active_iommu(iommu, drhd)
4188 kfree(iommu->iommu_state);
4191 static struct syscore_ops iommu_syscore_ops = {
4192 .resume = iommu_resume,
4193 .suspend = iommu_suspend,
4196 static void __init init_iommu_pm_ops(void)
4198 register_syscore_ops(&iommu_syscore_ops);
4202 static inline void init_iommu_pm_ops(void) {}
4203 #endif /* CONFIG_PM */
4206 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4208 struct acpi_dmar_reserved_memory *rmrr;
4209 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4210 struct dmar_rmrr_unit *rmrru;
4213 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4217 rmrru->hdr = header;
4218 rmrr = (struct acpi_dmar_reserved_memory *)header;
4219 rmrru->base_address = rmrr->base_address;
4220 rmrru->end_address = rmrr->end_address;
4222 length = rmrr->end_address - rmrr->base_address + 1;
4223 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4228 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4229 ((void *)rmrr) + rmrr->header.length,
4230 &rmrru->devices_cnt);
4231 if (rmrru->devices_cnt && rmrru->devices == NULL)
4234 list_add(&rmrru->list, &dmar_rmrr_units);
4245 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4247 struct dmar_atsr_unit *atsru;
4248 struct acpi_dmar_atsr *tmp;
4250 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4251 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4252 if (atsr->segment != tmp->segment)
4254 if (atsr->header.length != tmp->header.length)
4256 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4263 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4265 struct acpi_dmar_atsr *atsr;
4266 struct dmar_atsr_unit *atsru;
4268 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4271 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4272 atsru = dmar_find_atsr(atsr);
4276 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4281 * If memory is allocated from slab by ACPI _DSM method, we need to
4282 * copy the memory content because the memory buffer will be freed
4285 atsru->hdr = (void *)(atsru + 1);
4286 memcpy(atsru->hdr, hdr, hdr->length);
4287 atsru->include_all = atsr->flags & 0x1;
4288 if (!atsru->include_all) {
4289 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4290 (void *)atsr + atsr->header.length,
4291 &atsru->devices_cnt);
4292 if (atsru->devices_cnt && atsru->devices == NULL) {
4298 list_add_rcu(&atsru->list, &dmar_atsr_units);
4303 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4305 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4309 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4311 struct acpi_dmar_atsr *atsr;
4312 struct dmar_atsr_unit *atsru;
4314 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4315 atsru = dmar_find_atsr(atsr);
4317 list_del_rcu(&atsru->list);
4319 intel_iommu_free_atsr(atsru);
4325 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4329 struct acpi_dmar_atsr *atsr;
4330 struct dmar_atsr_unit *atsru;
4332 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4333 atsru = dmar_find_atsr(atsr);
4337 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4338 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4346 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4349 struct intel_iommu *iommu = dmaru->iommu;
4351 if (g_iommus[iommu->seq_id])
4354 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4355 pr_warn("%s: Doesn't support hardware pass through.\n",
4359 if (!ecap_sc_support(iommu->ecap) &&
4360 domain_update_iommu_snooping(iommu)) {
4361 pr_warn("%s: Doesn't support snooping.\n",
4365 sp = domain_update_iommu_superpage(iommu) - 1;
4366 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4367 pr_warn("%s: Doesn't support large page.\n",
4373 * Disable translation if already enabled prior to OS handover.
4375 if (iommu->gcmd & DMA_GCMD_TE)
4376 iommu_disable_translation(iommu);
4378 g_iommus[iommu->seq_id] = iommu;
4379 ret = iommu_init_domains(iommu);
4381 ret = iommu_alloc_root_entry(iommu);
4385 #ifdef CONFIG_INTEL_IOMMU_SVM
4386 if (pasid_supported(iommu))
4387 intel_svm_init(iommu);
4390 if (dmaru->ignored) {
4392 * we always have to disable PMRs or DMA may fail on this device
4395 iommu_disable_protect_mem_regions(iommu);
4399 intel_iommu_init_qi(iommu);
4400 iommu_flush_write_buffer(iommu);
4402 #ifdef CONFIG_INTEL_IOMMU_SVM
4403 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
4404 ret = intel_svm_enable_prq(iommu);
4409 ret = dmar_set_interrupt(iommu);
4413 iommu_set_root_entry(iommu);
4414 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4415 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4416 iommu_enable_translation(iommu);
4418 iommu_disable_protect_mem_regions(iommu);
4422 disable_dmar_iommu(iommu);
4424 free_dmar_iommu(iommu);
4428 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4431 struct intel_iommu *iommu = dmaru->iommu;
4433 if (!intel_iommu_enabled)
4439 ret = intel_iommu_add(dmaru);
4441 disable_dmar_iommu(iommu);
4442 free_dmar_iommu(iommu);
4448 static void intel_iommu_free_dmars(void)
4450 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4451 struct dmar_atsr_unit *atsru, *atsr_n;
4453 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4454 list_del(&rmrru->list);
4455 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4460 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4461 list_del(&atsru->list);
4462 intel_iommu_free_atsr(atsru);
4466 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4469 struct pci_bus *bus;
4470 struct pci_dev *bridge = NULL;
4472 struct acpi_dmar_atsr *atsr;
4473 struct dmar_atsr_unit *atsru;
4475 dev = pci_physfn(dev);
4476 for (bus = dev->bus; bus; bus = bus->parent) {
4478 /* If it's an integrated device, allow ATS */
4481 /* Connected via non-PCIe: no ATS */
4482 if (!pci_is_pcie(bridge) ||
4483 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4485 /* If we found the root port, look it up in the ATSR */
4486 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4491 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4492 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4493 if (atsr->segment != pci_domain_nr(dev->bus))
4496 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4497 if (tmp == &bridge->dev)
4500 if (atsru->include_all)
4510 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4513 struct dmar_rmrr_unit *rmrru;
4514 struct dmar_atsr_unit *atsru;
4515 struct acpi_dmar_atsr *atsr;
4516 struct acpi_dmar_reserved_memory *rmrr;
4518 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4521 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4522 rmrr = container_of(rmrru->hdr,
4523 struct acpi_dmar_reserved_memory, header);
4524 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4525 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4526 ((void *)rmrr) + rmrr->header.length,
4527 rmrr->segment, rmrru->devices,
4528 rmrru->devices_cnt);
4531 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4532 dmar_remove_dev_scope(info, rmrr->segment,
4533 rmrru->devices, rmrru->devices_cnt);
4537 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4538 if (atsru->include_all)
4541 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4542 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4543 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4544 (void *)atsr + atsr->header.length,
4545 atsr->segment, atsru->devices,
4546 atsru->devices_cnt);
4551 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4552 if (dmar_remove_dev_scope(info, atsr->segment,
4553 atsru->devices, atsru->devices_cnt))
4562 * Here we only respond to action of unbound device from driver.
4564 * Added device is not attached to its DMAR domain here yet. That will happen
4565 * when mapping the device to iova.
4567 static int device_notifier(struct notifier_block *nb,
4568 unsigned long action, void *data)
4570 struct device *dev = data;
4571 struct dmar_domain *domain;
4573 if (iommu_dummy(dev))
4576 if (action == BUS_NOTIFY_REMOVED_DEVICE) {
4577 domain = find_domain(dev);
4581 dmar_remove_one_dev_info(dev);
4582 if (!domain_type_is_vm_or_si(domain) &&
4583 list_empty(&domain->devices))
4584 domain_exit(domain);
4585 } else if (action == BUS_NOTIFY_ADD_DEVICE) {
4586 if (iommu_should_identity_map(dev, 1))
4587 domain_add_dev_info(si_domain, dev);
4593 static struct notifier_block device_nb = {
4594 .notifier_call = device_notifier,
4597 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4598 unsigned long val, void *v)
4600 struct memory_notify *mhp = v;
4601 unsigned long long start, end;
4602 unsigned long start_vpfn, last_vpfn;
4605 case MEM_GOING_ONLINE:
4606 start = mhp->start_pfn << PAGE_SHIFT;
4607 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4608 if (iommu_domain_identity_map(si_domain, start, end)) {
4609 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4616 case MEM_CANCEL_ONLINE:
4617 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4618 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4619 while (start_vpfn <= last_vpfn) {
4621 struct dmar_drhd_unit *drhd;
4622 struct intel_iommu *iommu;
4623 struct page *freelist;
4625 iova = find_iova(&si_domain->iovad, start_vpfn);
4627 pr_debug("Failed get IOVA for PFN %lx\n",
4632 iova = split_and_remove_iova(&si_domain->iovad, iova,
4633 start_vpfn, last_vpfn);
4635 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4636 start_vpfn, last_vpfn);
4640 freelist = domain_unmap(si_domain, iova->pfn_lo,
4644 for_each_active_iommu(iommu, drhd)
4645 iommu_flush_iotlb_psi(iommu, si_domain,
4646 iova->pfn_lo, iova_size(iova),
4649 dma_free_pagelist(freelist);
4651 start_vpfn = iova->pfn_hi + 1;
4652 free_iova_mem(iova);
4660 static struct notifier_block intel_iommu_memory_nb = {
4661 .notifier_call = intel_iommu_memory_notifier,
4665 static void free_all_cpu_cached_iovas(unsigned int cpu)
4669 for (i = 0; i < g_num_of_iommus; i++) {
4670 struct intel_iommu *iommu = g_iommus[i];
4671 struct dmar_domain *domain;
4677 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4678 domain = get_iommu_domain(iommu, (u16)did);
4682 free_cpu_cached_iovas(cpu, &domain->iovad);
4687 static int intel_iommu_cpu_dead(unsigned int cpu)
4689 free_all_cpu_cached_iovas(cpu);
4693 static void intel_disable_iommus(void)
4695 struct intel_iommu *iommu = NULL;
4696 struct dmar_drhd_unit *drhd;
4698 for_each_iommu(iommu, drhd)
4699 iommu_disable_translation(iommu);
4702 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4704 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4706 return container_of(iommu_dev, struct intel_iommu, iommu);
4709 static ssize_t intel_iommu_show_version(struct device *dev,
4710 struct device_attribute *attr,
4713 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4714 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4715 return sprintf(buf, "%d:%d\n",
4716 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4718 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4720 static ssize_t intel_iommu_show_address(struct device *dev,
4721 struct device_attribute *attr,
4724 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4725 return sprintf(buf, "%llx\n", iommu->reg_phys);
4727 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4729 static ssize_t intel_iommu_show_cap(struct device *dev,
4730 struct device_attribute *attr,
4733 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4734 return sprintf(buf, "%llx\n", iommu->cap);
4736 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4738 static ssize_t intel_iommu_show_ecap(struct device *dev,
4739 struct device_attribute *attr,
4742 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4743 return sprintf(buf, "%llx\n", iommu->ecap);
4745 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4747 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4748 struct device_attribute *attr,
4751 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4752 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4754 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4756 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4757 struct device_attribute *attr,
4760 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4761 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4762 cap_ndoms(iommu->cap)));
4764 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4766 static struct attribute *intel_iommu_attrs[] = {
4767 &dev_attr_version.attr,
4768 &dev_attr_address.attr,
4770 &dev_attr_ecap.attr,
4771 &dev_attr_domains_supported.attr,
4772 &dev_attr_domains_used.attr,
4776 static struct attribute_group intel_iommu_group = {
4777 .name = "intel-iommu",
4778 .attrs = intel_iommu_attrs,
4781 const struct attribute_group *intel_iommu_groups[] = {
4786 static int __init platform_optin_force_iommu(void)
4788 struct pci_dev *pdev = NULL;
4789 bool has_untrusted_dev = false;
4791 if (!dmar_platform_optin() || no_platform_optin)
4794 for_each_pci_dev(pdev) {
4795 if (pdev->untrusted) {
4796 has_untrusted_dev = true;
4801 if (!has_untrusted_dev)
4804 if (no_iommu || dmar_disabled)
4805 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
4808 * If Intel-IOMMU is disabled by default, we will apply identity
4809 * map for all devices except those marked as being untrusted.
4812 iommu_identity_mapping |= IDENTMAP_ALL;
4815 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4823 int __init intel_iommu_init(void)
4826 struct dmar_drhd_unit *drhd;
4827 struct intel_iommu *iommu;
4830 * Intel IOMMU is required for a TXT/tboot launch or platform
4831 * opt in, so enforce that.
4833 force_on = tboot_force_iommu() || platform_optin_force_iommu();
4835 if (iommu_init_mempool()) {
4837 panic("tboot: Failed to initialize iommu memory\n");
4841 down_write(&dmar_global_lock);
4842 if (dmar_table_init()) {
4844 panic("tboot: Failed to initialize DMAR table\n");
4848 if (dmar_dev_scope_init() < 0) {
4850 panic("tboot: Failed to initialize DMAR device scope\n");
4854 up_write(&dmar_global_lock);
4857 * The bus notifier takes the dmar_global_lock, so lockdep will
4858 * complain later when we register it under the lock.
4860 dmar_register_bus_notifier();
4862 down_write(&dmar_global_lock);
4864 if (no_iommu || dmar_disabled) {
4866 * We exit the function here to ensure IOMMU's remapping and
4867 * mempool aren't setup, which means that the IOMMU's PMRs
4868 * won't be disabled via the call to init_dmars(). So disable
4869 * it explicitly here. The PMRs were setup by tboot prior to
4870 * calling SENTER, but the kernel is expected to reset/tear
4873 if (intel_iommu_tboot_noforce) {
4874 for_each_iommu(iommu, drhd)
4875 iommu_disable_protect_mem_regions(iommu);
4879 * Make sure the IOMMUs are switched off, even when we
4880 * boot into a kexec kernel and the previous kernel left
4883 intel_disable_iommus();
4887 if (list_empty(&dmar_rmrr_units))
4888 pr_info("No RMRR found\n");
4890 if (list_empty(&dmar_atsr_units))
4891 pr_info("No ATSR found\n");
4893 if (dmar_init_reserved_ranges()) {
4895 panic("tboot: Failed to reserve iommu ranges\n");
4896 goto out_free_reserved_range;
4899 init_no_remapping_devices();
4904 panic("tboot: Failed to initialize DMARs\n");
4905 pr_err("Initialization failed\n");
4906 goto out_free_reserved_range;
4908 up_write(&dmar_global_lock);
4909 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4911 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4914 dma_ops = &intel_dma_ops;
4916 init_iommu_pm_ops();
4918 for_each_active_iommu(iommu, drhd) {
4919 iommu_device_sysfs_add(&iommu->iommu, NULL,
4922 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4923 iommu_device_register(&iommu->iommu);
4926 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4927 bus_register_notifier(&pci_bus_type, &device_nb);
4928 if (si_domain && !hw_pass_through)
4929 register_memory_notifier(&intel_iommu_memory_nb);
4930 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4931 intel_iommu_cpu_dead);
4932 intel_iommu_enabled = 1;
4933 intel_iommu_debugfs_init();
4937 out_free_reserved_range:
4938 put_iova_domain(&reserved_iova_list);
4940 intel_iommu_free_dmars();
4941 up_write(&dmar_global_lock);
4942 iommu_exit_mempool();
4946 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4948 struct intel_iommu *iommu = opaque;
4950 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4955 * NB - intel-iommu lacks any sort of reference counting for the users of
4956 * dependent devices. If multiple endpoints have intersecting dependent
4957 * devices, unbinding the driver from any one of them will possibly leave
4958 * the others unable to operate.
4960 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4962 if (!iommu || !dev || !dev_is_pci(dev))
4965 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4968 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4970 struct intel_iommu *iommu;
4971 unsigned long flags;
4973 assert_spin_locked(&device_domain_lock);
4978 iommu = info->iommu;
4981 if (dev_is_pci(info->dev) && sm_supported(iommu))
4982 intel_pasid_tear_down_entry(iommu, info->dev,
4985 iommu_disable_dev_iotlb(info);
4986 domain_context_clear(iommu, info->dev);
4987 intel_pasid_free_table(info->dev);
4990 unlink_domain_info(info);
4992 spin_lock_irqsave(&iommu->lock, flags);
4993 domain_detach_iommu(info->domain, iommu);
4994 spin_unlock_irqrestore(&iommu->lock, flags);
4996 free_devinfo_mem(info);
4999 static void dmar_remove_one_dev_info(struct device *dev)
5001 struct device_domain_info *info;
5002 unsigned long flags;
5004 spin_lock_irqsave(&device_domain_lock, flags);
5005 info = dev->archdata.iommu;
5006 __dmar_remove_one_dev_info(info);
5007 spin_unlock_irqrestore(&device_domain_lock, flags);
5010 static int md_domain_init(struct dmar_domain *domain, int guest_width)
5014 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5015 domain_reserve_special_ranges(domain);
5017 /* calculate AGAW */
5018 domain->gaw = guest_width;
5019 adjust_width = guestwidth_to_adjustwidth(guest_width);
5020 domain->agaw = width_to_agaw(adjust_width);
5022 domain->iommu_coherency = 0;
5023 domain->iommu_snooping = 0;
5024 domain->iommu_superpage = 0;
5025 domain->max_addr = 0;
5027 /* always allocate the top pgd */
5028 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5031 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5035 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5037 struct dmar_domain *dmar_domain;
5038 struct iommu_domain *domain;
5040 if (type != IOMMU_DOMAIN_UNMANAGED)
5043 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5045 pr_err("Can't allocate dmar_domain\n");
5048 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5049 pr_err("Domain initialization failed\n");
5050 domain_exit(dmar_domain);
5053 domain_update_iommu_cap(dmar_domain);
5055 domain = &dmar_domain->domain;
5056 domain->geometry.aperture_start = 0;
5057 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5058 domain->geometry.force_aperture = true;
5063 static void intel_iommu_domain_free(struct iommu_domain *domain)
5065 domain_exit(to_dmar_domain(domain));
5068 static int intel_iommu_attach_device(struct iommu_domain *domain,
5071 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5072 struct intel_iommu *iommu;
5076 if (device_is_rmrr_locked(dev)) {
5077 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5081 /* normally dev is not mapped */
5082 if (unlikely(domain_context_mapped(dev))) {
5083 struct dmar_domain *old_domain;
5085 old_domain = find_domain(dev);
5088 dmar_remove_one_dev_info(dev);
5091 if (!domain_type_is_vm_or_si(old_domain) &&
5092 list_empty(&old_domain->devices))
5093 domain_exit(old_domain);
5097 iommu = device_to_iommu(dev, &bus, &devfn);
5101 /* check if this iommu agaw is sufficient for max mapped address */
5102 addr_width = agaw_to_width(iommu->agaw);
5103 if (addr_width > cap_mgaw(iommu->cap))
5104 addr_width = cap_mgaw(iommu->cap);
5106 if (dmar_domain->max_addr > (1LL << addr_width)) {
5107 dev_err(dev, "%s: iommu width (%d) is not "
5108 "sufficient for the mapped address (%llx)\n",
5109 __func__, addr_width, dmar_domain->max_addr);
5112 dmar_domain->gaw = addr_width;
5115 * Knock out extra levels of page tables if necessary
5117 while (iommu->agaw < dmar_domain->agaw) {
5118 struct dma_pte *pte;
5120 pte = dmar_domain->pgd;
5121 if (dma_pte_present(pte)) {
5122 dmar_domain->pgd = (struct dma_pte *)
5123 phys_to_virt(dma_pte_addr(pte));
5124 free_pgtable_page(pte);
5126 dmar_domain->agaw--;
5129 return domain_add_dev_info(dmar_domain, dev);
5132 static void intel_iommu_detach_device(struct iommu_domain *domain,
5135 dmar_remove_one_dev_info(dev);
5138 static int intel_iommu_map(struct iommu_domain *domain,
5139 unsigned long iova, phys_addr_t hpa,
5140 size_t size, int iommu_prot)
5142 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5147 if (iommu_prot & IOMMU_READ)
5148 prot |= DMA_PTE_READ;
5149 if (iommu_prot & IOMMU_WRITE)
5150 prot |= DMA_PTE_WRITE;
5151 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5152 prot |= DMA_PTE_SNP;
5154 max_addr = iova + size;
5155 if (dmar_domain->max_addr < max_addr) {
5158 /* check if minimum agaw is sufficient for mapped address */
5159 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5160 if (end < max_addr) {
5161 pr_err("%s: iommu width (%d) is not "
5162 "sufficient for the mapped address (%llx)\n",
5163 __func__, dmar_domain->gaw, max_addr);
5166 dmar_domain->max_addr = max_addr;
5168 /* Round up size to next multiple of PAGE_SIZE, if it and
5169 the low bits of hpa would take us onto the next page */
5170 size = aligned_nrpages(hpa, size);
5171 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5172 hpa >> VTD_PAGE_SHIFT, size, prot);
5176 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5177 unsigned long iova, size_t size)
5179 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5180 struct page *freelist = NULL;
5181 unsigned long start_pfn, last_pfn;
5182 unsigned int npages;
5183 int iommu_id, level = 0;
5185 /* Cope with horrid API which requires us to unmap more than the
5186 size argument if it happens to be a large-page mapping. */
5187 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5189 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5190 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5192 start_pfn = iova >> VTD_PAGE_SHIFT;
5193 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5195 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5197 npages = last_pfn - start_pfn + 1;
5199 for_each_domain_iommu(iommu_id, dmar_domain)
5200 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5201 start_pfn, npages, !freelist, 0);
5203 dma_free_pagelist(freelist);
5205 if (dmar_domain->max_addr == iova + size)
5206 dmar_domain->max_addr = iova;
5211 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5214 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5215 struct dma_pte *pte;
5219 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5221 phys = dma_pte_addr(pte);
5226 static bool intel_iommu_capable(enum iommu_cap cap)
5228 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5229 return domain_update_iommu_snooping(NULL) == 1;
5230 if (cap == IOMMU_CAP_INTR_REMAP)
5231 return irq_remapping_enabled == 1;
5236 static int intel_iommu_add_device(struct device *dev)
5238 struct intel_iommu *iommu;
5239 struct iommu_group *group;
5242 iommu = device_to_iommu(dev, &bus, &devfn);
5246 iommu_device_link(&iommu->iommu, dev);
5248 group = iommu_group_get_for_dev(dev);
5251 return PTR_ERR(group);
5253 iommu_group_put(group);
5257 static void intel_iommu_remove_device(struct device *dev)
5259 struct intel_iommu *iommu;
5262 iommu = device_to_iommu(dev, &bus, &devfn);
5266 iommu_group_remove_device(dev);
5268 iommu_device_unlink(&iommu->iommu, dev);
5271 static void intel_iommu_get_resv_regions(struct device *device,
5272 struct list_head *head)
5274 struct iommu_resv_region *reg;
5275 struct dmar_rmrr_unit *rmrr;
5276 struct device *i_dev;
5280 for_each_rmrr_units(rmrr) {
5281 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5283 if (i_dev != device)
5286 list_add_tail(&rmrr->resv->list, head);
5291 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5292 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5296 list_add_tail(®->list, head);
5299 static void intel_iommu_put_resv_regions(struct device *dev,
5300 struct list_head *head)
5302 struct iommu_resv_region *entry, *next;
5304 list_for_each_entry_safe(entry, next, head, list) {
5305 if (entry->type == IOMMU_RESV_MSI)
5310 #ifdef CONFIG_INTEL_IOMMU_SVM
5311 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5313 struct device_domain_info *info;
5314 struct context_entry *context;
5315 struct dmar_domain *domain;
5316 unsigned long flags;
5320 domain = get_valid_domain_for_dev(sdev->dev);
5324 spin_lock_irqsave(&device_domain_lock, flags);
5325 spin_lock(&iommu->lock);
5328 info = sdev->dev->archdata.iommu;
5329 if (!info || !info->pasid_supported)
5332 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5333 if (WARN_ON(!context))
5336 ctx_lo = context[0].lo;
5338 sdev->did = FLPT_DEFAULT_DID;
5339 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5341 if (!(ctx_lo & CONTEXT_PASIDE)) {
5342 ctx_lo |= CONTEXT_PASIDE;
5343 context[0].lo = ctx_lo;
5345 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5346 DMA_CCMD_MASK_NOBIT,
5347 DMA_CCMD_DEVICE_INVL);
5350 /* Enable PASID support in the device, if it wasn't already */
5351 if (!info->pasid_enabled)
5352 iommu_enable_dev_iotlb(info);
5354 if (info->ats_enabled) {
5355 sdev->dev_iotlb = 1;
5356 sdev->qdep = info->ats_qdep;
5357 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5363 spin_unlock(&iommu->lock);
5364 spin_unlock_irqrestore(&device_domain_lock, flags);
5369 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5371 struct intel_iommu *iommu;
5374 if (iommu_dummy(dev)) {
5376 "No IOMMU translation for device; cannot enable SVM\n");
5380 iommu = device_to_iommu(dev, &bus, &devfn);
5382 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5388 #endif /* CONFIG_INTEL_IOMMU_SVM */
5390 const struct iommu_ops intel_iommu_ops = {
5391 .capable = intel_iommu_capable,
5392 .domain_alloc = intel_iommu_domain_alloc,
5393 .domain_free = intel_iommu_domain_free,
5394 .attach_dev = intel_iommu_attach_device,
5395 .detach_dev = intel_iommu_detach_device,
5396 .map = intel_iommu_map,
5397 .unmap = intel_iommu_unmap,
5398 .iova_to_phys = intel_iommu_iova_to_phys,
5399 .add_device = intel_iommu_add_device,
5400 .remove_device = intel_iommu_remove_device,
5401 .get_resv_regions = intel_iommu_get_resv_regions,
5402 .put_resv_regions = intel_iommu_put_resv_regions,
5403 .device_group = pci_device_group,
5404 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5407 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5409 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5410 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
5414 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5415 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5416 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5417 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5418 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5419 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5420 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5422 static void quirk_iommu_rwbf(struct pci_dev *dev)
5425 * Mobile 4 Series Chipset neglects to set RWBF capability,
5426 * but needs it. Same seems to hold for the desktop versions.
5428 pci_info(dev, "Forcing write-buffer flush capability\n");
5432 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5433 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5434 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5435 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5436 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5437 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5438 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5441 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5442 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5443 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5444 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5445 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5446 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5447 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5448 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5450 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5454 if (pci_read_config_word(dev, GGC, &ggc))
5457 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5458 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5460 } else if (dmar_map_gfx) {
5461 /* we have to ensure the gfx device is idle before we flush */
5462 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
5463 intel_iommu_strict = 1;
5466 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5467 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5468 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5469 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5471 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5472 ISOCH DMAR unit for the Azalia sound device, but not give it any
5473 TLB entries, which causes it to deadlock. Check for that. We do
5474 this in a function called from init_dmars(), instead of in a PCI
5475 quirk, because we don't want to print the obnoxious "BIOS broken"
5476 message if VT-d is actually disabled.
5478 static void __init check_tylersburg_isoch(void)
5480 struct pci_dev *pdev;
5481 uint32_t vtisochctrl;
5483 /* If there's no Azalia in the system anyway, forget it. */
5484 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5489 /* System Management Registers. Might be hidden, in which case
5490 we can't do the sanity check. But that's OK, because the
5491 known-broken BIOSes _don't_ actually hide it, so far. */
5492 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5496 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5503 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5504 if (vtisochctrl & 1)
5507 /* Drop all bits other than the number of TLB entries */
5508 vtisochctrl &= 0x1c;
5510 /* If we have the recommended number of TLB entries (16), fine. */
5511 if (vtisochctrl == 0x10)
5514 /* Zero TLB entries? You get to ride the short bus to school. */
5516 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5517 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5518 dmi_get_system_info(DMI_BIOS_VENDOR),
5519 dmi_get_system_info(DMI_BIOS_VERSION),
5520 dmi_get_system_info(DMI_PRODUCT_VERSION));
5521 iommu_identity_mapping |= IDENTMAP_AZALIA;
5525 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",