projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: fix a race window in allocating domain ID for virtual machines
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
54485c30 27#include <linux/export.h>
ba395927
KA		 28#include <linux/slab.h>
29#include <linux/irq.h>
30#include <linux/interrupt.h>
ba395927
KA		 31#include <linux/spinlock.h>
32#include <linux/pci.h>
33#include <linux/dmar.h>
34#include <linux/dma-mapping.h>
35#include <linux/mempool.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
8a8f422d 45#include <asm/irq_remapping.h>
ba395927 46#include <asm/cacheflush.h>
46a7fa27 47#include <asm/iommu.h>
ba395927 48
078e1ee2 49#include "irq_remapping.h"
61e015ac 50#include "pci.h"
078e1ee2 51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927
KA		 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
56#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 57#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 58
59#define IOAPIC_RANGE_START (0xfee00000)
60#define IOAPIC_RANGE_END (0xfeefffff)
61#define IOVA_START_ADDR (0x1000)
62
63#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
64
4ed0d3e6
FY		 65#define MAX_AGAW_WIDTH 64
66
2ebe3151
DW		 67#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
68#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
69
70/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
71 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
72#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
73 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
74#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 75
f27be03b 76#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 77#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 78#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 79
df08cdc7
AM		 80/* page table handling */
81#define LEVEL_STRIDE (9)
82#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83
6d1c56a9
OBC		 84/*
85 * This bitmap is used to advertise the page sizes our hardware support
86 * to the IOMMU core, which will then use this information to split
87 * physically contiguous memory regions it is mapping into page sizes
88 * that we support.
89 *
90 * Traditionally the IOMMU core just handed us the mappings directly,
91 * after making sure the size is an order of a 4KiB page and that the
92 * mapping has natural alignment.
93 *
94 * To retain this behavior, we currently advertise that we support
95 * all page sizes that are an order of 4KiB.
96 *
97 * If at some point we'd like to utilize the IOMMU core's new behavior,
98 * we could change this to advertise the real page sizes we support.
99 */
100#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
101
df08cdc7
AM		 102static inline int agaw_to_level(int agaw)
103{
104 return agaw + 2;
105}
106
107static inline int agaw_to_width(int agaw)
108{
109 return 30 + agaw * LEVEL_STRIDE;
110}
111
112static inline int width_to_agaw(int width)
113{
114 return (width - 30) / LEVEL_STRIDE;
115}
116
117static inline unsigned int level_to_offset_bits(int level)
118{
119 return (level - 1) * LEVEL_STRIDE;
120}
121
122static inline int pfn_level_offset(unsigned long pfn, int level)
123{
124 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
125}
126
127static inline unsigned long level_mask(int level)
128{
129 return -1UL << level_to_offset_bits(level);
130}
131
132static inline unsigned long level_size(int level)
133{
134 return 1UL << level_to_offset_bits(level);
135}
136
137static inline unsigned long align_to_level(unsigned long pfn, int level)
138{
139 return (pfn + level_size(level) - 1) & level_mask(level);
140}
fd18de50 141
6dd9a7c7
YS		 142static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
143{
144 return 1 << ((lvl - 1) * LEVEL_STRIDE);
145}
146
dd4e8319
DW		 147/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
148 are never going to work. */
149static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
150{
151 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
152}
153
154static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
155{
156 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
157}
158static inline unsigned long page_to_dma_pfn(struct page *pg)
159{
160 return mm_to_dma_pfn(page_to_pfn(pg));
161}
162static inline unsigned long virt_to_dma_pfn(void *p)
163{
164 return page_to_dma_pfn(virt_to_page(p));
165}
166
d9630fe9
WH		 167/* global iommu list, set NULL for ignored DMAR units */
168static struct intel_iommu **g_iommus;
169
e0fc7e0b 170static void __init check_tylersburg_isoch(void);
9af88143
DW		 171static int rwbf_quirk;
172
b779260b
JC		 173/*
174 * set to 1 to panic kernel if can't successfully enable VT-d
175 * (used when kernel is launched w/ TXT)
176 */
177static int force_on = 0;
178
46b08e1a
MM		 179/*
180 * 0: Present
181 * 1-11: Reserved
182 * 12-63: Context Ptr (12 - (haw-1))
183 * 64-127: Reserved
184 */
185struct root_entry {
186 u64 val;
187 u64 rsvd1;
188};
189#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
190static inline bool root_present(struct root_entry *root)
191{
192 return (root->val & 1);
193}
194static inline void set_root_present(struct root_entry *root)
195{
196 root->val |= 1;
197}
198static inline void set_root_value(struct root_entry *root, unsigned long value)
199{
200 root->val |= value & VTD_PAGE_MASK;
201}
202
203static inline struct context_entry *
204get_context_addr_from_root(struct root_entry *root)
205{
206 return (struct context_entry *)
207 (root_present(root)?phys_to_virt(
208 root->val & VTD_PAGE_MASK) :
209 NULL);
210}
211
7a8fc25e
MM		 212/*
213 * low 64 bits:
214 * 0: present
215 * 1: fault processing disable
216 * 2-3: translation type
217 * 12-63: address space root
218 * high 64 bits:
219 * 0-2: address width
220 * 3-6: aval
221 * 8-23: domain id
222 */
223struct context_entry {
224 u64 lo;
225 u64 hi;
226};
c07e7d21
MM		 227
228static inline bool context_present(struct context_entry *context)
229{
230 return (context->lo & 1);
231}
232static inline void context_set_present(struct context_entry *context)
233{
234 context->lo |= 1;
235}
236
237static inline void context_set_fault_enable(struct context_entry *context)
238{
239 context->lo &= (((u64)-1) << 2) | 1;
240}
241
c07e7d21
MM		 242static inline void context_set_translation_type(struct context_entry *context,
243 unsigned long value)
244{
245 context->lo &= (((u64)-1) << 4) | 3;
246 context->lo |= (value & 3) << 2;
247}
248
249static inline void context_set_address_root(struct context_entry *context,
250 unsigned long value)
251{
252 context->lo |= value & VTD_PAGE_MASK;
253}
254
255static inline void context_set_address_width(struct context_entry *context,
256 unsigned long value)
257{
258 context->hi |= value & 7;
259}
260
261static inline void context_set_domain_id(struct context_entry *context,
262 unsigned long value)
263{
264 context->hi |= (value & ((1 << 16) - 1)) << 8;
265}
266
267static inline void context_clear_entry(struct context_entry *context)
268{
269 context->lo = 0;
270 context->hi = 0;
271}
7a8fc25e 272
622ba12a
MM		 273/*
274 * 0: readable
275 * 1: writable
276 * 2-6: reserved
277 * 7: super page
9cf06697
SY		 278 * 8-10: available
279 * 11: snoop behavior
622ba12a
MM		 280 * 12-63: Host physcial address
281 */
282struct dma_pte {
283 u64 val;
284};
622ba12a 285
19c239ce
MM		 286static inline void dma_clear_pte(struct dma_pte *pte)
287{
288 pte->val = 0;
289}
290
291static inline void dma_set_pte_readable(struct dma_pte *pte)
292{
293 pte->val |= DMA_PTE_READ;
294}
295
296static inline void dma_set_pte_writable(struct dma_pte *pte)
297{
298 pte->val |= DMA_PTE_WRITE;
299}
300
9cf06697
SY		 301static inline void dma_set_pte_snp(struct dma_pte *pte)
302{
303 pte->val |= DMA_PTE_SNP;
304}
305
19c239ce
MM		 306static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
307{
308 pte->val = (pte->val & ~3) | (prot & 3);
309}
310
311static inline u64 dma_pte_addr(struct dma_pte *pte)
312{
c85994e4
DW		 313#ifdef CONFIG_64BIT
314 return pte->val & VTD_PAGE_MASK;
315#else
316 /* Must have a full atomic 64-bit read */
1a8bd481 317 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 318#endif
19c239ce
MM		 319}
320
dd4e8319 321static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 322{
dd4e8319 323 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 324}
325
326static inline bool dma_pte_present(struct dma_pte *pte)
327{
328 return (pte->val & 3) != 0;
329}
622ba12a 330
4399c8bf
AK		 331static inline bool dma_pte_superpage(struct dma_pte *pte)
332{
333 return (pte->val & (1 << 7));
334}
335
75e6bf96
DW		 336static inline int first_pte_in_page(struct dma_pte *pte)
337{
338 return !((unsigned long)pte & ~VTD_PAGE_MASK);
339}
340
2c2e2c38
FY		 341/*
342 * This domain is a statically identity mapping domain.
343 * 1. This domain creats a static 1:1 mapping to all usable memory.
344 * 2. It maps to each iommu if successful.
345 * 3. Each iommu mapps to this domain if successful.
346 */
19943b0e
DW		 347static struct dmar_domain *si_domain;
348static int hw_pass_through = 1;
2c2e2c38 349
3b5410e7 350/* devices under the same p2p bridge are owned in one domain */
cdc7b837 351#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 352
1ce28feb
WH		 353/* domain represents a virtual machine, more than one devices
354 * across iommus may be owned in one domain, e.g. kvm guest.
355 */
356#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
357
2c2e2c38
FY		 358/* si_domain contains mulitple devices */
359#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
360
1b198bb0
MT		 361/* define the limit of IOMMUs supported in each domain */
362#ifdef CONFIG_X86
363# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
364#else
365# define IOMMU_UNITS_SUPPORTED 64
366#endif
367
99126f7c
MM		 368struct dmar_domain {
369 int id; /* domain id */
4c923d47 370 int nid; /* node id */
1b198bb0
MT		 371 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
372 /* bitmap of iommus this domain uses*/
99126f7c
MM		 373
374 struct list_head devices; /* all devices' list */
375 struct iova_domain iovad; /* iova's that belong to this domain */
376
377 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 378 int gaw; /* max guest address width */
379
380 /* adjusted guest address width, 0 is level 2 30-bit */
381 int agaw;
382
3b5410e7 383 int flags; /* flags to find out type of domain */
8e604097
WH		 384
385 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 386 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 387 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 388 int iommu_superpage;/* Level of superpages supported:
389 0 == 4KiB (no superpages), 1 == 2MiB,
390 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 391 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 392 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 393};
394
a647dacb
MM		 395/* PCI domain-device relationship */
396struct device_domain_info {
397 struct list_head link; /* link to domain siblings */
398 struct list_head global; /* link to global list */
276dbf99
DW		 399 int segment; /* PCI domain */
400 u8 bus; /* PCI bus number */
a647dacb 401 u8 devfn; /* PCI devfn number */
45e829ea 402 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 403 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 404 struct dmar_domain *domain; /* pointer to domain */
405};
406
5e0d2a6f 407static void flush_unmaps_timeout(unsigned long data);
408
409DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
410
80b20dd8 411#define HIGH_WATER_MARK 250
412struct deferred_flush_tables {
413 int next;
414 struct iova *iova[HIGH_WATER_MARK];
415 struct dmar_domain *domain[HIGH_WATER_MARK];
416};
417
418static struct deferred_flush_tables *deferred_flush;
419
5e0d2a6f 420/* bitmap for indexing intel_iommus */
5e0d2a6f 421static int g_num_of_iommus;
422
423static DEFINE_SPINLOCK(async_umap_flush_lock);
424static LIST_HEAD(unmaps_to_do);
425
426static int timer_on;
427static long list_size;
5e0d2a6f 428
ba395927
KA		 429static void domain_remove_dev_info(struct dmar_domain *domain);
430
d3f13810 431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 432int dmar_disabled = 0;
433#else
434int dmar_disabled = 1;
d3f13810 435#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 436
8bc1f85c
ED		 437int intel_iommu_enabled = 0;
438EXPORT_SYMBOL_GPL(intel_iommu_enabled);
439
2d9e667e 440static int dmar_map_gfx = 1;
7d3b03ce 441static int dmar_forcedac;
5e0d2a6f 442static int intel_iommu_strict;
6dd9a7c7 443static int intel_iommu_superpage = 1;
ba395927 444
c0771df8
DW		 445int intel_iommu_gfx_mapped;
446EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
447
ba395927
KA		 448#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
449static DEFINE_SPINLOCK(device_domain_lock);
450static LIST_HEAD(device_domain_list);
451
a8bcbb0d
JR		 452static struct iommu_ops intel_iommu_ops;
453
ba395927
KA		 454static int __init intel_iommu_setup(char *str)
455{
456 if (!str)
457 return -EINVAL;
458 while (*str) {
0cd5c3c8
KM		 459 if (!strncmp(str, "on", 2)) {
460 dmar_disabled = 0;
461 printk(KERN_INFO "Intel-IOMMU: enabled\n");
462 } else if (!strncmp(str, "off", 3)) {
ba395927 463 dmar_disabled = 1;
0cd5c3c8 464 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 465 } else if (!strncmp(str, "igfx_off", 8)) {
466 dmar_map_gfx = 0;
467 printk(KERN_INFO
468 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 469 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 470 printk(KERN_INFO
7d3b03ce
KA		 471 "Intel-IOMMU: Forcing DAC for PCI devices\n");
472 dmar_forcedac = 1;
5e0d2a6f 473 } else if (!strncmp(str, "strict", 6)) {
474 printk(KERN_INFO
475 "Intel-IOMMU: disable batched IOTLB flush\n");
476 intel_iommu_strict = 1;
6dd9a7c7
YS		 477 } else if (!strncmp(str, "sp_off", 6)) {
478 printk(KERN_INFO
479 "Intel-IOMMU: disable supported super page\n");
480 intel_iommu_superpage = 0;
ba395927
KA		 481 }
482
483 str += strcspn(str, ",");
484 while (*str == ',')
485 str++;
486 }
487 return 0;
488}
489__setup("intel_iommu=", intel_iommu_setup);
490
491static struct kmem_cache *iommu_domain_cache;
492static struct kmem_cache *iommu_devinfo_cache;
493static struct kmem_cache *iommu_iova_cache;
494
4c923d47 495static inline void *alloc_pgtable_page(int node)
eb3fa7cb 496{
4c923d47
SS		 497 struct page *page;
498 void *vaddr = NULL;
eb3fa7cb 499
4c923d47
SS		 500 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
501 if (page)
502 vaddr = page_address(page);
eb3fa7cb 503 return vaddr;
ba395927
KA		 504}
505
506static inline void free_pgtable_page(void *vaddr)
507{
508 free_page((unsigned long)vaddr);
509}
510
511static inline void *alloc_domain_mem(void)
512{
354bb65e 513 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 514}
515
38717946 516static void free_domain_mem(void *vaddr)
ba395927
KA		 517{
518 kmem_cache_free(iommu_domain_cache, vaddr);
519}
520
521static inline void * alloc_devinfo_mem(void)
522{
354bb65e 523 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 524}
525
526static inline void free_devinfo_mem(void *vaddr)
527{
528 kmem_cache_free(iommu_devinfo_cache, vaddr);
529}
530
531struct iova *alloc_iova_mem(void)
532{
354bb65e 533 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 534}
535
536void free_iova_mem(struct iova *iova)
537{
538 kmem_cache_free(iommu_iova_cache, iova);
539}
540
1b573683 541
4ed0d3e6 542static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 543{
544 unsigned long sagaw;
545 int agaw = -1;
546
547 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 548 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 549 agaw >= 0; agaw--) {
550 if (test_bit(agaw, &sagaw))
551 break;
552 }
553
554 return agaw;
555}
556
4ed0d3e6
FY		 557/*
558 * Calculate max SAGAW for each iommu.
559 */
560int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
561{
562 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
563}
564
565/*
566 * calculate agaw for each iommu.
567 * "SAGAW" may be different across iommus, use a default agaw, and
568 * get a supported less agaw for iommus that don't support the default agaw.
569 */
570int iommu_calculate_agaw(struct intel_iommu *iommu)
571{
572 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
573}
574
2c2e2c38 575/* This functionin only returns single iommu in a domain */
8c11e798
WH		 576static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
577{
578 int iommu_id;
579
2c2e2c38 580 /* si_domain and vm domain should not get here. */
1ce28feb 581 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 582 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 583
1b198bb0 584 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 585 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
586 return NULL;
587
588 return g_iommus[iommu_id];
589}
590
8e604097
WH		 591static void domain_update_iommu_coherency(struct dmar_domain *domain)
592{
593 int i;
594
2e12bc29
AW		 595 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
596
597 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 598
1b198bb0 599 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 600 if (!ecap_coherent(g_iommus[i]->ecap)) {
601 domain->iommu_coherency = 0;
602 break;
603 }
8e604097
WH		 604 }
605}
606
58c610bd
SY		 607static void domain_update_iommu_snooping(struct dmar_domain *domain)
608{
609 int i;
610
611 domain->iommu_snooping = 1;
612
1b198bb0 613 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 614 if (!ecap_sc_support(g_iommus[i]->ecap)) {
615 domain->iommu_snooping = 0;
616 break;
617 }
58c610bd
SY		 618 }
619}
620
6dd9a7c7
YS		 621static void domain_update_iommu_superpage(struct dmar_domain *domain)
622{
8140a95d
AK		 623 struct dmar_drhd_unit *drhd;
624 struct intel_iommu *iommu = NULL;
625 int mask = 0xf;
6dd9a7c7
YS		 626
627 if (!intel_iommu_superpage) {
628 domain->iommu_superpage = 0;
629 return;
630 }
631
8140a95d
AK		 632 /* set iommu_superpage to the smallest common denominator */
633 for_each_active_iommu(iommu, drhd) {
634 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 635 if (!mask) {
636 break;
637 }
638 }
639 domain->iommu_superpage = fls(mask);
640}
641
58c610bd
SY		 642/* Some capabilities may be different across iommus */
643static void domain_update_iommu_cap(struct dmar_domain *domain)
644{
645 domain_update_iommu_coherency(domain);
646 domain_update_iommu_snooping(domain);
6dd9a7c7 647 domain_update_iommu_superpage(domain);
58c610bd
SY		 648}
649
276dbf99 650static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 651{
652 struct dmar_drhd_unit *drhd = NULL;
653 int i;
654
655 for_each_drhd_unit(drhd) {
656 if (drhd->ignored)
657 continue;
276dbf99
DW		 658 if (segment != drhd->segment)
659 continue;
c7151a8d 660
924b6231 661 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 662 if (drhd->devices[i] &&
663 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 664 drhd->devices[i]->devfn == devfn)
665 return drhd->iommu;
4958c5dc
DW		 666 if (drhd->devices[i] &&
667 drhd->devices[i]->subordinate &&
924b6231 668 drhd->devices[i]->subordinate->number <= bus &&
b918c62e 669 drhd->devices[i]->subordinate->busn_res.end >= bus)
924b6231
DW		 670 return drhd->iommu;
671 }
c7151a8d
WH		 672
673 if (drhd->include_all)
674 return drhd->iommu;
675 }
676
677 return NULL;
678}
679
5331fe6f
WH		 680static void domain_flush_cache(struct dmar_domain *domain,
681 void *addr, int size)
682{
683 if (!domain->iommu_coherency)
684 clflush_cache_range(addr, size);
685}
686
ba395927
KA		 687/* Gets context entry for a given bus and devfn */
688static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
689 u8 bus, u8 devfn)
690{
691 struct root_entry *root;
692 struct context_entry *context;
693 unsigned long phy_addr;
694 unsigned long flags;
695
696 spin_lock_irqsave(&iommu->lock, flags);
697 root = &iommu->root_entry[bus];
698 context = get_context_addr_from_root(root);
699 if (!context) {
4c923d47
SS		 700 context = (struct context_entry *)
701 alloc_pgtable_page(iommu->node);
ba395927
KA		 702 if (!context) {
703 spin_unlock_irqrestore(&iommu->lock, flags);
704 return NULL;
705 }
5b6985ce 706 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 707 phy_addr = virt_to_phys((void *)context);
708 set_root_value(root, phy_addr);
709 set_root_present(root);
710 __iommu_flush_cache(iommu, root, sizeof(*root));
711 }
712 spin_unlock_irqrestore(&iommu->lock, flags);
713 return &context[devfn];
714}
715
716static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
717{
718 struct root_entry *root;
719 struct context_entry *context;
720 int ret;
721 unsigned long flags;
722
723 spin_lock_irqsave(&iommu->lock, flags);
724 root = &iommu->root_entry[bus];
725 context = get_context_addr_from_root(root);
726 if (!context) {
727 ret = 0;
728 goto out;
729 }
c07e7d21 730 ret = context_present(&context[devfn]);
ba395927
KA		 731out:
732 spin_unlock_irqrestore(&iommu->lock, flags);
733 return ret;
734}
735
736static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
737{
738 struct root_entry *root;
739 struct context_entry *context;
740 unsigned long flags;
741
742 spin_lock_irqsave(&iommu->lock, flags);
743 root = &iommu->root_entry[bus];
744 context = get_context_addr_from_root(root);
745 if (context) {
c07e7d21 746 context_clear_entry(&context[devfn]);
ba395927
KA		 747 __iommu_flush_cache(iommu, &context[devfn], \
748 sizeof(*context));
749 }
750 spin_unlock_irqrestore(&iommu->lock, flags);
751}
752
753static void free_context_table(struct intel_iommu *iommu)
754{
755 struct root_entry *root;
756 int i;
757 unsigned long flags;
758 struct context_entry *context;
759
760 spin_lock_irqsave(&iommu->lock, flags);
761 if (!iommu->root_entry) {
762 goto out;
763 }
764 for (i = 0; i < ROOT_ENTRY_NR; i++) {
765 root = &iommu->root_entry[i];
766 context = get_context_addr_from_root(root);
767 if (context)
768 free_pgtable_page(context);
769 }
770 free_pgtable_page(iommu->root_entry);
771 iommu->root_entry = NULL;
772out:
773 spin_unlock_irqrestore(&iommu->lock, flags);
774}
775
b026fd28 776static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
4399c8bf 777 unsigned long pfn, int target_level)
ba395927 778{
b026fd28 779 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 780 struct dma_pte *parent, *pte = NULL;
781 int level = agaw_to_level(domain->agaw);
4399c8bf 782 int offset;
ba395927
KA		 783
784 BUG_ON(!domain->pgd);
f9423606
JS		 785
786 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
787 /* Address beyond IOMMU's addressing capabilities. */
788 return NULL;
789
ba395927
KA		 790 parent = domain->pgd;
791
ba395927
KA		 792 while (level > 0) {
793 void *tmp_page;
794
b026fd28 795 offset = pfn_level_offset(pfn, level);
ba395927 796 pte = &parent[offset];
4399c8bf 797 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7
YS		 798 break;
799 if (level == target_level)
ba395927
KA		 800 break;
801
19c239ce 802 if (!dma_pte_present(pte)) {
c85994e4
DW		 803 uint64_t pteval;
804
4c923d47 805 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 806
206a73c1 807 if (!tmp_page)
ba395927 808 return NULL;
206a73c1 809
c85994e4 810 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 811 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 812 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
813 /* Someone else set it while we were thinking; use theirs. */
814 free_pgtable_page(tmp_page);
815 } else {
816 dma_pte_addr(pte);
817 domain_flush_cache(domain, pte, sizeof(*pte));
818 }
ba395927 819 }
19c239ce 820 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 821 level--;
822 }
823
ba395927
KA		 824 return pte;
825}
826
6dd9a7c7 827
ba395927 828/* return address's pte at specific level */
90dcfb5e
DW		 829static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
830 unsigned long pfn,
6dd9a7c7 831 int level, int *large_page)
ba395927
KA		 832{
833 struct dma_pte *parent, *pte = NULL;
834 int total = agaw_to_level(domain->agaw);
835 int offset;
836
837 parent = domain->pgd;
838 while (level <= total) {
90dcfb5e 839 offset = pfn_level_offset(pfn, total);
ba395927
KA		 840 pte = &parent[offset];
841 if (level == total)
842 return pte;
843
6dd9a7c7
YS		 844 if (!dma_pte_present(pte)) {
845 *large_page = total;
ba395927 846 break;
6dd9a7c7
YS		 847 }
848
849 if (pte->val & DMA_PTE_LARGE_PAGE) {
850 *large_page = total;
851 return pte;
852 }
853
19c239ce 854 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 855 total--;
856 }
857 return NULL;
858}
859
ba395927 860/* clear last level pte, a tlb flush should be followed */
292827cb 861static int dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 862 unsigned long start_pfn,
863 unsigned long last_pfn)
ba395927 864{
04b18e65 865 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 866 unsigned int large_page = 1;
310a5ab9 867 struct dma_pte *first_pte, *pte;
292827cb 868 int order;
66eae846 869
04b18e65 870 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 871 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 872 BUG_ON(start_pfn > last_pfn);
ba395927 873
04b18e65 874 /* we don't need lock here; nobody else touches the iova range */
59c36286 875 do {
6dd9a7c7
YS		 876 large_page = 1;
877 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 878 if (!pte) {
6dd9a7c7 879 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 880 continue;
881 }
6dd9a7c7 882 do {
310a5ab9 883 dma_clear_pte(pte);
6dd9a7c7 884 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 885 pte++;
75e6bf96
DW		 886 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
887
310a5ab9
DW		 888 domain_flush_cache(domain, first_pte,
889 (void *)pte - (void *)first_pte);
59c36286
DW		 890
891 } while (start_pfn && start_pfn <= last_pfn);
292827cb
AK		 892
893 order = (large_page - 1) * 9;
894 return order;
ba395927
KA		 895}
896
3269ee0b
AW		 897static void dma_pte_free_level(struct dmar_domain *domain, int level,
898 struct dma_pte *pte, unsigned long pfn,
899 unsigned long start_pfn, unsigned long last_pfn)
900{
901 pfn = max(start_pfn, pfn);
902 pte = &pte[pfn_level_offset(pfn, level)];
903
904 do {
905 unsigned long level_pfn;
906 struct dma_pte *level_pte;
907
908 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
909 goto next;
910
911 level_pfn = pfn & level_mask(level - 1);
912 level_pte = phys_to_virt(dma_pte_addr(pte));
913
914 if (level > 2)
915 dma_pte_free_level(domain, level - 1, level_pte,
916 level_pfn, start_pfn, last_pfn);
917
918 /* If range covers entire pagetable, free it */
919 if (!(start_pfn > level_pfn ||
920 last_pfn < level_pfn + level_size(level))) {
921 dma_clear_pte(pte);
922 domain_flush_cache(domain, pte, sizeof(*pte));
923 free_pgtable_page(level_pte);
924 }
925next:
926 pfn += level_size(level);
927 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
928}
929
ba395927
KA		 930/* free page table pages. last level pte should already be cleared */
931static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 932 unsigned long start_pfn,
933 unsigned long last_pfn)
ba395927 934{
6660c63a 935 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 936
6660c63a
DW		 937 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
938 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 939 BUG_ON(start_pfn > last_pfn);
ba395927 940
f3a0a52f 941 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 942 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
943 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 944
ba395927 945 /* free pgd */
d794dc9b 946 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 947 free_pgtable_page(domain->pgd);
948 domain->pgd = NULL;
949 }
950}
951
952/* iommu handling */
953static int iommu_alloc_root_entry(struct intel_iommu *iommu)
954{
955 struct root_entry *root;
956 unsigned long flags;
957
4c923d47 958 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 959 if (!root)
960 return -ENOMEM;
961
5b6985ce 962 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 963
964 spin_lock_irqsave(&iommu->lock, flags);
965 iommu->root_entry = root;
966 spin_unlock_irqrestore(&iommu->lock, flags);
967
968 return 0;
969}
970
ba395927
KA		 971static void iommu_set_root_entry(struct intel_iommu *iommu)
972{
973 void *addr;
c416daa9 974 u32 sts;
ba395927
KA		 975 unsigned long flag;
976
977 addr = iommu->root_entry;
978
1f5b3c3f 979 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 980 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
981
c416daa9 982 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 983
984 /* Make sure hardware complete it */
985 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 986 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 987
1f5b3c3f 988 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 989}
990
991static void iommu_flush_write_buffer(struct intel_iommu *iommu)
992{
993 u32 val;
994 unsigned long flag;
995
9af88143 996 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 997 return;
ba395927 998
1f5b3c3f 999 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1000 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1001
1002 /* Make sure hardware complete it */
1003 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1004 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1005
1f5b3c3f 1006 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1007}
1008
1009/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1010static void __iommu_flush_context(struct intel_iommu *iommu,
1011 u16 did, u16 source_id, u8 function_mask,
1012 u64 type)
ba395927
KA		 1013{
1014 u64 val = 0;
1015 unsigned long flag;
1016
ba395927
KA		 1017 switch (type) {
1018 case DMA_CCMD_GLOBAL_INVL:
1019 val = DMA_CCMD_GLOBAL_INVL;
1020 break;
1021 case DMA_CCMD_DOMAIN_INVL:
1022 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1023 break;
1024 case DMA_CCMD_DEVICE_INVL:
1025 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1026 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1027 break;
1028 default:
1029 BUG();
1030 }
1031 val |= DMA_CCMD_ICC;
1032
1f5b3c3f 1033 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1034 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1035
1036 /* Make sure hardware complete it */
1037 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1038 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1039
1f5b3c3f 1040 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1041}
1042
ba395927 1043/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1044static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1045 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1046{
1047 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1048 u64 val = 0, val_iva = 0;
1049 unsigned long flag;
1050
ba395927
KA		 1051 switch (type) {
1052 case DMA_TLB_GLOBAL_FLUSH:
1053 /* global flush doesn't need set IVA_REG */
1054 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1055 break;
1056 case DMA_TLB_DSI_FLUSH:
1057 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1058 break;
1059 case DMA_TLB_PSI_FLUSH:
1060 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1061 /* Note: always flush non-leaf currently */
1062 val_iva = size_order | addr;
1063 break;
1064 default:
1065 BUG();
1066 }
1067 /* Note: set drain read/write */
1068#if 0
1069 /*
1070 * This is probably to be super secure.. Looks like we can
1071 * ignore it without any impact.
1072 */
1073 if (cap_read_drain(iommu->cap))
1074 val |= DMA_TLB_READ_DRAIN;
1075#endif
1076 if (cap_write_drain(iommu->cap))
1077 val |= DMA_TLB_WRITE_DRAIN;
1078
1f5b3c3f 1079 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1080 /* Note: Only uses first TLB reg currently */
1081 if (val_iva)
1082 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1083 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1084
1085 /* Make sure hardware complete it */
1086 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1087 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1088
1f5b3c3f 1089 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1090
1091 /* check IOTLB invalidation granularity */
1092 if (DMA_TLB_IAIG(val) == 0)
1093 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1094 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1095 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1096 (unsigned long long)DMA_TLB_IIRG(type),
1097 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1098}
1099
93a23a72
YZ		 1100static struct device_domain_info *iommu_support_dev_iotlb(
1101 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1102{
1103 int found = 0;
1104 unsigned long flags;
1105 struct device_domain_info *info;
1106 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1107
1108 if (!ecap_dev_iotlb_support(iommu->ecap))
1109 return NULL;
1110
1111 if (!iommu->qi)
1112 return NULL;
1113
1114 spin_lock_irqsave(&device_domain_lock, flags);
1115 list_for_each_entry(info, &domain->devices, link)
1116 if (info->bus == bus && info->devfn == devfn) {
1117 found = 1;
1118 break;
1119 }
1120 spin_unlock_irqrestore(&device_domain_lock, flags);
1121
1122 if (!found || !info->dev)
1123 return NULL;
1124
1125 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1126 return NULL;
1127
1128 if (!dmar_find_matched_atsr_unit(info->dev))
1129 return NULL;
1130
1131 info->iommu = iommu;
1132
1133 return info;
1134}
1135
1136static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1137{
93a23a72
YZ		 1138 if (!info)
1139 return;
1140
1141 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1142}
1143
1144static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1145{
1146 if (!info->dev || !pci_ats_enabled(info->dev))
1147 return;
1148
1149 pci_disable_ats(info->dev);
1150}
1151
1152static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1153 u64 addr, unsigned mask)
1154{
1155 u16 sid, qdep;
1156 unsigned long flags;
1157 struct device_domain_info *info;
1158
1159 spin_lock_irqsave(&device_domain_lock, flags);
1160 list_for_each_entry(info, &domain->devices, link) {
1161 if (!info->dev || !pci_ats_enabled(info->dev))
1162 continue;
1163
1164 sid = info->bus << 8 | info->devfn;
1165 qdep = pci_ats_queue_depth(info->dev);
1166 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1167 }
1168 spin_unlock_irqrestore(&device_domain_lock, flags);
1169}
1170
1f0ef2aa 1171static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1172 unsigned long pfn, unsigned int pages, int map)
ba395927 1173{
9dd2fe89 1174 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1175 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1176
ba395927
KA		 1177 BUG_ON(pages == 0);
1178
ba395927 1179 /*
9dd2fe89
YZ		 1180 * Fallback to domain selective flush if no PSI support or the size is
1181 * too big.
ba395927
KA		 1182 * PSI requires page size to be 2 ^ x, and the base address is naturally
1183 * aligned to the size
1184 */
9dd2fe89
YZ		 1185 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1186 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1187 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1188 else
1189 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1190 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1191
1192 /*
82653633
NA		 1193 * In caching mode, changes of pages from non-present to present require
1194 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1195 */
82653633 1196 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1197 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1198}
1199
f8bab735 1200static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1201{
1202 u32 pmen;
1203 unsigned long flags;
1204
1f5b3c3f 1205 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1206 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1207 pmen &= ~DMA_PMEN_EPM;
1208 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1209
1210 /* wait for the protected region status bit to clear */
1211 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1212 readl, !(pmen & DMA_PMEN_PRS), pmen);
1213
1f5b3c3f 1214 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1215}
1216
ba395927
KA		 1217static int iommu_enable_translation(struct intel_iommu *iommu)
1218{
1219 u32 sts;
1220 unsigned long flags;
1221
1f5b3c3f 1222 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1223 iommu->gcmd |= DMA_GCMD_TE;
1224 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1225
1226 /* Make sure hardware complete it */
1227 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1228 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1229
1f5b3c3f 1230 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1231 return 0;
1232}
1233
1234static int iommu_disable_translation(struct intel_iommu *iommu)
1235{
1236 u32 sts;
1237 unsigned long flag;
1238
1f5b3c3f 1239 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1240 iommu->gcmd &= ~DMA_GCMD_TE;
1241 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1242
1243 /* Make sure hardware complete it */
1244 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1245 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1246
1f5b3c3f 1247 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1248 return 0;
1249}
1250
3460a6d9 1251
ba395927
KA		 1252static int iommu_init_domains(struct intel_iommu *iommu)
1253{
1254 unsigned long ndomains;
1255 unsigned long nlongs;
1256
1257 ndomains = cap_ndoms(iommu->cap);
68aeb968 1258 pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
680a7524 1259 ndomains);
ba395927
KA		 1260 nlongs = BITS_TO_LONGS(ndomains);
1261
94a91b50
DD		 1262 spin_lock_init(&iommu->lock);
1263
ba395927
KA		 1264 /* TBD: there might be 64K domains,
1265 * consider other allocation for future chip
1266 */
1267 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1268 if (!iommu->domain_ids) {
1269 printk(KERN_ERR "Allocating domain id array failed\n");
1270 return -ENOMEM;
1271 }
1272 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1273 GFP_KERNEL);
1274 if (!iommu->domains) {
1275 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1276 return -ENOMEM;
1277 }
1278
1279 /*
1280 * if Caching mode is set, then invalid translations are tagged
1281 * with domainid 0. Hence we need to pre-allocate it.
1282 */
1283 if (cap_caching_mode(iommu->cap))
1284 set_bit(0, iommu->domain_ids);
1285 return 0;
1286}
ba395927 1287
ba395927
KA		 1288
1289static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1290static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1291
1292void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1293{
1294 struct dmar_domain *domain;
1295 int i;
c7151a8d 1296 unsigned long flags;
ba395927 1297
94a91b50 1298 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1299 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1300 domain = iommu->domains[i];
1301 clear_bit(i, iommu->domain_ids);
1302
1303 spin_lock_irqsave(&domain->iommu_lock, flags);
1304 if (--domain->iommu_count == 0) {
1305 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1306 vm_domain_exit(domain);
1307 else
1308 domain_exit(domain);
1309 }
1310 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1311 }
ba395927
KA		 1312 }
1313
1314 if (iommu->gcmd & DMA_GCMD_TE)
1315 iommu_disable_translation(iommu);
1316
1317 if (iommu->irq) {
dced35ae 1318 irq_set_handler_data(iommu->irq, NULL);
ba395927
KA		 1319 /* This will mask the irq */
1320 free_irq(iommu->irq, iommu);
1321 destroy_irq(iommu->irq);
1322 }
1323
1324 kfree(iommu->domains);
1325 kfree(iommu->domain_ids);
1326
d9630fe9
WH		 1327 g_iommus[iommu->seq_id] = NULL;
1328
1329 /* if all iommus are freed, free g_iommus */
1330 for (i = 0; i < g_num_of_iommus; i++) {
1331 if (g_iommus[i])
1332 break;
1333 }
1334
1335 if (i == g_num_of_iommus)
1336 kfree(g_iommus);
1337
ba395927
KA		 1338 /* free context mapping */
1339 free_context_table(iommu);
ba395927
KA		 1340}
1341
2c2e2c38 1342static struct dmar_domain *alloc_domain(void)
ba395927 1343{
ba395927 1344 struct dmar_domain *domain;
ba395927
KA		 1345
1346 domain = alloc_domain_mem();
1347 if (!domain)
1348 return NULL;
1349
4c923d47 1350 domain->nid = -1;
1b198bb0 1351 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38
FY		 1352 domain->flags = 0;
1353
1354 return domain;
1355}
1356
1357static int iommu_attach_domain(struct dmar_domain *domain,
1358 struct intel_iommu *iommu)
1359{
1360 int num;
1361 unsigned long ndomains;
1362 unsigned long flags;
1363
ba395927
KA		 1364 ndomains = cap_ndoms(iommu->cap);
1365
1366 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1367
ba395927
KA		 1368 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1369 if (num >= ndomains) {
1370 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1371 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1372 return -ENOMEM;
ba395927
KA		 1373 }
1374
ba395927 1375 domain->id = num;
2c2e2c38 1376 set_bit(num, iommu->domain_ids);
1b198bb0 1377 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1378 iommu->domains[num] = domain;
1379 spin_unlock_irqrestore(&iommu->lock, flags);
1380
2c2e2c38 1381 return 0;
ba395927
KA		 1382}
1383
2c2e2c38
FY		 1384static void iommu_detach_domain(struct dmar_domain *domain,
1385 struct intel_iommu *iommu)
ba395927
KA		 1386{
1387 unsigned long flags;
2c2e2c38
FY		 1388 int num, ndomains;
1389 int found = 0;
ba395927 1390
8c11e798 1391 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1392 ndomains = cap_ndoms(iommu->cap);
a45946ab 1393 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1394 if (iommu->domains[num] == domain) {
1395 found = 1;
1396 break;
1397 }
2c2e2c38
FY		 1398 }
1399
1400 if (found) {
1401 clear_bit(num, iommu->domain_ids);
1b198bb0 1402 clear_bit(iommu->seq_id, domain->iommu_bmp);
2c2e2c38
FY		 1403 iommu->domains[num] = NULL;
1404 }
8c11e798 1405 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1406}
1407
1408static struct iova_domain reserved_iova_list;
8a443df4 1409static struct lock_class_key reserved_rbtree_key;
ba395927 1410
51a63e67 1411static int dmar_init_reserved_ranges(void)
ba395927
KA		 1412{
1413 struct pci_dev *pdev = NULL;
1414 struct iova *iova;
1415 int i;
ba395927 1416
f661197e 1417 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1418
8a443df4
MG		 1419 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1420 &reserved_rbtree_key);
1421
ba395927
KA		 1422 /* IOAPIC ranges shouldn't be accessed by DMA */
1423 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1424 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1425 if (!iova) {
ba395927 1426 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1427 return -ENODEV;
1428 }
ba395927
KA		 1429
1430 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1431 for_each_pci_dev(pdev) {
1432 struct resource *r;
1433
1434 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1435 r = &pdev->resource[i];
1436 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1437 continue;
1a4a4551
DW		 1438 iova = reserve_iova(&reserved_iova_list,
1439 IOVA_PFN(r->start),
1440 IOVA_PFN(r->end));
51a63e67 1441 if (!iova) {
ba395927 1442 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1443 return -ENODEV;
1444 }
ba395927
KA		 1445 }
1446 }
51a63e67 1447 return 0;
ba395927
KA		 1448}
1449
1450static void domain_reserve_special_ranges(struct dmar_domain *domain)
1451{
1452 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1453}
1454
1455static inline int guestwidth_to_adjustwidth(int gaw)
1456{
1457 int agaw;
1458 int r = (gaw - 12) % 9;
1459
1460 if (r == 0)
1461 agaw = gaw;
1462 else
1463 agaw = gaw + 9 - r;
1464 if (agaw > 64)
1465 agaw = 64;
1466 return agaw;
1467}
1468
1469static int domain_init(struct dmar_domain *domain, int guest_width)
1470{
1471 struct intel_iommu *iommu;
1472 int adjust_width, agaw;
1473 unsigned long sagaw;
1474
f661197e 1475 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1476 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1477
1478 domain_reserve_special_ranges(domain);
1479
1480 /* calculate AGAW */
8c11e798 1481 iommu = domain_get_iommu(domain);
ba395927
KA		 1482 if (guest_width > cap_mgaw(iommu->cap))
1483 guest_width = cap_mgaw(iommu->cap);
1484 domain->gaw = guest_width;
1485 adjust_width = guestwidth_to_adjustwidth(guest_width);
1486 agaw = width_to_agaw(adjust_width);
1487 sagaw = cap_sagaw(iommu->cap);
1488 if (!test_bit(agaw, &sagaw)) {
1489 /* hardware doesn't support it, choose a bigger one */
1490 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1491 agaw = find_next_bit(&sagaw, 5, agaw);
1492 if (agaw >= 5)
1493 return -ENODEV;
1494 }
1495 domain->agaw = agaw;
1496 INIT_LIST_HEAD(&domain->devices);
1497
8e604097
WH		 1498 if (ecap_coherent(iommu->ecap))
1499 domain->iommu_coherency = 1;
1500 else
1501 domain->iommu_coherency = 0;
1502
58c610bd
SY		 1503 if (ecap_sc_support(iommu->ecap))
1504 domain->iommu_snooping = 1;
1505 else
1506 domain->iommu_snooping = 0;
1507
6dd9a7c7 1508 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1509 domain->iommu_count = 1;
4c923d47 1510 domain->nid = iommu->node;
c7151a8d 1511
ba395927 1512 /* always allocate the top pgd */
4c923d47 1513 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1514 if (!domain->pgd)
1515 return -ENOMEM;
5b6985ce 1516 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1517 return 0;
1518}
1519
1520static void domain_exit(struct dmar_domain *domain)
1521{
2c2e2c38
FY		 1522 struct dmar_drhd_unit *drhd;
1523 struct intel_iommu *iommu;
ba395927
KA		 1524
1525 /* Domain 0 is reserved, so dont process it */
1526 if (!domain)
1527 return;
1528
7b668357
AW		 1529 /* Flush any lazy unmaps that may reference this domain */
1530 if (!intel_iommu_strict)
1531 flush_unmaps_timeout(0);
1532
ba395927
KA		 1533 domain_remove_dev_info(domain);
1534 /* destroy iovas */
1535 put_iova_domain(&domain->iovad);
ba395927
KA		 1536
1537 /* clear ptes */
595badf5 1538 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1539
1540 /* free page tables */
d794dc9b 1541 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1542
2c2e2c38 1543 for_each_active_iommu(iommu, drhd)
1b198bb0 1544 if (test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38
FY		 1545 iommu_detach_domain(domain, iommu);
1546
ba395927
KA		 1547 free_domain_mem(domain);
1548}
1549
4ed0d3e6
FY		 1550static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1551 u8 bus, u8 devfn, int translation)
ba395927
KA		 1552{
1553 struct context_entry *context;
ba395927 1554 unsigned long flags;
5331fe6f 1555 struct intel_iommu *iommu;
ea6606b0
WH		 1556 struct dma_pte *pgd;
1557 unsigned long num;
1558 unsigned long ndomains;
1559 int id;
1560 int agaw;
93a23a72 1561 struct device_domain_info *info = NULL;
ba395927
KA		 1562
1563 pr_debug("Set context mapping for %02x:%02x.%d\n",
1564 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1565
ba395927 1566 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1567 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1568 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1569
276dbf99 1570 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1571 if (!iommu)
1572 return -ENODEV;
1573
ba395927
KA		 1574 context = device_to_context_entry(iommu, bus, devfn);
1575 if (!context)
1576 return -ENOMEM;
1577 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1578 if (context_present(context)) {
ba395927
KA		 1579 spin_unlock_irqrestore(&iommu->lock, flags);
1580 return 0;
1581 }
1582
ea6606b0
WH		 1583 id = domain->id;
1584 pgd = domain->pgd;
1585
2c2e2c38
FY		 1586 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1587 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1588 int found = 0;
1589
1590 /* find an available domain id for this device in iommu */
1591 ndomains = cap_ndoms(iommu->cap);
a45946ab 1592 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1593 if (iommu->domains[num] == domain) {
1594 id = num;
1595 found = 1;
1596 break;
1597 }
ea6606b0
WH		 1598 }
1599
1600 if (found == 0) {
1601 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1602 if (num >= ndomains) {
1603 spin_unlock_irqrestore(&iommu->lock, flags);
1604 printk(KERN_ERR "IOMMU: no free domain ids\n");
1605 return -EFAULT;
1606 }
1607
1608 set_bit(num, iommu->domain_ids);
1609 iommu->domains[num] = domain;
1610 id = num;
1611 }
1612
1613 /* Skip top levels of page tables for
1614 * iommu which has less agaw than default.
1672af11 1615 * Unnecessary for PT mode.
ea6606b0 1616 */
1672af11
CW		 1617 if (translation != CONTEXT_TT_PASS_THROUGH) {
1618 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1619 pgd = phys_to_virt(dma_pte_addr(pgd));
1620 if (!dma_pte_present(pgd)) {
1621 spin_unlock_irqrestore(&iommu->lock, flags);
1622 return -ENOMEM;
1623 }
ea6606b0
WH		 1624 }
1625 }
1626 }
1627
1628 context_set_domain_id(context, id);
4ed0d3e6 1629
93a23a72
YZ		 1630 if (translation != CONTEXT_TT_PASS_THROUGH) {
1631 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1632 translation = info ? CONTEXT_TT_DEV_IOTLB :
1633 CONTEXT_TT_MULTI_LEVEL;
1634 }
4ed0d3e6
FY		 1635 /*
1636 * In pass through mode, AW must be programmed to indicate the largest
1637 * AGAW value supported by hardware. And ASR is ignored by hardware.
1638 */
93a23a72 1639 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1640 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1641 else {
1642 context_set_address_root(context, virt_to_phys(pgd));
1643 context_set_address_width(context, iommu->agaw);
1644 }
4ed0d3e6
FY		 1645
1646 context_set_translation_type(context, translation);
c07e7d21
MM		 1647 context_set_fault_enable(context);
1648 context_set_present(context);
5331fe6f 1649 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1650
4c25a2c1
DW		 1651 /*
1652 * It's a non-present to present mapping. If hardware doesn't cache
1653 * non-present entry we only need to flush the write-buffer. If the
1654 * _does_ cache non-present entries, then it does so in the special
1655 * domain #0, which we have to flush:
1656 */
1657 if (cap_caching_mode(iommu->cap)) {
1658 iommu->flush.flush_context(iommu, 0,
1659 (((u16)bus) << 8) | devfn,
1660 DMA_CCMD_MASK_NOBIT,
1661 DMA_CCMD_DEVICE_INVL);
82653633 1662 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1663 } else {
ba395927 1664 iommu_flush_write_buffer(iommu);
4c25a2c1 1665 }
93a23a72 1666 iommu_enable_dev_iotlb(info);
ba395927 1667 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1668
1669 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1670 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1671 domain->iommu_count++;
4c923d47
SS		 1672 if (domain->iommu_count == 1)
1673 domain->nid = iommu->node;
58c610bd 1674 domain_update_iommu_cap(domain);
c7151a8d
WH		 1675 }
1676 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1677 return 0;
1678}
1679
1680static int
4ed0d3e6
FY		 1681domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1682 int translation)
ba395927
KA		 1683{
1684 int ret;
1685 struct pci_dev *tmp, *parent;
1686
276dbf99 1687 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1688 pdev->bus->number, pdev->devfn,
1689 translation);
ba395927
KA		 1690 if (ret)
1691 return ret;
1692
1693 /* dependent device mapping */
1694 tmp = pci_find_upstream_pcie_bridge(pdev);
1695 if (!tmp)
1696 return 0;
1697 /* Secondary interface's bus number and devfn 0 */
1698 parent = pdev->bus->self;
1699 while (parent != tmp) {
276dbf99
DW		 1700 ret = domain_context_mapping_one(domain,
1701 pci_domain_nr(parent->bus),
1702 parent->bus->number,
4ed0d3e6 1703 parent->devfn, translation);
ba395927
KA		 1704 if (ret)
1705 return ret;
1706 parent = parent->bus->self;
1707 }
45e829ea 1708 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1709 return domain_context_mapping_one(domain,
276dbf99 1710 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1711 tmp->subordinate->number, 0,
1712 translation);
ba395927
KA		 1713 else /* this is a legacy PCI bridge */
1714 return domain_context_mapping_one(domain,
276dbf99
DW		 1715 pci_domain_nr(tmp->bus),
1716 tmp->bus->number,
4ed0d3e6
FY		 1717 tmp->devfn,
1718 translation);
ba395927
KA		 1719}
1720
5331fe6f 1721static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1722{
1723 int ret;
1724 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1725 struct intel_iommu *iommu;
1726
276dbf99
DW		 1727 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1728 pdev->devfn);
5331fe6f
WH		 1729 if (!iommu)
1730 return -ENODEV;
ba395927 1731
276dbf99 1732 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1733 if (!ret)
1734 return ret;
1735 /* dependent device mapping */
1736 tmp = pci_find_upstream_pcie_bridge(pdev);
1737 if (!tmp)
1738 return ret;
1739 /* Secondary interface's bus number and devfn 0 */
1740 parent = pdev->bus->self;
1741 while (parent != tmp) {
8c11e798 1742 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1743 parent->devfn);
ba395927
KA		 1744 if (!ret)
1745 return ret;
1746 parent = parent->bus->self;
1747 }
5f4d91a1 1748 if (pci_is_pcie(tmp))
276dbf99
DW		 1749 return device_context_mapped(iommu, tmp->subordinate->number,
1750 0);
ba395927 1751 else
276dbf99
DW		 1752 return device_context_mapped(iommu, tmp->bus->number,
1753 tmp->devfn);
ba395927
KA		 1754}
1755
f532959b
FY		 1756/* Returns a number of VTD pages, but aligned to MM page size */
1757static inline unsigned long aligned_nrpages(unsigned long host_addr,
1758 size_t size)
1759{
1760 host_addr &= ~PAGE_MASK;
1761 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1762}
1763
6dd9a7c7
YS		 1764/* Return largest possible superpage level for a given mapping */
1765static inline int hardware_largepage_caps(struct dmar_domain *domain,
1766 unsigned long iov_pfn,
1767 unsigned long phy_pfn,
1768 unsigned long pages)
1769{
1770 int support, level = 1;
1771 unsigned long pfnmerge;
1772
1773 support = domain->iommu_superpage;
1774
1775 /* To use a large page, the virtual *and* physical addresses
1776 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1777 of them will mean we have to use smaller pages. So just
1778 merge them and check both at once. */
1779 pfnmerge = iov_pfn | phy_pfn;
1780
1781 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1782 pages >>= VTD_STRIDE_SHIFT;
1783 if (!pages)
1784 break;
1785 pfnmerge >>= VTD_STRIDE_SHIFT;
1786 level++;
1787 support--;
1788 }
1789 return level;
1790}
1791
9051aa02
DW		 1792static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1793 struct scatterlist *sg, unsigned long phys_pfn,
1794 unsigned long nr_pages, int prot)
e1605495
DW		 1795{
1796 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1797 phys_addr_t uninitialized_var(pteval);
e1605495 1798 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1799 unsigned long sg_res;
6dd9a7c7
YS		 1800 unsigned int largepage_lvl = 0;
1801 unsigned long lvl_pages = 0;
e1605495
DW		 1802
1803 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1804
1805 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1806 return -EINVAL;
1807
1808 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1809
9051aa02
DW		 1810 if (sg)
1811 sg_res = 0;
1812 else {
1813 sg_res = nr_pages + 1;
1814 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1815 }
1816
6dd9a7c7 1817 while (nr_pages > 0) {
c85994e4
DW		 1818 uint64_t tmp;
1819
e1605495 1820 if (!sg_res) {
f532959b 1821 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1822 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1823 sg->dma_length = sg->length;
1824 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1825 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1826 }
6dd9a7c7 1827
e1605495 1828 if (!pte) {
6dd9a7c7
YS		 1829 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1830
1831 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1832 if (!pte)
1833 return -ENOMEM;
6dd9a7c7 1834 /* It is large page*/
6491d4d0 1835 if (largepage_lvl > 1) {
6dd9a7c7 1836 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1837 /* Ensure that old small page tables are removed to make room
1838 for superpage, if they exist. */
1839 dma_pte_clear_range(domain, iov_pfn,
1840 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1841 dma_pte_free_pagetable(domain, iov_pfn,
1842 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1843 } else {
6dd9a7c7 1844 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1845 }
6dd9a7c7 1846
e1605495
DW		 1847 }
1848 /* We don't need lock here, nobody else
1849 * touches the iova range
1850 */
7766a3fb 1851 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1852 if (tmp) {
1bf20f0d 1853 static int dumps = 5;
c85994e4
DW		 1854 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1855 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1856 if (dumps) {
1857 dumps--;
1858 debug_dma_dump_mappings(NULL);
1859 }
1860 WARN_ON(1);
1861 }
6dd9a7c7
YS		 1862
1863 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1864
1865 BUG_ON(nr_pages < lvl_pages);
1866 BUG_ON(sg_res < lvl_pages);
1867
1868 nr_pages -= lvl_pages;
1869 iov_pfn += lvl_pages;
1870 phys_pfn += lvl_pages;
1871 pteval += lvl_pages * VTD_PAGE_SIZE;
1872 sg_res -= lvl_pages;
1873
1874 /* If the next PTE would be the first in a new page, then we
1875 need to flush the cache on the entries we've just written.
1876 And then we'll need to recalculate 'pte', so clear it and
1877 let it get set again in the if (!pte) block above.
1878
1879 If we're done (!nr_pages) we need to flush the cache too.
1880
1881 Also if we've been setting superpages, we may need to
1882 recalculate 'pte' and switch back to smaller pages for the
1883 end of the mapping, if the trailing size is not enough to
1884 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1885 pte++;
6dd9a7c7
YS		 1886 if (!nr_pages || first_pte_in_page(pte) ||
1887 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1888 domain_flush_cache(domain, first_pte,
1889 (void *)pte - (void *)first_pte);
1890 pte = NULL;
1891 }
6dd9a7c7
YS		 1892
1893 if (!sg_res && nr_pages)
e1605495
DW		 1894 sg = sg_next(sg);
1895 }
1896 return 0;
1897}
1898
9051aa02
DW		 1899static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1900 struct scatterlist *sg, unsigned long nr_pages,
1901 int prot)
ba395927 1902{
9051aa02
DW		 1903 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1904}
6f6a00e4 1905
9051aa02
DW		 1906static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1907 unsigned long phys_pfn, unsigned long nr_pages,
1908 int prot)
1909{
1910 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1911}
1912
c7151a8d 1913static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1914{
c7151a8d
WH		 1915 if (!iommu)
1916 return;
8c11e798
WH		 1917
1918 clear_context_table(iommu, bus, devfn);
1919 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1920 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1921 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1922}
1923
109b9b04
DW		 1924static inline void unlink_domain_info(struct device_domain_info *info)
1925{
1926 assert_spin_locked(&device_domain_lock);
1927 list_del(&info->link);
1928 list_del(&info->global);
1929 if (info->dev)
1930 info->dev->dev.archdata.iommu = NULL;
1931}
1932
ba395927
KA		 1933static void domain_remove_dev_info(struct dmar_domain *domain)
1934{
1935 struct device_domain_info *info;
1936 unsigned long flags;
c7151a8d 1937 struct intel_iommu *iommu;
ba395927
KA		 1938
1939 spin_lock_irqsave(&device_domain_lock, flags);
1940 while (!list_empty(&domain->devices)) {
1941 info = list_entry(domain->devices.next,
1942 struct device_domain_info, link);
109b9b04 1943 unlink_domain_info(info);
ba395927
KA		 1944 spin_unlock_irqrestore(&device_domain_lock, flags);
1945
93a23a72 1946 iommu_disable_dev_iotlb(info);
276dbf99 1947 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1948 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1949 free_devinfo_mem(info);
1950
1951 spin_lock_irqsave(&device_domain_lock, flags);
1952 }
1953 spin_unlock_irqrestore(&device_domain_lock, flags);
1954}
1955
1956/*
1957 * find_domain
358dd8ac 1958 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1959 */
38717946 1960static struct dmar_domain *
ba395927
KA		 1961find_domain(struct pci_dev *pdev)
1962{
1963 struct device_domain_info *info;
1964
1965 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1966 info = pdev->dev.archdata.iommu;
ba395927
KA		 1967 if (info)
1968 return info->domain;
1969 return NULL;
1970}
1971
ba395927
KA		 1972/* domain is initialized */
1973static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1974{
1975 struct dmar_domain *domain, *found = NULL;
1976 struct intel_iommu *iommu;
1977 struct dmar_drhd_unit *drhd;
1978 struct device_domain_info *info, *tmp;
1979 struct pci_dev *dev_tmp;
1980 unsigned long flags;
1981 int bus = 0, devfn = 0;
276dbf99 1982 int segment;
2c2e2c38 1983 int ret;
ba395927
KA		 1984
1985 domain = find_domain(pdev);
1986 if (domain)
1987 return domain;
1988
276dbf99
DW		 1989 segment = pci_domain_nr(pdev->bus);
1990
ba395927
KA		 1991 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1992 if (dev_tmp) {
5f4d91a1 1993 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1994 bus = dev_tmp->subordinate->number;
1995 devfn = 0;
1996 } else {
1997 bus = dev_tmp->bus->number;
1998 devfn = dev_tmp->devfn;
1999 }
2000 spin_lock_irqsave(&device_domain_lock, flags);
2001 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 2002 if (info->segment == segment &&
2003 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 2004 found = info->domain;
2005 break;
2006 }
2007 }
2008 spin_unlock_irqrestore(&device_domain_lock, flags);
2009 /* pcie-pci bridge already has a domain, uses it */
2010 if (found) {
2011 domain = found;
2012 goto found_domain;
2013 }
2014 }
2015
2c2e2c38
FY		 2016 domain = alloc_domain();
2017 if (!domain)
2018 goto error;
2019
ba395927
KA		 2020 /* Allocate new domain for the device */
2021 drhd = dmar_find_matched_drhd_unit(pdev);
2022 if (!drhd) {
2023 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2024 pci_name(pdev));
d2900bd6 2025 free_domain_mem(domain);
ba395927
KA		 2026 return NULL;
2027 }
2028 iommu = drhd->iommu;
2029
2c2e2c38
FY		 2030 ret = iommu_attach_domain(domain, iommu);
2031 if (ret) {
2fe9723d 2032 free_domain_mem(domain);
ba395927 2033 goto error;
2c2e2c38 2034 }
ba395927
KA		 2035
2036 if (domain_init(domain, gaw)) {
2037 domain_exit(domain);
2038 goto error;
2039 }
2040
2041 /* register pcie-to-pci device */
2042 if (dev_tmp) {
2043 info = alloc_devinfo_mem();
2044 if (!info) {
2045 domain_exit(domain);
2046 goto error;
2047 }
276dbf99 2048 info->segment = segment;
ba395927
KA		 2049 info->bus = bus;
2050 info->devfn = devfn;
2051 info->dev = NULL;
2052 info->domain = domain;
2053 /* This domain is shared by devices under p2p bridge */
3b5410e7 2054 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 2055
2056 /* pcie-to-pci bridge already has a domain, uses it */
2057 found = NULL;
2058 spin_lock_irqsave(&device_domain_lock, flags);
2059 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2060 if (tmp->segment == segment &&
2061 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2062 found = tmp->domain;
2063 break;
2064 }
2065 }
2066 if (found) {
00dfff77 2067 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2068 free_devinfo_mem(info);
2069 domain_exit(domain);
2070 domain = found;
2071 } else {
2072 list_add(&info->link, &domain->devices);
2073 list_add(&info->global, &device_domain_list);
00dfff77 2074 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2075 }
ba395927
KA		 2076 }
2077
2078found_domain:
2079 info = alloc_devinfo_mem();
2080 if (!info)
2081 goto error;
276dbf99 2082 info->segment = segment;
ba395927
KA		 2083 info->bus = pdev->bus->number;
2084 info->devfn = pdev->devfn;
2085 info->dev = pdev;
2086 info->domain = domain;
2087 spin_lock_irqsave(&device_domain_lock, flags);
2088 /* somebody is fast */
2089 found = find_domain(pdev);
2090 if (found != NULL) {
2091 spin_unlock_irqrestore(&device_domain_lock, flags);
2092 if (found != domain) {
2093 domain_exit(domain);
2094 domain = found;
2095 }
2096 free_devinfo_mem(info);
2097 return domain;
2098 }
2099 list_add(&info->link, &domain->devices);
2100 list_add(&info->global, &device_domain_list);
358dd8ac 2101 pdev->dev.archdata.iommu = info;
ba395927
KA		 2102 spin_unlock_irqrestore(&device_domain_lock, flags);
2103 return domain;
2104error:
2105 /* recheck it here, maybe others set it */
2106 return find_domain(pdev);
2107}
2108
2c2e2c38 2109static int iommu_identity_mapping;
e0fc7e0b
DW		 2110#define IDENTMAP_ALL 1
2111#define IDENTMAP_GFX 2
2112#define IDENTMAP_AZALIA 4
2c2e2c38 2113
b213203e
DW		 2114static int iommu_domain_identity_map(struct dmar_domain *domain,
2115 unsigned long long start,
2116 unsigned long long end)
ba395927 2117{
c5395d5c
DW		 2118 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2119 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2120
2121 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2122 dma_to_mm_pfn(last_vpfn))) {
ba395927 2123 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2124 return -ENOMEM;
ba395927
KA		 2125 }
2126
c5395d5c
DW		 2127 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2128 start, end, domain->id);
ba395927
KA		 2129 /*
2130 * RMRR range might have overlap with physical memory range,
2131 * clear it first
2132 */
c5395d5c 2133 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2134
c5395d5c
DW		 2135 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2136 last_vpfn - first_vpfn + 1,
61df7443 2137 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2138}
2139
2140static int iommu_prepare_identity_map(struct pci_dev *pdev,
2141 unsigned long long start,
2142 unsigned long long end)
2143{
2144 struct dmar_domain *domain;
2145 int ret;
2146
c7ab48d2 2147 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2148 if (!domain)
2149 return -ENOMEM;
2150
19943b0e
DW		 2151 /* For _hardware_ passthrough, don't bother. But for software
2152 passthrough, we do it anyway -- it may indicate a memory
2153 range which is reserved in E820, so which didn't get set
2154 up to start with in si_domain */
2155 if (domain == si_domain && hw_pass_through) {
2156 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2157 pci_name(pdev), start, end);
2158 return 0;
2159 }
2160
2161 printk(KERN_INFO
2162 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2163 pci_name(pdev), start, end);
2ff729f5 2164
5595b528
DW		 2165 if (end < start) {
2166 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2167 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2168 dmi_get_system_info(DMI_BIOS_VENDOR),
2169 dmi_get_system_info(DMI_BIOS_VERSION),
2170 dmi_get_system_info(DMI_PRODUCT_VERSION));
2171 ret = -EIO;
2172 goto error;
2173 }
2174
2ff729f5
DW		 2175 if (end >> agaw_to_width(domain->agaw)) {
2176 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2177 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2178 agaw_to_width(domain->agaw),
2179 dmi_get_system_info(DMI_BIOS_VENDOR),
2180 dmi_get_system_info(DMI_BIOS_VERSION),
2181 dmi_get_system_info(DMI_PRODUCT_VERSION));
2182 ret = -EIO;
2183 goto error;
2184 }
19943b0e 2185
b213203e 2186 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2187 if (ret)
2188 goto error;
2189
2190 /* context entry init */
4ed0d3e6 2191 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2192 if (ret)
2193 goto error;
2194
2195 return 0;
2196
2197 error:
ba395927
KA		 2198 domain_exit(domain);
2199 return ret;
ba395927
KA		 2200}
2201
2202static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2203 struct pci_dev *pdev)
2204{
358dd8ac 2205 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2206 return 0;
2207 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2208 rmrr->end_address);
ba395927
KA		 2209}
2210
d3f13810 2211#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2212static inline void iommu_prepare_isa(void)
2213{
2214 struct pci_dev *pdev;
2215 int ret;
2216
2217 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2218 if (!pdev)
2219 return;
2220
c7ab48d2 2221 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2222 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2223
2224 if (ret)
c7ab48d2
DW		 2225 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2226 "floppy might not work\n");
49a0429e
KA		 2227
2228}
2229#else
2230static inline void iommu_prepare_isa(void)
2231{
2232 return;
2233}
d3f13810 2234#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2235
2c2e2c38 2236static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2237
071e1374 2238static int __init si_domain_init(int hw)
2c2e2c38
FY		 2239{
2240 struct dmar_drhd_unit *drhd;
2241 struct intel_iommu *iommu;
c7ab48d2 2242 int nid, ret = 0;
2c2e2c38
FY		 2243
2244 si_domain = alloc_domain();
2245 if (!si_domain)
2246 return -EFAULT;
2247
c7ab48d2 2248 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2249
2250 for_each_active_iommu(iommu, drhd) {
2251 ret = iommu_attach_domain(si_domain, iommu);
2252 if (ret) {
2253 domain_exit(si_domain);
2254 return -EFAULT;
2255 }
2256 }
2257
2258 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2259 domain_exit(si_domain);
2260 return -EFAULT;
2261 }
2262
2263 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2264
19943b0e
DW		 2265 if (hw)
2266 return 0;
2267
c7ab48d2 2268 for_each_online_node(nid) {
5dfe8660
TH		 2269 unsigned long start_pfn, end_pfn;
2270 int i;
2271
2272 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2273 ret = iommu_domain_identity_map(si_domain,
2274 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2275 if (ret)
2276 return ret;
2277 }
c7ab48d2
DW		 2278 }
2279
2c2e2c38
FY		 2280 return 0;
2281}
2282
2283static void domain_remove_one_dev_info(struct dmar_domain *domain,
2284 struct pci_dev *pdev);
2285static int identity_mapping(struct pci_dev *pdev)
2286{
2287 struct device_domain_info *info;
2288
2289 if (likely(!iommu_identity_mapping))
2290 return 0;
2291
cb452a40
MT		 2292 info = pdev->dev.archdata.iommu;
2293 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2294 return (info->domain == si_domain);
2c2e2c38 2295
2c2e2c38
FY		 2296 return 0;
2297}
2298
2299static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2300 struct pci_dev *pdev,
2301 int translation)
2c2e2c38
FY		 2302{
2303 struct device_domain_info *info;
2304 unsigned long flags;
5fe60f4e 2305 int ret;
2c2e2c38
FY		 2306
2307 info = alloc_devinfo_mem();
2308 if (!info)
2309 return -ENOMEM;
2310
2311 info->segment = pci_domain_nr(pdev->bus);
2312 info->bus = pdev->bus->number;
2313 info->devfn = pdev->devfn;
2314 info->dev = pdev;
2315 info->domain = domain;
2316
2317 spin_lock_irqsave(&device_domain_lock, flags);
2318 list_add(&info->link, &domain->devices);
2319 list_add(&info->global, &device_domain_list);
2320 pdev->dev.archdata.iommu = info;
2321 spin_unlock_irqrestore(&device_domain_lock, flags);
2322
e2ad23d0
DW		 2323 ret = domain_context_mapping(domain, pdev, translation);
2324 if (ret) {
2325 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2326 unlink_domain_info(info);
e2ad23d0
DW		 2327 spin_unlock_irqrestore(&device_domain_lock, flags);
2328 free_devinfo_mem(info);
2329 return ret;
2330 }
2331
2c2e2c38
FY		 2332 return 0;
2333}
2334
ea2447f7
TM		 2335static bool device_has_rmrr(struct pci_dev *dev)
2336{
2337 struct dmar_rmrr_unit *rmrr;
2338 int i;
2339
2340 for_each_rmrr_units(rmrr) {
2341 for (i = 0; i < rmrr->devices_cnt; i++) {
2342 /*
2343 * Return TRUE if this RMRR contains the device that
2344 * is passed in.
2345 */
2346 if (rmrr->devices[i] == dev)
2347 return true;
2348 }
2349 }
2350 return false;
2351}
2352
6941af28
DW		 2353static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2354{
ea2447f7
TM		 2355
2356 /*
2357 * We want to prevent any device associated with an RMRR from
2358 * getting placed into the SI Domain. This is done because
2359 * problems exist when devices are moved in and out of domains
2360 * and their respective RMRR info is lost. We exempt USB devices
2361 * from this process due to their usage of RMRRs that are known
2362 * to not be needed after BIOS hand-off to OS.
2363 */
2364 if (device_has_rmrr(pdev) &&
2365 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2366 return 0;
2367
e0fc7e0b
DW		 2368 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2369 return 1;
2370
2371 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2372 return 1;
2373
2374 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2375 return 0;
6941af28 2376
3dfc813d
DW		 2377 /*
2378 * We want to start off with all devices in the 1:1 domain, and
2379 * take them out later if we find they can't access all of memory.
2380 *
2381 * However, we can't do this for PCI devices behind bridges,
2382 * because all PCI devices behind the same bridge will end up
2383 * with the same source-id on their transactions.
2384 *
2385 * Practically speaking, we can't change things around for these
2386 * devices at run-time, because we can't be sure there'll be no
2387 * DMA transactions in flight for any of their siblings.
2388 *
2389 * So PCI devices (unless they're on the root bus) as well as
2390 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2391 * the 1:1 domain, just in _case_ one of their siblings turns out
2392 * not to be able to map all of memory.
2393 */
5f4d91a1 2394 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2395 if (!pci_is_root_bus(pdev->bus))
2396 return 0;
2397 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2398 return 0;
62f87c0e 2399 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2400 return 0;
2401
2402 /*
2403 * At boot time, we don't yet know if devices will be 64-bit capable.
2404 * Assume that they will -- if they turn out not to be, then we can
2405 * take them out of the 1:1 domain later.
2406 */
8fcc5372
CW		 2407 if (!startup) {
2408 /*
2409 * If the device's dma_mask is less than the system's memory
2410 * size then this is not a candidate for identity mapping.
2411 */
2412 u64 dma_mask = pdev->dma_mask;
2413
2414 if (pdev->dev.coherent_dma_mask &&
2415 pdev->dev.coherent_dma_mask < dma_mask)
2416 dma_mask = pdev->dev.coherent_dma_mask;
2417
2418 return dma_mask >= dma_get_required_mask(&pdev->dev);
2419 }
6941af28
DW		 2420
2421 return 1;
2422}
2423
071e1374 2424static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2425{
2c2e2c38
FY		 2426 struct pci_dev *pdev = NULL;
2427 int ret;
2428
19943b0e 2429 ret = si_domain_init(hw);
2c2e2c38
FY		 2430 if (ret)
2431 return -EFAULT;
2432
2c2e2c38 2433 for_each_pci_dev(pdev) {
6941af28 2434 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2435 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2436 hw ? CONTEXT_TT_PASS_THROUGH :
2437 CONTEXT_TT_MULTI_LEVEL);
2438 if (ret) {
2439 /* device not associated with an iommu */
2440 if (ret == -ENODEV)
2441 continue;
62edf5dc 2442 return ret;
eae460b6
MT		 2443 }
2444 pr_info("IOMMU: %s identity mapping for device %s\n",
2445 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2446 }
2c2e2c38
FY		 2447 }
2448
2449 return 0;
2450}
2451
b779260b 2452static int __init init_dmars(void)
ba395927
KA		 2453{
2454 struct dmar_drhd_unit *drhd;
2455 struct dmar_rmrr_unit *rmrr;
2456 struct pci_dev *pdev;
2457 struct intel_iommu *iommu;
9d783ba0 2458 int i, ret;
2c2e2c38 2459
ba395927
KA		 2460 /*
2461 * for each drhd
2462 * allocate root
2463 * initialize and program root entry to not present
2464 * endfor
2465 */
2466 for_each_drhd_unit(drhd) {
5e0d2a6f 2467 /*
2468 * lock not needed as this is only incremented in the single
2469 * threaded kernel __init code path all other access are read
2470 * only
2471 */
1b198bb0
MT		 2472 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2473 g_num_of_iommus++;
2474 continue;
2475 }
2476 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2477 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2478 }
2479
d9630fe9
WH		 2480 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2481 GFP_KERNEL);
2482 if (!g_iommus) {
2483 printk(KERN_ERR "Allocating global iommu array failed\n");
2484 ret = -ENOMEM;
2485 goto error;
2486 }
2487
80b20dd8 2488 deferred_flush = kzalloc(g_num_of_iommus *
2489 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2490 if (!deferred_flush) {
5e0d2a6f 2491 ret = -ENOMEM;
2492 goto error;
2493 }
2494
5e0d2a6f 2495 for_each_drhd_unit(drhd) {
2496 if (drhd->ignored)
2497 continue;
1886e8a9
SS		 2498
2499 iommu = drhd->iommu;
d9630fe9 2500 g_iommus[iommu->seq_id] = iommu;
ba395927 2501
e61d98d8
SS		 2502 ret = iommu_init_domains(iommu);
2503 if (ret)
2504 goto error;
2505
ba395927
KA		 2506 /*
2507 * TBD:
2508 * we could share the same root & context tables
25985edc 2509 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2510 */
2511 ret = iommu_alloc_root_entry(iommu);
2512 if (ret) {
2513 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2514 goto error;
2515 }
4ed0d3e6 2516 if (!ecap_pass_through(iommu->ecap))
19943b0e 2517 hw_pass_through = 0;
ba395927
KA		 2518 }
2519
1531a6a6
SS		 2520 /*
2521 * Start from the sane iommu hardware state.
2522 */
a77b67d4
YS		 2523 for_each_drhd_unit(drhd) {
2524 if (drhd->ignored)
2525 continue;
2526
2527 iommu = drhd->iommu;
1531a6a6
SS		 2528
2529 /*
2530 * If the queued invalidation is already initialized by us
2531 * (for example, while enabling interrupt-remapping) then
2532 * we got the things already rolling from a sane state.
2533 */
2534 if (iommu->qi)
2535 continue;
2536
2537 /*
2538 * Clear any previous faults.
2539 */
2540 dmar_fault(-1, iommu);
2541 /*
2542 * Disable queued invalidation if supported and already enabled
2543 * before OS handover.
2544 */
2545 dmar_disable_qi(iommu);
2546 }
2547
2548 for_each_drhd_unit(drhd) {
2549 if (drhd->ignored)
2550 continue;
2551
2552 iommu = drhd->iommu;
2553
a77b67d4
YS		 2554 if (dmar_enable_qi(iommu)) {
2555 /*
2556 * Queued Invalidate not enabled, use Register Based
2557 * Invalidate
2558 */
2559 iommu->flush.flush_context = __iommu_flush_context;
2560 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2561 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2562 "invalidation\n",
680a7524 2563 iommu->seq_id,
b4e0f9eb 2564 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2565 } else {
2566 iommu->flush.flush_context = qi_flush_context;
2567 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2568 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2569 "invalidation\n",
680a7524 2570 iommu->seq_id,
b4e0f9eb 2571 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2572 }
2573 }
2574
19943b0e 2575 if (iommu_pass_through)
e0fc7e0b
DW		 2576 iommu_identity_mapping |= IDENTMAP_ALL;
2577
d3f13810 2578#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2579 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2580#endif
e0fc7e0b
DW		 2581
2582 check_tylersburg_isoch();
2583
ba395927 2584 /*
19943b0e
DW		 2585 * If pass through is not set or not enabled, setup context entries for
2586 * identity mappings for rmrr, gfx, and isa and may fall back to static
2587 * identity mapping if iommu_identity_mapping is set.
ba395927 2588 */
19943b0e
DW		 2589 if (iommu_identity_mapping) {
2590 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2591 if (ret) {
19943b0e
DW		 2592 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2593 goto error;
ba395927
KA		 2594 }
2595 }
ba395927 2596 /*
19943b0e
DW		 2597 * For each rmrr
2598 * for each dev attached to rmrr
2599 * do
2600 * locate drhd for dev, alloc domain for dev
2601 * allocate free domain
2602 * allocate page table entries for rmrr
2603 * if context not allocated for bus
2604 * allocate and init context
2605 * set present in root table for this bus
2606 * init context with domain, translation etc
2607 * endfor
2608 * endfor
ba395927 2609 */
19943b0e
DW		 2610 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2611 for_each_rmrr_units(rmrr) {
2612 for (i = 0; i < rmrr->devices_cnt; i++) {
2613 pdev = rmrr->devices[i];
2614 /*
2615 * some BIOS lists non-exist devices in DMAR
2616 * table.
2617 */
2618 if (!pdev)
2619 continue;
2620 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2621 if (ret)
2622 printk(KERN_ERR
2623 "IOMMU: mapping reserved region failed\n");
ba395927 2624 }
4ed0d3e6 2625 }
49a0429e 2626
19943b0e
DW		 2627 iommu_prepare_isa();
2628
ba395927
KA		 2629 /*
2630 * for each drhd
2631 * enable fault log
2632 * global invalidate context cache
2633 * global invalidate iotlb
2634 * enable translation
2635 */
2636 for_each_drhd_unit(drhd) {
51a63e67
JC		 2637 if (drhd->ignored) {
2638 /*
2639 * we always have to disable PMRs or DMA may fail on
2640 * this device
2641 */
2642 if (force_on)
2643 iommu_disable_protect_mem_regions(drhd->iommu);
ba395927 2644 continue;
51a63e67 2645 }
ba395927 2646 iommu = drhd->iommu;
ba395927
KA		 2647
2648 iommu_flush_write_buffer(iommu);
2649
3460a6d9
KA		 2650 ret = dmar_set_interrupt(iommu);
2651 if (ret)
2652 goto error;
2653
ba395927
KA		 2654 iommu_set_root_entry(iommu);
2655
4c25a2c1 2656 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2657 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2658
ba395927
KA		 2659 ret = iommu_enable_translation(iommu);
2660 if (ret)
2661 goto error;
b94996c9
DW		 2662
2663 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2664 }
2665
2666 return 0;
2667error:
2668 for_each_drhd_unit(drhd) {
2669 if (drhd->ignored)
2670 continue;
2671 iommu = drhd->iommu;
2672 free_iommu(iommu);
2673 }
d9630fe9 2674 kfree(g_iommus);
ba395927
KA		 2675 return ret;
2676}
2677
5a5e02a6 2678/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2679static struct iova *intel_alloc_iova(struct device *dev,
2680 struct dmar_domain *domain,
2681 unsigned long nrpages, uint64_t dma_mask)
ba395927 2682{
ba395927 2683 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2684 struct iova *iova = NULL;
ba395927 2685
875764de
DW		 2686 /* Restrict dma_mask to the width that the iommu can handle */
2687 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2688
2689 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2690 /*
2691 * First try to allocate an io virtual address in
284901a9 2692 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2693 * from higher range
ba395927 2694 */
875764de
DW		 2695 iova = alloc_iova(&domain->iovad, nrpages,
2696 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2697 if (iova)
2698 return iova;
2699 }
2700 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2701 if (unlikely(!iova)) {
2702 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2703 nrpages, pci_name(pdev));
f76aec76
KA		 2704 return NULL;
2705 }
2706
2707 return iova;
2708}
2709
147202aa 2710static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2711{
2712 struct dmar_domain *domain;
2713 int ret;
2714
2715 domain = get_domain_for_dev(pdev,
2716 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2717 if (!domain) {
2718 printk(KERN_ERR
2719 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2720 return NULL;
ba395927
KA		 2721 }
2722
2723 /* make sure context mapping is ok */
5331fe6f 2724 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2725 ret = domain_context_mapping(domain, pdev,
2726 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2727 if (ret) {
2728 printk(KERN_ERR
2729 "Domain context map for %s failed",
2730 pci_name(pdev));
4fe05bbc 2731 return NULL;
f76aec76 2732 }
ba395927
KA		 2733 }
2734
f76aec76
KA		 2735 return domain;
2736}
2737
147202aa
DW		 2738static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2739{
2740 struct device_domain_info *info;
2741
2742 /* No lock here, assumes no domain exit in normal case */
2743 info = dev->dev.archdata.iommu;
2744 if (likely(info))
2745 return info->domain;
2746
2747 return __get_valid_domain_for_dev(dev);
2748}
2749
2c2e2c38
FY		 2750static int iommu_dummy(struct pci_dev *pdev)
2751{
2752 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2753}
2754
2755/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2756static int iommu_no_mapping(struct device *dev)
2c2e2c38 2757{
73676832 2758 struct pci_dev *pdev;
2c2e2c38
FY		 2759 int found;
2760
dbad0864 2761 if (unlikely(!dev_is_pci(dev)))
73676832
DW		 2762 return 1;
2763
2764 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2765 if (iommu_dummy(pdev))
2766 return 1;
2767
2c2e2c38 2768 if (!iommu_identity_mapping)
1e4c64c4 2769 return 0;
2c2e2c38
FY		 2770
2771 found = identity_mapping(pdev);
2772 if (found) {
6941af28 2773 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2774 return 1;
2775 else {
2776 /*
2777 * 32 bit DMA is removed from si_domain and fall back
2778 * to non-identity mapping.
2779 */
2780 domain_remove_one_dev_info(si_domain, pdev);
2781 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2782 pci_name(pdev));
2783 return 0;
2784 }
2785 } else {
2786 /*
2787 * In case of a detached 64 bit DMA device from vm, the device
2788 * is put into si_domain for identity mapping.
2789 */
6941af28 2790 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2791 int ret;
5fe60f4e
DW		 2792 ret = domain_add_dev_info(si_domain, pdev,
2793 hw_pass_through ?
2794 CONTEXT_TT_PASS_THROUGH :
2795 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2796 if (!ret) {
2797 printk(KERN_INFO "64bit %s uses identity mapping\n",
2798 pci_name(pdev));
2799 return 1;
2800 }
2801 }
2802 }
2803
1e4c64c4 2804 return 0;
2c2e2c38
FY		 2805}
2806
bb9e6d65
FT		 2807static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2808 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2809{
2810 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2811 struct dmar_domain *domain;
5b6985ce 2812 phys_addr_t start_paddr;
f76aec76
KA		 2813 struct iova *iova;
2814 int prot = 0;
6865f0d1 2815 int ret;
8c11e798 2816 struct intel_iommu *iommu;
33041ec0 2817 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2818
2819 BUG_ON(dir == DMA_NONE);
2c2e2c38 2820
73676832 2821 if (iommu_no_mapping(hwdev))
6865f0d1 2822 return paddr;
f76aec76
KA		 2823
2824 domain = get_valid_domain_for_dev(pdev);
2825 if (!domain)
2826 return 0;
2827
8c11e798 2828 iommu = domain_get_iommu(domain);
88cb6a74 2829 size = aligned_nrpages(paddr, size);
f76aec76 2830
c681d0ba 2831 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2832 if (!iova)
2833 goto error;
2834
ba395927
KA		 2835 /*
2836 * Check if DMAR supports zero-length reads on write only
2837 * mappings..
2838 */
2839 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2840 !cap_zlr(iommu->cap))
ba395927
KA		 2841 prot |= DMA_PTE_READ;
2842 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2843 prot |= DMA_PTE_WRITE;
2844 /*
6865f0d1 2845 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2846 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2847 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2848 * is not a big problem
2849 */
0ab36de2 2850 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2851 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2852 if (ret)
2853 goto error;
2854
1f0ef2aa
DW		 2855 /* it's a non-present to present mapping. Only flush if caching mode */
2856 if (cap_caching_mode(iommu->cap))
82653633 2857 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2858 else
8c11e798 2859 iommu_flush_write_buffer(iommu);
f76aec76 2860
03d6a246
DW		 2861 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2862 start_paddr += paddr & ~PAGE_MASK;
2863 return start_paddr;
ba395927 2864
ba395927 2865error:
f76aec76
KA		 2866 if (iova)
2867 __free_iova(&domain->iovad, iova);
4cf2e75d 2868 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2869 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2870 return 0;
2871}
2872
ffbbef5c
FT		 2873static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2874 unsigned long offset, size_t size,
2875 enum dma_data_direction dir,
2876 struct dma_attrs *attrs)
bb9e6d65 2877{
ffbbef5c
FT		 2878 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2879 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2880}
2881
5e0d2a6f 2882static void flush_unmaps(void)
2883{
80b20dd8 2884 int i, j;
5e0d2a6f 2885
5e0d2a6f 2886 timer_on = 0;
2887
2888 /* just flush them all */
2889 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2890 struct intel_iommu *iommu = g_iommus[i];
2891 if (!iommu)
2892 continue;
c42d9f32 2893
9dd2fe89
YZ		 2894 if (!deferred_flush[i].next)
2895 continue;
2896
78d5f0f5
NA		 2897 /* In caching mode, global flushes turn emulation expensive */
2898 if (!cap_caching_mode(iommu->cap))
2899 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2900 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2901 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2902 unsigned long mask;
2903 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2904 struct dmar_domain *domain = deferred_flush[i].domain[j];
2905
2906 /* On real hardware multiple invalidations are expensive */
2907 if (cap_caching_mode(iommu->cap))
2908 iommu_flush_iotlb_psi(iommu, domain->id,
2909 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2910 else {
2911 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2912 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2913 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2914 }
93a23a72 2915 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2916 }
9dd2fe89 2917 deferred_flush[i].next = 0;
5e0d2a6f 2918 }
2919
5e0d2a6f 2920 list_size = 0;
5e0d2a6f 2921}
2922
2923static void flush_unmaps_timeout(unsigned long data)
2924{
80b20dd8 2925 unsigned long flags;
2926
2927 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2928 flush_unmaps();
80b20dd8 2929 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2930}
2931
2932static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2933{
2934 unsigned long flags;
80b20dd8 2935 int next, iommu_id;
8c11e798 2936 struct intel_iommu *iommu;
5e0d2a6f 2937
2938 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2939 if (list_size == HIGH_WATER_MARK)
2940 flush_unmaps();
2941
8c11e798
WH		 2942 iommu = domain_get_iommu(dom);
2943 iommu_id = iommu->seq_id;
c42d9f32 2944
80b20dd8 2945 next = deferred_flush[iommu_id].next;
2946 deferred_flush[iommu_id].domain[next] = dom;
2947 deferred_flush[iommu_id].iova[next] = iova;
2948 deferred_flush[iommu_id].next++;
5e0d2a6f 2949
2950 if (!timer_on) {
2951 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2952 timer_on = 1;
2953 }
2954 list_size++;
2955 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2956}
2957
ffbbef5c
FT		 2958static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2959 size_t size, enum dma_data_direction dir,
2960 struct dma_attrs *attrs)
ba395927 2961{
ba395927 2962 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2963 struct dmar_domain *domain;
d794dc9b 2964 unsigned long start_pfn, last_pfn;
ba395927 2965 struct iova *iova;
8c11e798 2966 struct intel_iommu *iommu;
ba395927 2967
73676832 2968 if (iommu_no_mapping(dev))
f76aec76 2969 return;
2c2e2c38 2970
ba395927
KA		 2971 domain = find_domain(pdev);
2972 BUG_ON(!domain);
2973
8c11e798
WH		 2974 iommu = domain_get_iommu(domain);
2975
ba395927 2976 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2977 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2978 (unsigned long long)dev_addr))
ba395927 2979 return;
ba395927 2980
d794dc9b
DW		 2981 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2982 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2983
d794dc9b
DW		 2984 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2985 pci_name(pdev), start_pfn, last_pfn);
ba395927 2986
f76aec76 2987 /* clear the whole page */
d794dc9b
DW		 2988 dma_pte_clear_range(domain, start_pfn, last_pfn);
2989
f76aec76 2990 /* free page tables */
d794dc9b
DW		 2991 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2992
5e0d2a6f 2993 if (intel_iommu_strict) {
03d6a246 2994 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2995 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2996 /* free iova */
2997 __free_iova(&domain->iovad, iova);
2998 } else {
2999 add_unmap(domain, iova);
3000 /*
3001 * queue up the release of the unmap to save the 1/6th of the
3002 * cpu used up by the iotlb flush operation...
3003 */
5e0d2a6f 3004 }
ba395927
KA		 3005}
3006
d7ab5c46 3007static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 3008 dma_addr_t *dma_handle, gfp_t flags,
3009 struct dma_attrs *attrs)
ba395927
KA		 3010{
3011 void *vaddr;
3012 int order;
3013
5b6985ce 3014 size = PAGE_ALIGN(size);
ba395927 3015 order = get_order(size);
e8bb910d
AW		 3016
3017 if (!iommu_no_mapping(hwdev))
3018 flags &= ~(GFP_DMA | GFP_DMA32);
3019 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3020 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3021 flags |= GFP_DMA;
3022 else
3023 flags |= GFP_DMA32;
3024 }
ba395927
KA		 3025
3026 vaddr = (void *)__get_free_pages(flags, order);
3027 if (!vaddr)
3028 return NULL;
3029 memset(vaddr, 0, size);
3030
bb9e6d65
FT		 3031 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3032 DMA_BIDIRECTIONAL,
3033 hwdev->coherent_dma_mask);
ba395927
KA		 3034 if (*dma_handle)
3035 return vaddr;
3036 free_pages((unsigned long)vaddr, order);
3037 return NULL;
3038}
3039
d7ab5c46 3040static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 3041 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3042{
3043 int order;
3044
5b6985ce 3045 size = PAGE_ALIGN(size);
ba395927
KA		 3046 order = get_order(size);
3047
0db9b7ae 3048 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3049 free_pages((unsigned long)vaddr, order);
3050}
3051
d7ab5c46
FT		 3052static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3053 int nelems, enum dma_data_direction dir,
3054 struct dma_attrs *attrs)
ba395927 3055{
ba395927
KA		 3056 struct pci_dev *pdev = to_pci_dev(hwdev);
3057 struct dmar_domain *domain;
d794dc9b 3058 unsigned long start_pfn, last_pfn;
f76aec76 3059 struct iova *iova;
8c11e798 3060 struct intel_iommu *iommu;
ba395927 3061
73676832 3062 if (iommu_no_mapping(hwdev))
ba395927
KA		 3063 return;
3064
3065 domain = find_domain(pdev);
8c11e798
WH		 3066 BUG_ON(!domain);
3067
3068 iommu = domain_get_iommu(domain);
ba395927 3069
c03ab37c 3070 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3071 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3072 (unsigned long long)sglist[0].dma_address))
f76aec76 3073 return;
f76aec76 3074
d794dc9b
DW		 3075 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3076 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 3077
3078 /* clear the whole page */
d794dc9b
DW		 3079 dma_pte_clear_range(domain, start_pfn, last_pfn);
3080
f76aec76 3081 /* free page tables */
d794dc9b 3082 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 3083
acea0018
DW		 3084 if (intel_iommu_strict) {
3085 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 3086 last_pfn - start_pfn + 1, 0);
acea0018
DW		 3087 /* free iova */
3088 __free_iova(&domain->iovad, iova);
3089 } else {
3090 add_unmap(domain, iova);
3091 /*
3092 * queue up the release of the unmap to save the 1/6th of the
3093 * cpu used up by the iotlb flush operation...
3094 */
3095 }
ba395927
KA		 3096}
3097
ba395927 3098static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3099 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3100{
3101 int i;
c03ab37c 3102 struct scatterlist *sg;
ba395927 3103
c03ab37c 3104 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3105 BUG_ON(!sg_page(sg));
4cf2e75d 3106 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3107 sg->dma_length = sg->length;
ba395927
KA		 3108 }
3109 return nelems;
3110}
3111
d7ab5c46
FT		 3112static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3113 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3114{
ba395927 3115 int i;
ba395927
KA		 3116 struct pci_dev *pdev = to_pci_dev(hwdev);
3117 struct dmar_domain *domain;
f76aec76
KA		 3118 size_t size = 0;
3119 int prot = 0;
f76aec76
KA		 3120 struct iova *iova = NULL;
3121 int ret;
c03ab37c 3122 struct scatterlist *sg;
b536d24d 3123 unsigned long start_vpfn;
8c11e798 3124 struct intel_iommu *iommu;
ba395927
KA		 3125
3126 BUG_ON(dir == DMA_NONE);
73676832 3127 if (iommu_no_mapping(hwdev))
c03ab37c 3128 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3129
f76aec76
KA		 3130 domain = get_valid_domain_for_dev(pdev);
3131 if (!domain)
3132 return 0;
3133
8c11e798
WH		 3134 iommu = domain_get_iommu(domain);
3135
b536d24d 3136 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3137 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3138
5a5e02a6
DW		 3139 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3140 pdev->dma_mask);
f76aec76 3141 if (!iova) {
c03ab37c 3142 sglist->dma_length = 0;
f76aec76
KA		 3143 return 0;
3144 }
3145
3146 /*
3147 * Check if DMAR supports zero-length reads on write only
3148 * mappings..
3149 */
3150 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3151 !cap_zlr(iommu->cap))
f76aec76
KA		 3152 prot |= DMA_PTE_READ;
3153 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3154 prot |= DMA_PTE_WRITE;
3155
b536d24d 3156 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3157
f532959b 3158 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3159 if (unlikely(ret)) {
3160 /* clear the page */
3161 dma_pte_clear_range(domain, start_vpfn,
3162 start_vpfn + size - 1);
3163 /* free page tables */
3164 dma_pte_free_pagetable(domain, start_vpfn,
3165 start_vpfn + size - 1);
3166 /* free iova */
3167 __free_iova(&domain->iovad, iova);
3168 return 0;
ba395927
KA		 3169 }
3170
1f0ef2aa
DW		 3171 /* it's a non-present to present mapping. Only flush if caching mode */
3172 if (cap_caching_mode(iommu->cap))
82653633 3173 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3174 else
8c11e798 3175 iommu_flush_write_buffer(iommu);
1f0ef2aa 3176
ba395927
KA		 3177 return nelems;
3178}
3179
dfb805e8
FT		 3180static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3181{
3182 return !dma_addr;
3183}
3184
160c1d8e 3185struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3186 .alloc = intel_alloc_coherent,
3187 .free = intel_free_coherent,
ba395927
KA		 3188 .map_sg = intel_map_sg,
3189 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3190 .map_page = intel_map_page,
3191 .unmap_page = intel_unmap_page,
dfb805e8 3192 .mapping_error = intel_mapping_error,
ba395927
KA		 3193};
3194
3195static inline int iommu_domain_cache_init(void)
3196{
3197 int ret = 0;
3198
3199 iommu_domain_cache = kmem_cache_create("iommu_domain",
3200 sizeof(struct dmar_domain),
3201 0,
3202 SLAB_HWCACHE_ALIGN,
3203
3204 NULL);
3205 if (!iommu_domain_cache) {
3206 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3207 ret = -ENOMEM;
3208 }
3209
3210 return ret;
3211}
3212
3213static inline int iommu_devinfo_cache_init(void)
3214{
3215 int ret = 0;
3216
3217 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3218 sizeof(struct device_domain_info),
3219 0,
3220 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3221 NULL);
3222 if (!iommu_devinfo_cache) {
3223 printk(KERN_ERR "Couldn't create devinfo cache\n");
3224 ret = -ENOMEM;
3225 }
3226
3227 return ret;
3228}
3229
3230static inline int iommu_iova_cache_init(void)
3231{
3232 int ret = 0;
3233
3234 iommu_iova_cache = kmem_cache_create("iommu_iova",
3235 sizeof(struct iova),
3236 0,
3237 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3238 NULL);
3239 if (!iommu_iova_cache) {
3240 printk(KERN_ERR "Couldn't create iova cache\n");
3241 ret = -ENOMEM;
3242 }
3243
3244 return ret;
3245}
3246
3247static int __init iommu_init_mempool(void)
3248{
3249 int ret;
3250 ret = iommu_iova_cache_init();
3251 if (ret)
3252 return ret;
3253
3254 ret = iommu_domain_cache_init();
3255 if (ret)
3256 goto domain_error;
3257
3258 ret = iommu_devinfo_cache_init();
3259 if (!ret)
3260 return ret;
3261
3262 kmem_cache_destroy(iommu_domain_cache);
3263domain_error:
3264 kmem_cache_destroy(iommu_iova_cache);
3265
3266 return -ENOMEM;
3267}
3268
3269static void __init iommu_exit_mempool(void)
3270{
3271 kmem_cache_destroy(iommu_devinfo_cache);
3272 kmem_cache_destroy(iommu_domain_cache);
3273 kmem_cache_destroy(iommu_iova_cache);
3274
3275}
3276
556ab45f
DW		 3277static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3278{
3279 struct dmar_drhd_unit *drhd;
3280 u32 vtbar;
3281 int rc;
3282
3283 /* We know that this device on this chipset has its own IOMMU.
3284 * If we find it under a different IOMMU, then the BIOS is lying
3285 * to us. Hope that the IOMMU for this device is actually
3286 * disabled, and it needs no translation...
3287 */
3288 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3289 if (rc) {
3290 /* "can't" happen */
3291 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3292 return;
3293 }
3294 vtbar &= 0xffff0000;
3295
3296 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3297 drhd = dmar_find_matched_drhd_unit(pdev);
3298 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3299 TAINT_FIRMWARE_WORKAROUND,
3300 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3301 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3302}
3303DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3304
ba395927
KA		 3305static void __init init_no_remapping_devices(void)
3306{
3307 struct dmar_drhd_unit *drhd;
3308
3309 for_each_drhd_unit(drhd) {
3310 if (!drhd->include_all) {
3311 int i;
3312 for (i = 0; i < drhd->devices_cnt; i++)
3313 if (drhd->devices[i] != NULL)
3314 break;
3315 /* ignore DMAR unit if no pci devices exist */
3316 if (i == drhd->devices_cnt)
3317 drhd->ignored = 1;
3318 }
3319 }
3320
ba395927
KA		 3321 for_each_drhd_unit(drhd) {
3322 int i;
3323 if (drhd->ignored || drhd->include_all)
3324 continue;
3325
3326 for (i = 0; i < drhd->devices_cnt; i++)
3327 if (drhd->devices[i] &&
c0771df8 3328 !IS_GFX_DEVICE(drhd->devices[i]))
ba395927
KA		 3329 break;
3330
3331 if (i < drhd->devices_cnt)
3332 continue;
3333
c0771df8
DW		 3334 /* This IOMMU has *only* gfx devices. Either bypass it or
3335 set the gfx_mapped flag, as appropriate */
3336 if (dmar_map_gfx) {
3337 intel_iommu_gfx_mapped = 1;
3338 } else {
3339 drhd->ignored = 1;
3340 for (i = 0; i < drhd->devices_cnt; i++) {
3341 if (!drhd->devices[i])
3342 continue;
3343 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3344 }
ba395927
KA		 3345 }
3346 }
3347}
3348
f59c7b69
FY		 3349#ifdef CONFIG_SUSPEND
3350static int init_iommu_hw(void)
3351{
3352 struct dmar_drhd_unit *drhd;
3353 struct intel_iommu *iommu = NULL;
3354
3355 for_each_active_iommu(iommu, drhd)
3356 if (iommu->qi)
3357 dmar_reenable_qi(iommu);
3358
b779260b
JC		 3359 for_each_iommu(iommu, drhd) {
3360 if (drhd->ignored) {
3361 /*
3362 * we always have to disable PMRs or DMA may fail on
3363 * this device
3364 */
3365 if (force_on)
3366 iommu_disable_protect_mem_regions(iommu);
3367 continue;
3368 }
3369
f59c7b69
FY		 3370 iommu_flush_write_buffer(iommu);
3371
3372 iommu_set_root_entry(iommu);
3373
3374 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3375 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3376 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3377 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3378 if (iommu_enable_translation(iommu))
3379 return 1;
b94996c9 3380 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3381 }
3382
3383 return 0;
3384}
3385
3386static void iommu_flush_all(void)
3387{
3388 struct dmar_drhd_unit *drhd;
3389 struct intel_iommu *iommu;
3390
3391 for_each_active_iommu(iommu, drhd) {
3392 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3393 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3394 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3395 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3396 }
3397}
3398
134fac3f 3399static int iommu_suspend(void)
f59c7b69
FY		 3400{
3401 struct dmar_drhd_unit *drhd;
3402 struct intel_iommu *iommu = NULL;
3403 unsigned long flag;
3404
3405 for_each_active_iommu(iommu, drhd) {
3406 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3407 GFP_ATOMIC);
3408 if (!iommu->iommu_state)
3409 goto nomem;
3410 }
3411
3412 iommu_flush_all();
3413
3414 for_each_active_iommu(iommu, drhd) {
3415 iommu_disable_translation(iommu);
3416
1f5b3c3f 3417 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3418
3419 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3420 readl(iommu->reg + DMAR_FECTL_REG);
3421 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3422 readl(iommu->reg + DMAR_FEDATA_REG);
3423 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3424 readl(iommu->reg + DMAR_FEADDR_REG);
3425 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3426 readl(iommu->reg + DMAR_FEUADDR_REG);
3427
1f5b3c3f 3428 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3429 }
3430 return 0;
3431
3432nomem:
3433 for_each_active_iommu(iommu, drhd)
3434 kfree(iommu->iommu_state);
3435
3436 return -ENOMEM;
3437}
3438
134fac3f 3439static void iommu_resume(void)
f59c7b69
FY		 3440{
3441 struct dmar_drhd_unit *drhd;
3442 struct intel_iommu *iommu = NULL;
3443 unsigned long flag;
3444
3445 if (init_iommu_hw()) {
b779260b
JC		 3446 if (force_on)
3447 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3448 else
3449 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3450 return;
f59c7b69
FY		 3451 }
3452
3453 for_each_active_iommu(iommu, drhd) {
3454
1f5b3c3f 3455 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3456
3457 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3458 iommu->reg + DMAR_FECTL_REG);
3459 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3460 iommu->reg + DMAR_FEDATA_REG);
3461 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3462 iommu->reg + DMAR_FEADDR_REG);
3463 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3464 iommu->reg + DMAR_FEUADDR_REG);
3465
1f5b3c3f 3466 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3467 }
3468
3469 for_each_active_iommu(iommu, drhd)
3470 kfree(iommu->iommu_state);
f59c7b69
FY		 3471}
3472
134fac3f 3473static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3474 .resume = iommu_resume,
3475 .suspend = iommu_suspend,
3476};
3477
134fac3f 3478static void __init init_iommu_pm_ops(void)
f59c7b69 3479{
134fac3f 3480 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3481}
3482
3483#else
99592ba4 3484static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3485#endif /* CONFIG_PM */
3486
318fe7df
SS		 3487LIST_HEAD(dmar_rmrr_units);
3488
3489static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3490{
3491 list_add(&rmrr->list, &dmar_rmrr_units);
3492}
3493
3494
3495int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3496{
3497 struct acpi_dmar_reserved_memory *rmrr;
3498 struct dmar_rmrr_unit *rmrru;
3499
3500 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3501 if (!rmrru)
3502 return -ENOMEM;
3503
3504 rmrru->hdr = header;
3505 rmrr = (struct acpi_dmar_reserved_memory *)header;
3506 rmrru->base_address = rmrr->base_address;
3507 rmrru->end_address = rmrr->end_address;
3508
3509 dmar_register_rmrr_unit(rmrru);
3510 return 0;
3511}
3512
3513static int __init
3514rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3515{
3516 struct acpi_dmar_reserved_memory *rmrr;
3517 int ret;
3518
3519 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3520 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3521 ((void *)rmrr) + rmrr->header.length,
3522 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3523
3524 if (ret || (rmrru->devices_cnt == 0)) {
3525 list_del(&rmrru->list);
3526 kfree(rmrru);
3527 }
3528 return ret;
3529}
3530
3531static LIST_HEAD(dmar_atsr_units);
3532
3533int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3534{
3535 struct acpi_dmar_atsr *atsr;
3536 struct dmar_atsr_unit *atsru;
3537
3538 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3539 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3540 if (!atsru)
3541 return -ENOMEM;
3542
3543 atsru->hdr = hdr;
3544 atsru->include_all = atsr->flags & 0x1;
3545
3546 list_add(&atsru->list, &dmar_atsr_units);
3547
3548 return 0;
3549}
3550
3551static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3552{
3553 int rc;
3554 struct acpi_dmar_atsr *atsr;
3555
3556 if (atsru->include_all)
3557 return 0;
3558
3559 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3560 rc = dmar_parse_dev_scope((void *)(atsr + 1),
3561 (void *)atsr + atsr->header.length,
3562 &atsru->devices_cnt, &atsru->devices,
3563 atsr->segment);
3564 if (rc || !atsru->devices_cnt) {
3565 list_del(&atsru->list);
3566 kfree(atsru);
3567 }
3568
3569 return rc;
3570}
3571
3572int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3573{
3574 int i;
3575 struct pci_bus *bus;
3576 struct acpi_dmar_atsr *atsr;
3577 struct dmar_atsr_unit *atsru;
3578
3579 dev = pci_physfn(dev);
3580
3581 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3582 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3583 if (atsr->segment == pci_domain_nr(dev->bus))
3584 goto found;
3585 }
3586
3587 return 0;
3588
3589found:
3590 for (bus = dev->bus; bus; bus = bus->parent) {
3591 struct pci_dev *bridge = bus->self;
3592
3593 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3594 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df
SS		 3595 return 0;
3596
62f87c0e 3597 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
318fe7df
SS		 3598 for (i = 0; i < atsru->devices_cnt; i++)
3599 if (atsru->devices[i] == bridge)
3600 return 1;
3601 break;
3602 }
3603 }
3604
3605 if (atsru->include_all)
3606 return 1;
3607
3608 return 0;
3609}
3610
c8f369ab 3611int __init dmar_parse_rmrr_atsr_dev(void)
318fe7df
SS		 3612{
3613 struct dmar_rmrr_unit *rmrr, *rmrr_n;
3614 struct dmar_atsr_unit *atsr, *atsr_n;
3615 int ret = 0;
3616
3617 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3618 ret = rmrr_parse_dev(rmrr);
3619 if (ret)
3620 return ret;
3621 }
3622
3623 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3624 ret = atsr_parse_dev(atsr);
3625 if (ret)
3626 return ret;
3627 }
3628
3629 return ret;
3630}
3631
99dcaded
FY		 3632/*
3633 * Here we only respond to action of unbound device from driver.
3634 *
3635 * Added device is not attached to its DMAR domain here yet. That will happen
3636 * when mapping the device to iova.
3637 */
3638static int device_notifier(struct notifier_block *nb,
3639 unsigned long action, void *data)
3640{
3641 struct device *dev = data;
3642 struct pci_dev *pdev = to_pci_dev(dev);
3643 struct dmar_domain *domain;
3644
44cd613c
DW		 3645 if (iommu_no_mapping(dev))
3646 return 0;
3647
99dcaded
FY		 3648 domain = find_domain(pdev);
3649 if (!domain)
3650 return 0;
3651
a97590e5 3652 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3653 domain_remove_one_dev_info(domain, pdev);
3654
a97590e5
AW		 3655 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3656 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3657 list_empty(&domain->devices))
3658 domain_exit(domain);
3659 }
3660
99dcaded
FY		 3661 return 0;
3662}
3663
3664static struct notifier_block device_nb = {
3665 .notifier_call = device_notifier,
3666};
3667
ba395927
KA		 3668int __init intel_iommu_init(void)
3669{
3670 int ret = 0;
3a93c841 3671 struct dmar_drhd_unit *drhd;
ba395927 3672
a59b50e9
JC		 3673 /* VT-d is required for a TXT/tboot launch, so enforce that */
3674 force_on = tboot_force_iommu();
3675
3676 if (dmar_table_init()) {
3677 if (force_on)
3678 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3679 return -ENODEV;
a59b50e9 3680 }
ba395927 3681
3a93c841
TI		 3682 /*
3683 * Disable translation if already enabled prior to OS handover.
3684 */
3685 for_each_drhd_unit(drhd) {
3686 struct intel_iommu *iommu;
3687
3688 if (drhd->ignored)
3689 continue;
3690
3691 iommu = drhd->iommu;
3692 if (iommu->gcmd & DMA_GCMD_TE)
3693 iommu_disable_translation(iommu);
3694 }
3695
c2c7286a 3696 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3697 if (force_on)
3698 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3699 return -ENODEV;
a59b50e9 3700 }
1886e8a9 3701
75f1cdf1 3702 if (no_iommu || dmar_disabled)
2ae21010
SS		 3703 return -ENODEV;
3704
51a63e67
JC		 3705 if (iommu_init_mempool()) {
3706 if (force_on)
3707 panic("tboot: Failed to initialize iommu memory\n");
3708 return -ENODEV;
3709 }
3710
318fe7df
SS		 3711 if (list_empty(&dmar_rmrr_units))
3712 printk(KERN_INFO "DMAR: No RMRR found\n");
3713
3714 if (list_empty(&dmar_atsr_units))
3715 printk(KERN_INFO "DMAR: No ATSR found\n");
3716
51a63e67
JC		 3717 if (dmar_init_reserved_ranges()) {
3718 if (force_on)
3719 panic("tboot: Failed to reserve iommu ranges\n");
3720 return -ENODEV;
3721 }
ba395927
KA		 3722
3723 init_no_remapping_devices();
3724
b779260b 3725 ret = init_dmars();
ba395927 3726 if (ret) {
a59b50e9
JC		 3727 if (force_on)
3728 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3729 printk(KERN_ERR "IOMMU: dmar init failed\n");
3730 put_iova_domain(&reserved_iova_list);
3731 iommu_exit_mempool();
3732 return ret;
3733 }
3734 printk(KERN_INFO
3735 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3736
5e0d2a6f 3737 init_timer(&unmap_timer);
75f1cdf1
FT		 3738#ifdef CONFIG_SWIOTLB
3739 swiotlb = 0;
3740#endif
19943b0e 3741 dma_ops = &intel_dma_ops;
4ed0d3e6 3742
134fac3f 3743 init_iommu_pm_ops();
a8bcbb0d 3744
4236d97d 3745 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
a8bcbb0d 3746
99dcaded
FY		 3747 bus_register_notifier(&pci_bus_type, &device_nb);
3748
8bc1f85c
ED		 3749 intel_iommu_enabled = 1;
3750
ba395927
KA		 3751 return 0;
3752}
e820482c 3753
3199aa6b
HW		 3754static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3755 struct pci_dev *pdev)
3756{
3757 struct pci_dev *tmp, *parent;
3758
3759 if (!iommu || !pdev)
3760 return;
3761
3762 /* dependent device detach */
3763 tmp = pci_find_upstream_pcie_bridge(pdev);
3764 /* Secondary interface's bus number and devfn 0 */
3765 if (tmp) {
3766 parent = pdev->bus->self;
3767 while (parent != tmp) {
3768 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3769 parent->devfn);
3199aa6b
HW		 3770 parent = parent->bus->self;
3771 }
45e829ea 3772 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3773 iommu_detach_dev(iommu,
3774 tmp->subordinate->number, 0);
3775 else /* this is a legacy PCI bridge */
276dbf99
DW		 3776 iommu_detach_dev(iommu, tmp->bus->number,
3777 tmp->devfn);
3199aa6b
HW		 3778 }
3779}
3780
2c2e2c38 3781static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3782 struct pci_dev *pdev)
3783{
bca2b916 3784 struct device_domain_info *info, *tmp;
c7151a8d
WH		 3785 struct intel_iommu *iommu;
3786 unsigned long flags;
3787 int found = 0;
c7151a8d 3788
276dbf99
DW		 3789 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3790 pdev->devfn);
c7151a8d
WH		 3791 if (!iommu)
3792 return;
3793
3794 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 3795 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
8519dc44
MH		 3796 if (info->segment == pci_domain_nr(pdev->bus) &&
3797 info->bus == pdev->bus->number &&
c7151a8d 3798 info->devfn == pdev->devfn) {
109b9b04 3799 unlink_domain_info(info);
c7151a8d
WH		 3800 spin_unlock_irqrestore(&device_domain_lock, flags);
3801
93a23a72 3802 iommu_disable_dev_iotlb(info);
c7151a8d 3803 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3804 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3805 free_devinfo_mem(info);
3806
3807 spin_lock_irqsave(&device_domain_lock, flags);
3808
3809 if (found)
3810 break;
3811 else
3812 continue;
3813 }
3814
3815 /* if there is no other devices under the same iommu
3816 * owned by this domain, clear this iommu in iommu_bmp
3817 * update iommu count and coherency
3818 */
276dbf99
DW		 3819 if (iommu == device_to_iommu(info->segment, info->bus,
3820 info->devfn))
c7151a8d
WH		 3821 found = 1;
3822 }
3823
3e7abe25
RD		 3824 spin_unlock_irqrestore(&device_domain_lock, flags);
3825
c7151a8d
WH		 3826 if (found == 0) {
3827 unsigned long tmp_flags;
3828 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 3829 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 3830 domain->iommu_count--;
58c610bd 3831 domain_update_iommu_cap(domain);
c7151a8d 3832 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3833
9b4554b2
AW		 3834 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3835 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3836 spin_lock_irqsave(&iommu->lock, tmp_flags);
3837 clear_bit(domain->id, iommu->domain_ids);
3838 iommu->domains[domain->id] = NULL;
3839 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3840 }
c7151a8d 3841 }
c7151a8d
WH		 3842}
3843
3844static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3845{
3846 struct device_domain_info *info;
3847 struct intel_iommu *iommu;
3848 unsigned long flags1, flags2;
3849
3850 spin_lock_irqsave(&device_domain_lock, flags1);
3851 while (!list_empty(&domain->devices)) {
3852 info = list_entry(domain->devices.next,
3853 struct device_domain_info, link);
109b9b04 3854 unlink_domain_info(info);
c7151a8d
WH		 3855 spin_unlock_irqrestore(&device_domain_lock, flags1);
3856
93a23a72 3857 iommu_disable_dev_iotlb(info);
276dbf99 3858 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3859 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3860 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3861
3862 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3863 * and capabilities
c7151a8d
WH		 3864 */
3865 spin_lock_irqsave(&domain->iommu_lock, flags2);
3866 if (test_and_clear_bit(iommu->seq_id,
1b198bb0 3867 domain->iommu_bmp)) {
c7151a8d 3868 domain->iommu_count--;
58c610bd 3869 domain_update_iommu_cap(domain);
c7151a8d
WH		 3870 }
3871 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3872
3873 free_devinfo_mem(info);
3874 spin_lock_irqsave(&device_domain_lock, flags1);
3875 }
3876 spin_unlock_irqrestore(&device_domain_lock, flags1);
3877}
3878
5e98c4b1 3879/* domain id for virtual machine, it won't be set in context */
18d99165 3880static atomic_t vm_domid = ATOMIC_INIT(0);
5e98c4b1
WH		 3881
3882static struct dmar_domain *iommu_alloc_vm_domain(void)
3883{
3884 struct dmar_domain *domain;
3885
3886 domain = alloc_domain_mem();
3887 if (!domain)
3888 return NULL;
3889
18d99165 3890 domain->id = atomic_inc_return(&vm_domid);
4c923d47 3891 domain->nid = -1;
1b198bb0 3892 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
5e98c4b1
WH		 3893 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3894
3895 return domain;
3896}
3897
2c2e2c38 3898static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3899{
3900 int adjust_width;
3901
3902 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3903 spin_lock_init(&domain->iommu_lock);
3904
3905 domain_reserve_special_ranges(domain);
3906
3907 /* calculate AGAW */
3908 domain->gaw = guest_width;
3909 adjust_width = guestwidth_to_adjustwidth(guest_width);
3910 domain->agaw = width_to_agaw(adjust_width);
3911
3912 INIT_LIST_HEAD(&domain->devices);
3913
3914 domain->iommu_count = 0;
3915 domain->iommu_coherency = 0;
c5b15255 3916 domain->iommu_snooping = 0;
6dd9a7c7 3917 domain->iommu_superpage = 0;
fe40f1e0 3918 domain->max_addr = 0;
4c923d47 3919 domain->nid = -1;
5e98c4b1
WH		 3920
3921 /* always allocate the top pgd */
4c923d47 3922 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3923 if (!domain->pgd)
3924 return -ENOMEM;
3925 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3926 return 0;
3927}
3928
3929static void iommu_free_vm_domain(struct dmar_domain *domain)
3930{
3931 unsigned long flags;
3932 struct dmar_drhd_unit *drhd;
3933 struct intel_iommu *iommu;
3934 unsigned long i;
3935 unsigned long ndomains;
3936
3937 for_each_drhd_unit(drhd) {
3938 if (drhd->ignored)
3939 continue;
3940 iommu = drhd->iommu;
3941
3942 ndomains = cap_ndoms(iommu->cap);
a45946ab 3943 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3944 if (iommu->domains[i] == domain) {
3945 spin_lock_irqsave(&iommu->lock, flags);
3946 clear_bit(i, iommu->domain_ids);
3947 iommu->domains[i] = NULL;
3948 spin_unlock_irqrestore(&iommu->lock, flags);
3949 break;
3950 }
5e98c4b1
WH		 3951 }
3952 }
3953}
3954
3955static void vm_domain_exit(struct dmar_domain *domain)
3956{
5e98c4b1
WH		 3957 /* Domain 0 is reserved, so dont process it */
3958 if (!domain)
3959 return;
3960
3961 vm_domain_remove_all_dev_info(domain);
3962 /* destroy iovas */
3963 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3964
3965 /* clear ptes */
595badf5 3966 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3967
3968 /* free page tables */
d794dc9b 3969 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3970
3971 iommu_free_vm_domain(domain);
3972 free_domain_mem(domain);
3973}
3974
5d450806 3975static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3976{
5d450806 3977 struct dmar_domain *dmar_domain;
38717946 3978
5d450806
JR		 3979 dmar_domain = iommu_alloc_vm_domain();
3980 if (!dmar_domain) {
38717946 3981 printk(KERN_ERR
5d450806
JR		 3982 "intel_iommu_domain_init: dmar_domain == NULL\n");
3983 return -ENOMEM;
38717946 3984 }
2c2e2c38 3985 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3986 printk(KERN_ERR
5d450806
JR		 3987 "intel_iommu_domain_init() failed\n");
3988 vm_domain_exit(dmar_domain);
3989 return -ENOMEM;
38717946 3990 }
8140a95d 3991 domain_update_iommu_cap(dmar_domain);
5d450806 3992 domain->priv = dmar_domain;
faa3d6f5 3993
8a0e715b
JR		 3994 domain->geometry.aperture_start = 0;
3995 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3996 domain->geometry.force_aperture = true;
3997
5d450806 3998 return 0;
38717946 3999}
38717946 4000
5d450806 4001static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4002{
5d450806
JR		 4003 struct dmar_domain *dmar_domain = domain->priv;
4004
4005 domain->priv = NULL;
4006 vm_domain_exit(dmar_domain);
38717946 4007}
38717946 4008
4c5478c9
JR		 4009static int intel_iommu_attach_device(struct iommu_domain *domain,
4010 struct device *dev)
38717946 4011{
4c5478c9
JR		 4012 struct dmar_domain *dmar_domain = domain->priv;
4013 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 4014 struct intel_iommu *iommu;
4015 int addr_width;
faa3d6f5
WH		 4016
4017 /* normally pdev is not mapped */
4018 if (unlikely(domain_context_mapped(pdev))) {
4019 struct dmar_domain *old_domain;
4020
4021 old_domain = find_domain(pdev);
4022 if (old_domain) {
2c2e2c38
FY		 4023 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4024 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4025 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 4026 else
4027 domain_remove_dev_info(old_domain);
4028 }
4029 }
4030
276dbf99
DW		 4031 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4032 pdev->devfn);
fe40f1e0
WH		 4033 if (!iommu)
4034 return -ENODEV;
4035
4036 /* check if this iommu agaw is sufficient for max mapped address */
4037 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4038 if (addr_width > cap_mgaw(iommu->cap))
4039 addr_width = cap_mgaw(iommu->cap);
4040
4041 if (dmar_domain->max_addr > (1LL << addr_width)) {
4042 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4043 "sufficient for the mapped address (%llx)\n",
a99c47a2 4044 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4045 return -EFAULT;
4046 }
a99c47a2
TL		 4047 dmar_domain->gaw = addr_width;
4048
4049 /*
4050 * Knock out extra levels of page tables if necessary
4051 */
4052 while (iommu->agaw < dmar_domain->agaw) {
4053 struct dma_pte *pte;
4054
4055 pte = dmar_domain->pgd;
4056 if (dma_pte_present(pte)) {
25cbff16
SY		 4057 dmar_domain->pgd = (struct dma_pte *)
4058 phys_to_virt(dma_pte_addr(pte));
7a661013 4059 free_pgtable_page(pte);
a99c47a2
TL		 4060 }
4061 dmar_domain->agaw--;
4062 }
fe40f1e0 4063
5fe60f4e 4064 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4065}
38717946 4066
4c5478c9
JR		 4067static void intel_iommu_detach_device(struct iommu_domain *domain,
4068 struct device *dev)
38717946 4069{
4c5478c9
JR		 4070 struct dmar_domain *dmar_domain = domain->priv;
4071 struct pci_dev *pdev = to_pci_dev(dev);
4072
2c2e2c38 4073 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4074}
c7151a8d 4075
b146a1c9
JR		 4076static int intel_iommu_map(struct iommu_domain *domain,
4077 unsigned long iova, phys_addr_t hpa,
5009065d 4078 size_t size, int iommu_prot)
faa3d6f5 4079{
dde57a21 4080 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4081 u64 max_addr;
dde57a21 4082 int prot = 0;
faa3d6f5 4083 int ret;
fe40f1e0 4084
dde57a21
JR		 4085 if (iommu_prot & IOMMU_READ)
4086 prot |= DMA_PTE_READ;
4087 if (iommu_prot & IOMMU_WRITE)
4088 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4089 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4090 prot |= DMA_PTE_SNP;
dde57a21 4091
163cc52c 4092 max_addr = iova + size;
dde57a21 4093 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4094 u64 end;
4095
4096 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4097 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4098 if (end < max_addr) {
8954da1f 4099 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4100 "sufficient for the mapped address (%llx)\n",
8954da1f 4101 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4102 return -EFAULT;
4103 }
dde57a21 4104 dmar_domain->max_addr = max_addr;
fe40f1e0 4105 }
ad051221
DW		 4106 /* Round up size to next multiple of PAGE_SIZE, if it and
4107 the low bits of hpa would take us onto the next page */
88cb6a74 4108 size = aligned_nrpages(hpa, size);
ad051221
DW		 4109 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4110 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4111 return ret;
38717946 4112}
38717946 4113
5009065d
OBC		 4114static size_t intel_iommu_unmap(struct iommu_domain *domain,
4115 unsigned long iova, size_t size)
38717946 4116{
dde57a21 4117 struct dmar_domain *dmar_domain = domain->priv;
292827cb 4118 int order;
4b99d352 4119
292827cb 4120 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
163cc52c 4121 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 4122
163cc52c
DW		 4123 if (dmar_domain->max_addr == iova + size)
4124 dmar_domain->max_addr = iova;
b146a1c9 4125
5009065d 4126 return PAGE_SIZE << order;
38717946 4127}
38717946 4128
d14d6577 4129static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4130 dma_addr_t iova)
38717946 4131{
d14d6577 4132 struct dmar_domain *dmar_domain = domain->priv;
38717946 4133 struct dma_pte *pte;
faa3d6f5 4134 u64 phys = 0;
38717946 4135
6dd9a7c7 4136 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 4137 if (pte)
faa3d6f5 4138 phys = dma_pte_addr(pte);
38717946 4139
faa3d6f5 4140 return phys;
38717946 4141}
a8bcbb0d 4142
dbb9fd86
SY		 4143static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4144 unsigned long cap)
4145{
4146 struct dmar_domain *dmar_domain = domain->priv;
4147
4148 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4149 return dmar_domain->iommu_snooping;
323f99cb 4150 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4151 return irq_remapping_enabled;
dbb9fd86
SY		 4152
4153 return 0;
4154}
4155
783f157b 4156#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4157
abdfdde2
AW		 4158static int intel_iommu_add_device(struct device *dev)
4159{
4160 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4161 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4162 struct iommu_group *group;
4163 int ret;
70ae6f0d 4164
abdfdde2
AW		 4165 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4166 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4167 return -ENODEV;
4168
4169 bridge = pci_find_upstream_pcie_bridge(pdev);
4170 if (bridge) {
abdfdde2
AW		 4171 if (pci_is_pcie(bridge))
4172 dma_pdev = pci_get_domain_bus_and_slot(
4173 pci_domain_nr(pdev->bus),
4174 bridge->subordinate->number, 0);
3da4af0a 4175 if (!dma_pdev)
abdfdde2
AW		 4176 dma_pdev = pci_dev_get(bridge);
4177 } else
4178 dma_pdev = pci_dev_get(pdev);
4179
a4ff1fc2 4180 /* Account for quirked devices */
783f157b
AW		 4181 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4182
a4ff1fc2
AW		 4183 /*
4184 * If it's a multifunction device that does not support our
c14d2690
AW		 4185 * required ACS flags, add to the same group as lowest numbered
4186 * function that also does not suport the required ACS flags.
a4ff1fc2 4187 */
783f157b 4188 if (dma_pdev->multifunction &&
c14d2690
AW		 4189 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4190 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4191
4192 for (i = 0; i < 8; i++) {
4193 struct pci_dev *tmp;
4194
4195 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4196 if (!tmp)
4197 continue;
4198
4199 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4200 swap_pci_ref(&dma_pdev, tmp);
4201 break;
4202 }
4203 pci_dev_put(tmp);
4204 }
4205 }
783f157b 4206
a4ff1fc2
AW		 4207 /*
4208 * Devices on the root bus go through the iommu. If that's not us,
4209 * find the next upstream device and test ACS up to the root bus.
4210 * Finding the next device may require skipping virtual buses.
4211 */
783f157b 4212 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4213 struct pci_bus *bus = dma_pdev->bus;
4214
4215 while (!bus->self) {
4216 if (!pci_is_root_bus(bus))
4217 bus = bus->parent;
4218 else
4219 goto root_bus;
4220 }
4221
4222 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4223 break;
4224
a4ff1fc2 4225 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4226 }
4227
a4ff1fc2 4228root_bus:
abdfdde2
AW		 4229 group = iommu_group_get(&dma_pdev->dev);
4230 pci_dev_put(dma_pdev);
4231 if (!group) {
4232 group = iommu_group_alloc();
4233 if (IS_ERR(group))
4234 return PTR_ERR(group);
70ae6f0d
AW		 4235 }
4236
abdfdde2 4237 ret = iommu_group_add_device(group, dev);
bcb71abe 4238
abdfdde2
AW		 4239 iommu_group_put(group);
4240 return ret;
4241}
70ae6f0d 4242
abdfdde2
AW		 4243static void intel_iommu_remove_device(struct device *dev)
4244{
4245 iommu_group_remove_device(dev);
70ae6f0d
AW		 4246}
4247
a8bcbb0d
JR		 4248static struct iommu_ops intel_iommu_ops = {
4249 .domain_init = intel_iommu_domain_init,
4250 .domain_destroy = intel_iommu_domain_destroy,
4251 .attach_dev = intel_iommu_attach_device,
4252 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4253 .map = intel_iommu_map,
4254 .unmap = intel_iommu_unmap,
a8bcbb0d 4255 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4256 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4257 .add_device = intel_iommu_add_device,
4258 .remove_device = intel_iommu_remove_device,
6d1c56a9 4259 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4260};
9af88143 4261
9452618e
DV		 4262static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4263{
4264 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4265 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4266 dmar_map_gfx = 0;
4267}
4268
4269DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4270DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4271DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4272DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4273DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4274DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4275DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4276
d34d6517 4277static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4278{
4279 /*
4280 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4281 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4282 */
4283 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4284 rwbf_quirk = 1;
4285}
4286
4287DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4288DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4289DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4290DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4291DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4292DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4293DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4294
eecfd57f
AJ		 4295#define GGC 0x52
4296#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4297#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4298#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4299#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4300#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4301#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4302#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4303#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4304
d34d6517 4305static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4306{
4307 unsigned short ggc;
4308
eecfd57f 4309 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4310 return;
4311
eecfd57f 4312 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4313 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4314 dmar_map_gfx = 0;
6fbcfb3e
DW		 4315 } else if (dmar_map_gfx) {
4316 /* we have to ensure the gfx device is idle before we flush */
4317 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4318 intel_iommu_strict = 1;
4319 }
9eecabcb
DW		 4320}
4321DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4322DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4323DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4324DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4325
e0fc7e0b
DW		 4326/* On Tylersburg chipsets, some BIOSes have been known to enable the
4327 ISOCH DMAR unit for the Azalia sound device, but not give it any
4328 TLB entries, which causes it to deadlock. Check for that. We do
4329 this in a function called from init_dmars(), instead of in a PCI
4330 quirk, because we don't want to print the obnoxious "BIOS broken"
4331 message if VT-d is actually disabled.
4332*/
4333static void __init check_tylersburg_isoch(void)
4334{
4335 struct pci_dev *pdev;
4336 uint32_t vtisochctrl;
4337
4338 /* If there's no Azalia in the system anyway, forget it. */
4339 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4340 if (!pdev)
4341 return;
4342 pci_dev_put(pdev);
4343
4344 /* System Management Registers. Might be hidden, in which case
4345 we can't do the sanity check. But that's OK, because the
4346 known-broken BIOSes _don't_ actually hide it, so far. */
4347 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4348 if (!pdev)
4349 return;
4350
4351 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4352 pci_dev_put(pdev);
4353 return;
4354 }
4355
4356 pci_dev_put(pdev);
4357
4358 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4359 if (vtisochctrl & 1)
4360 return;
4361
4362 /* Drop all bits other than the number of TLB entries */
4363 vtisochctrl &= 0x1c;
4364
4365 /* If we have the recommended number of TLB entries (16), fine. */
4366 if (vtisochctrl == 0x10)
4367 return;
4368
4369 /* Zero TLB entries? You get to ride the short bus to school. */
4370 if (!vtisochctrl) {
4371 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4372 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4373 dmi_get_system_info(DMI_BIOS_VENDOR),
4374 dmi_get_system_info(DMI_BIOS_VERSION),
4375 dmi_get_system_info(DMI_PRODUCT_VERSION));
4376 iommu_identity_mapping |= IDENTMAP_AZALIA;
4377 return;
4378 }
4379
4380 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4381 vtisochctrl);
4382}
Linux 6.x block layer and io_uring tree(s)