projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'asoc/fix/omap' into asoc-linus
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
54485c30 27#include <linux/export.h>
ba395927
KA		 28#include <linux/slab.h>
29#include <linux/irq.h>
30#include <linux/interrupt.h>
ba395927
KA		 31#include <linux/spinlock.h>
32#include <linux/pci.h>
33#include <linux/dmar.h>
34#include <linux/dma-mapping.h>
35#include <linux/mempool.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
8a8f422d 45#include <asm/irq_remapping.h>
ba395927 46#include <asm/cacheflush.h>
46a7fa27 47#include <asm/iommu.h>
ba395927 48
078e1ee2 49#include "irq_remapping.h"
61e015ac 50#include "pci.h"
078e1ee2 51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927
KA		 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
56#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 57#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 58
59#define IOAPIC_RANGE_START (0xfee00000)
60#define IOAPIC_RANGE_END (0xfeefffff)
61#define IOVA_START_ADDR (0x1000)
62
63#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
64
4ed0d3e6
FY		 65#define MAX_AGAW_WIDTH 64
66
2ebe3151
DW		 67#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
68#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
69
70/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
71 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
72#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
73 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
74#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 75
f27be03b 76#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 77#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 78#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 79
df08cdc7
AM		 80/* page table handling */
81#define LEVEL_STRIDE (9)
82#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83
6d1c56a9
OBC		 84/*
85 * This bitmap is used to advertise the page sizes our hardware support
86 * to the IOMMU core, which will then use this information to split
87 * physically contiguous memory regions it is mapping into page sizes
88 * that we support.
89 *
90 * Traditionally the IOMMU core just handed us the mappings directly,
91 * after making sure the size is an order of a 4KiB page and that the
92 * mapping has natural alignment.
93 *
94 * To retain this behavior, we currently advertise that we support
95 * all page sizes that are an order of 4KiB.
96 *
97 * If at some point we'd like to utilize the IOMMU core's new behavior,
98 * we could change this to advertise the real page sizes we support.
99 */
100#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
101
df08cdc7
AM		 102static inline int agaw_to_level(int agaw)
103{
104 return agaw + 2;
105}
106
107static inline int agaw_to_width(int agaw)
108{
109 return 30 + agaw * LEVEL_STRIDE;
110}
111
112static inline int width_to_agaw(int width)
113{
114 return (width - 30) / LEVEL_STRIDE;
115}
116
117static inline unsigned int level_to_offset_bits(int level)
118{
119 return (level - 1) * LEVEL_STRIDE;
120}
121
122static inline int pfn_level_offset(unsigned long pfn, int level)
123{
124 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
125}
126
127static inline unsigned long level_mask(int level)
128{
129 return -1UL << level_to_offset_bits(level);
130}
131
132static inline unsigned long level_size(int level)
133{
134 return 1UL << level_to_offset_bits(level);
135}
136
137static inline unsigned long align_to_level(unsigned long pfn, int level)
138{
139 return (pfn + level_size(level) - 1) & level_mask(level);
140}
fd18de50 141
6dd9a7c7
YS		 142static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
143{
144 return 1 << ((lvl - 1) * LEVEL_STRIDE);
145}
146
dd4e8319
DW		 147/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
148 are never going to work. */
149static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
150{
151 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
152}
153
154static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
155{
156 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
157}
158static inline unsigned long page_to_dma_pfn(struct page *pg)
159{
160 return mm_to_dma_pfn(page_to_pfn(pg));
161}
162static inline unsigned long virt_to_dma_pfn(void *p)
163{
164 return page_to_dma_pfn(virt_to_page(p));
165}
166
d9630fe9
WH		 167/* global iommu list, set NULL for ignored DMAR units */
168static struct intel_iommu **g_iommus;
169
e0fc7e0b 170static void __init check_tylersburg_isoch(void);
9af88143
DW		 171static int rwbf_quirk;
172
b779260b
JC		 173/*
174 * set to 1 to panic kernel if can't successfully enable VT-d
175 * (used when kernel is launched w/ TXT)
176 */
177static int force_on = 0;
178
46b08e1a
MM		 179/*
180 * 0: Present
181 * 1-11: Reserved
182 * 12-63: Context Ptr (12 - (haw-1))
183 * 64-127: Reserved
184 */
185struct root_entry {
186 u64 val;
187 u64 rsvd1;
188};
189#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
190static inline bool root_present(struct root_entry *root)
191{
192 return (root->val & 1);
193}
194static inline void set_root_present(struct root_entry *root)
195{
196 root->val |= 1;
197}
198static inline void set_root_value(struct root_entry *root, unsigned long value)
199{
200 root->val |= value & VTD_PAGE_MASK;
201}
202
203static inline struct context_entry *
204get_context_addr_from_root(struct root_entry *root)
205{
206 return (struct context_entry *)
207 (root_present(root)?phys_to_virt(
208 root->val & VTD_PAGE_MASK) :
209 NULL);
210}
211
7a8fc25e
MM		 212/*
213 * low 64 bits:
214 * 0: present
215 * 1: fault processing disable
216 * 2-3: translation type
217 * 12-63: address space root
218 * high 64 bits:
219 * 0-2: address width
220 * 3-6: aval
221 * 8-23: domain id
222 */
223struct context_entry {
224 u64 lo;
225 u64 hi;
226};
c07e7d21
MM		 227
228static inline bool context_present(struct context_entry *context)
229{
230 return (context->lo & 1);
231}
232static inline void context_set_present(struct context_entry *context)
233{
234 context->lo |= 1;
235}
236
237static inline void context_set_fault_enable(struct context_entry *context)
238{
239 context->lo &= (((u64)-1) << 2) | 1;
240}
241
c07e7d21
MM		 242static inline void context_set_translation_type(struct context_entry *context,
243 unsigned long value)
244{
245 context->lo &= (((u64)-1) << 4) | 3;
246 context->lo |= (value & 3) << 2;
247}
248
249static inline void context_set_address_root(struct context_entry *context,
250 unsigned long value)
251{
252 context->lo |= value & VTD_PAGE_MASK;
253}
254
255static inline void context_set_address_width(struct context_entry *context,
256 unsigned long value)
257{
258 context->hi |= value & 7;
259}
260
261static inline void context_set_domain_id(struct context_entry *context,
262 unsigned long value)
263{
264 context->hi |= (value & ((1 << 16) - 1)) << 8;
265}
266
267static inline void context_clear_entry(struct context_entry *context)
268{
269 context->lo = 0;
270 context->hi = 0;
271}
7a8fc25e 272
622ba12a
MM		 273/*
274 * 0: readable
275 * 1: writable
276 * 2-6: reserved
277 * 7: super page
9cf06697
SY		 278 * 8-10: available
279 * 11: snoop behavior
622ba12a
MM		 280 * 12-63: Host physcial address
281 */
282struct dma_pte {
283 u64 val;
284};
622ba12a 285
19c239ce
MM		 286static inline void dma_clear_pte(struct dma_pte *pte)
287{
288 pte->val = 0;
289}
290
291static inline void dma_set_pte_readable(struct dma_pte *pte)
292{
293 pte->val |= DMA_PTE_READ;
294}
295
296static inline void dma_set_pte_writable(struct dma_pte *pte)
297{
298 pte->val |= DMA_PTE_WRITE;
299}
300
9cf06697
SY		 301static inline void dma_set_pte_snp(struct dma_pte *pte)
302{
303 pte->val |= DMA_PTE_SNP;
304}
305
19c239ce
MM		 306static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
307{
308 pte->val = (pte->val & ~3) | (prot & 3);
309}
310
311static inline u64 dma_pte_addr(struct dma_pte *pte)
312{
c85994e4
DW		 313#ifdef CONFIG_64BIT
314 return pte->val & VTD_PAGE_MASK;
315#else
316 /* Must have a full atomic 64-bit read */
1a8bd481 317 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 318#endif
19c239ce
MM		 319}
320
dd4e8319 321static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 322{
dd4e8319 323 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 324}
325
326static inline bool dma_pte_present(struct dma_pte *pte)
327{
328 return (pte->val & 3) != 0;
329}
622ba12a 330
4399c8bf
AK		 331static inline bool dma_pte_superpage(struct dma_pte *pte)
332{
333 return (pte->val & (1 << 7));
334}
335
75e6bf96
DW		 336static inline int first_pte_in_page(struct dma_pte *pte)
337{
338 return !((unsigned long)pte & ~VTD_PAGE_MASK);
339}
340
2c2e2c38
FY		 341/*
342 * This domain is a statically identity mapping domain.
343 * 1. This domain creats a static 1:1 mapping to all usable memory.
344 * 2. It maps to each iommu if successful.
345 * 3. Each iommu mapps to this domain if successful.
346 */
19943b0e
DW		 347static struct dmar_domain *si_domain;
348static int hw_pass_through = 1;
2c2e2c38 349
3b5410e7 350/* devices under the same p2p bridge are owned in one domain */
cdc7b837 351#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 352
1ce28feb
WH		 353/* domain represents a virtual machine, more than one devices
354 * across iommus may be owned in one domain, e.g. kvm guest.
355 */
356#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
357
2c2e2c38
FY		 358/* si_domain contains mulitple devices */
359#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
360
1b198bb0
MT		 361/* define the limit of IOMMUs supported in each domain */
362#ifdef CONFIG_X86
363# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
364#else
365# define IOMMU_UNITS_SUPPORTED 64
366#endif
367
99126f7c
MM		 368struct dmar_domain {
369 int id; /* domain id */
4c923d47 370 int nid; /* node id */
1b198bb0
MT		 371 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
372 /* bitmap of iommus this domain uses*/
99126f7c
MM		 373
374 struct list_head devices; /* all devices' list */
375 struct iova_domain iovad; /* iova's that belong to this domain */
376
377 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 378 int gaw; /* max guest address width */
379
380 /* adjusted guest address width, 0 is level 2 30-bit */
381 int agaw;
382
3b5410e7 383 int flags; /* flags to find out type of domain */
8e604097
WH		 384
385 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 386 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 387 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 388 int iommu_superpage;/* Level of superpages supported:
389 0 == 4KiB (no superpages), 1 == 2MiB,
390 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 391 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 392 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 393};
394
a647dacb
MM		 395/* PCI domain-device relationship */
396struct device_domain_info {
397 struct list_head link; /* link to domain siblings */
398 struct list_head global; /* link to global list */
276dbf99
DW		 399 int segment; /* PCI domain */
400 u8 bus; /* PCI bus number */
a647dacb 401 u8 devfn; /* PCI devfn number */
45e829ea 402 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 403 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 404 struct dmar_domain *domain; /* pointer to domain */
405};
406
5e0d2a6f 407static void flush_unmaps_timeout(unsigned long data);
408
409DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
410
80b20dd8 411#define HIGH_WATER_MARK 250
412struct deferred_flush_tables {
413 int next;
414 struct iova *iova[HIGH_WATER_MARK];
415 struct dmar_domain *domain[HIGH_WATER_MARK];
416};
417
418static struct deferred_flush_tables *deferred_flush;
419
5e0d2a6f 420/* bitmap for indexing intel_iommus */
5e0d2a6f 421static int g_num_of_iommus;
422
423static DEFINE_SPINLOCK(async_umap_flush_lock);
424static LIST_HEAD(unmaps_to_do);
425
426static int timer_on;
427static long list_size;
5e0d2a6f 428
ba395927
KA		 429static void domain_remove_dev_info(struct dmar_domain *domain);
430
d3f13810 431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 432int dmar_disabled = 0;
433#else
434int dmar_disabled = 1;
d3f13810 435#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 436
8bc1f85c
ED		 437int intel_iommu_enabled = 0;
438EXPORT_SYMBOL_GPL(intel_iommu_enabled);
439
2d9e667e 440static int dmar_map_gfx = 1;
7d3b03ce 441static int dmar_forcedac;
5e0d2a6f 442static int intel_iommu_strict;
6dd9a7c7 443static int intel_iommu_superpage = 1;
ba395927 444
c0771df8
DW		 445int intel_iommu_gfx_mapped;
446EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
447
ba395927
KA		 448#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
449static DEFINE_SPINLOCK(device_domain_lock);
450static LIST_HEAD(device_domain_list);
451
a8bcbb0d
JR		 452static struct iommu_ops intel_iommu_ops;
453
ba395927
KA		 454static int __init intel_iommu_setup(char *str)
455{
456 if (!str)
457 return -EINVAL;
458 while (*str) {
0cd5c3c8
KM		 459 if (!strncmp(str, "on", 2)) {
460 dmar_disabled = 0;
461 printk(KERN_INFO "Intel-IOMMU: enabled\n");
462 } else if (!strncmp(str, "off", 3)) {
ba395927 463 dmar_disabled = 1;
0cd5c3c8 464 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 465 } else if (!strncmp(str, "igfx_off", 8)) {
466 dmar_map_gfx = 0;
467 printk(KERN_INFO
468 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 469 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 470 printk(KERN_INFO
7d3b03ce
KA		 471 "Intel-IOMMU: Forcing DAC for PCI devices\n");
472 dmar_forcedac = 1;
5e0d2a6f 473 } else if (!strncmp(str, "strict", 6)) {
474 printk(KERN_INFO
475 "Intel-IOMMU: disable batched IOTLB flush\n");
476 intel_iommu_strict = 1;
6dd9a7c7
YS		 477 } else if (!strncmp(str, "sp_off", 6)) {
478 printk(KERN_INFO
479 "Intel-IOMMU: disable supported super page\n");
480 intel_iommu_superpage = 0;
ba395927
KA		 481 }
482
483 str += strcspn(str, ",");
484 while (*str == ',')
485 str++;
486 }
487 return 0;
488}
489__setup("intel_iommu=", intel_iommu_setup);
490
491static struct kmem_cache *iommu_domain_cache;
492static struct kmem_cache *iommu_devinfo_cache;
493static struct kmem_cache *iommu_iova_cache;
494
4c923d47 495static inline void *alloc_pgtable_page(int node)
eb3fa7cb 496{
4c923d47
SS		 497 struct page *page;
498 void *vaddr = NULL;
eb3fa7cb 499
4c923d47
SS		 500 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
501 if (page)
502 vaddr = page_address(page);
eb3fa7cb 503 return vaddr;
ba395927
KA		 504}
505
506static inline void free_pgtable_page(void *vaddr)
507{
508 free_page((unsigned long)vaddr);
509}
510
511static inline void *alloc_domain_mem(void)
512{
354bb65e 513 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 514}
515
38717946 516static void free_domain_mem(void *vaddr)
ba395927
KA		 517{
518 kmem_cache_free(iommu_domain_cache, vaddr);
519}
520
521static inline void * alloc_devinfo_mem(void)
522{
354bb65e 523 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 524}
525
526static inline void free_devinfo_mem(void *vaddr)
527{
528 kmem_cache_free(iommu_devinfo_cache, vaddr);
529}
530
531struct iova *alloc_iova_mem(void)
532{
354bb65e 533 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 534}
535
536void free_iova_mem(struct iova *iova)
537{
538 kmem_cache_free(iommu_iova_cache, iova);
539}
540
1b573683 541
4ed0d3e6 542static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 543{
544 unsigned long sagaw;
545 int agaw = -1;
546
547 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 548 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 549 agaw >= 0; agaw--) {
550 if (test_bit(agaw, &sagaw))
551 break;
552 }
553
554 return agaw;
555}
556
4ed0d3e6
FY		 557/*
558 * Calculate max SAGAW for each iommu.
559 */
560int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
561{
562 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
563}
564
565/*
566 * calculate agaw for each iommu.
567 * "SAGAW" may be different across iommus, use a default agaw, and
568 * get a supported less agaw for iommus that don't support the default agaw.
569 */
570int iommu_calculate_agaw(struct intel_iommu *iommu)
571{
572 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
573}
574
2c2e2c38 575/* This functionin only returns single iommu in a domain */
8c11e798
WH		 576static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
577{
578 int iommu_id;
579
2c2e2c38 580 /* si_domain and vm domain should not get here. */
1ce28feb 581 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 582 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 583
1b198bb0 584 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 585 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
586 return NULL;
587
588 return g_iommus[iommu_id];
589}
590
8e604097
WH		 591static void domain_update_iommu_coherency(struct dmar_domain *domain)
592{
593 int i;
594
2e12bc29
AW		 595 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
596
597 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 598
1b198bb0 599 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 600 if (!ecap_coherent(g_iommus[i]->ecap)) {
601 domain->iommu_coherency = 0;
602 break;
603 }
8e604097
WH		 604 }
605}
606
58c610bd
SY		 607static void domain_update_iommu_snooping(struct dmar_domain *domain)
608{
609 int i;
610
611 domain->iommu_snooping = 1;
612
1b198bb0 613 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 614 if (!ecap_sc_support(g_iommus[i]->ecap)) {
615 domain->iommu_snooping = 0;
616 break;
617 }
58c610bd
SY		 618 }
619}
620
6dd9a7c7
YS		 621static void domain_update_iommu_superpage(struct dmar_domain *domain)
622{
8140a95d
AK		 623 struct dmar_drhd_unit *drhd;
624 struct intel_iommu *iommu = NULL;
625 int mask = 0xf;
6dd9a7c7
YS		 626
627 if (!intel_iommu_superpage) {
628 domain->iommu_superpage = 0;
629 return;
630 }
631
8140a95d
AK		 632 /* set iommu_superpage to the smallest common denominator */
633 for_each_active_iommu(iommu, drhd) {
634 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 635 if (!mask) {
636 break;
637 }
638 }
639 domain->iommu_superpage = fls(mask);
640}
641
58c610bd
SY		 642/* Some capabilities may be different across iommus */
643static void domain_update_iommu_cap(struct dmar_domain *domain)
644{
645 domain_update_iommu_coherency(domain);
646 domain_update_iommu_snooping(domain);
6dd9a7c7 647 domain_update_iommu_superpage(domain);
58c610bd
SY		 648}
649
276dbf99 650static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 651{
652 struct dmar_drhd_unit *drhd = NULL;
653 int i;
654
655 for_each_drhd_unit(drhd) {
656 if (drhd->ignored)
657 continue;
276dbf99
DW		 658 if (segment != drhd->segment)
659 continue;
c7151a8d 660
924b6231 661 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 662 if (drhd->devices[i] &&
663 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 664 drhd->devices[i]->devfn == devfn)
665 return drhd->iommu;
4958c5dc
DW		 666 if (drhd->devices[i] &&
667 drhd->devices[i]->subordinate &&
924b6231 668 drhd->devices[i]->subordinate->number <= bus &&
b918c62e 669 drhd->devices[i]->subordinate->busn_res.end >= bus)
924b6231
DW		 670 return drhd->iommu;
671 }
c7151a8d
WH		 672
673 if (drhd->include_all)
674 return drhd->iommu;
675 }
676
677 return NULL;
678}
679
5331fe6f
WH		 680static void domain_flush_cache(struct dmar_domain *domain,
681 void *addr, int size)
682{
683 if (!domain->iommu_coherency)
684 clflush_cache_range(addr, size);
685}
686
ba395927
KA		 687/* Gets context entry for a given bus and devfn */
688static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
689 u8 bus, u8 devfn)
690{
691 struct root_entry *root;
692 struct context_entry *context;
693 unsigned long phy_addr;
694 unsigned long flags;
695
696 spin_lock_irqsave(&iommu->lock, flags);
697 root = &iommu->root_entry[bus];
698 context = get_context_addr_from_root(root);
699 if (!context) {
4c923d47
SS		 700 context = (struct context_entry *)
701 alloc_pgtable_page(iommu->node);
ba395927
KA		 702 if (!context) {
703 spin_unlock_irqrestore(&iommu->lock, flags);
704 return NULL;
705 }
5b6985ce 706 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 707 phy_addr = virt_to_phys((void *)context);
708 set_root_value(root, phy_addr);
709 set_root_present(root);
710 __iommu_flush_cache(iommu, root, sizeof(*root));
711 }
712 spin_unlock_irqrestore(&iommu->lock, flags);
713 return &context[devfn];
714}
715
716static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
717{
718 struct root_entry *root;
719 struct context_entry *context;
720 int ret;
721 unsigned long flags;
722
723 spin_lock_irqsave(&iommu->lock, flags);
724 root = &iommu->root_entry[bus];
725 context = get_context_addr_from_root(root);
726 if (!context) {
727 ret = 0;
728 goto out;
729 }
c07e7d21 730 ret = context_present(&context[devfn]);
ba395927
KA		 731out:
732 spin_unlock_irqrestore(&iommu->lock, flags);
733 return ret;
734}
735
736static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
737{
738 struct root_entry *root;
739 struct context_entry *context;
740 unsigned long flags;
741
742 spin_lock_irqsave(&iommu->lock, flags);
743 root = &iommu->root_entry[bus];
744 context = get_context_addr_from_root(root);
745 if (context) {
c07e7d21 746 context_clear_entry(&context[devfn]);
ba395927
KA		 747 __iommu_flush_cache(iommu, &context[devfn], \
748 sizeof(*context));
749 }
750 spin_unlock_irqrestore(&iommu->lock, flags);
751}
752
753static void free_context_table(struct intel_iommu *iommu)
754{
755 struct root_entry *root;
756 int i;
757 unsigned long flags;
758 struct context_entry *context;
759
760 spin_lock_irqsave(&iommu->lock, flags);
761 if (!iommu->root_entry) {
762 goto out;
763 }
764 for (i = 0; i < ROOT_ENTRY_NR; i++) {
765 root = &iommu->root_entry[i];
766 context = get_context_addr_from_root(root);
767 if (context)
768 free_pgtable_page(context);
769 }
770 free_pgtable_page(iommu->root_entry);
771 iommu->root_entry = NULL;
772out:
773 spin_unlock_irqrestore(&iommu->lock, flags);
774}
775
b026fd28 776static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
4399c8bf 777 unsigned long pfn, int target_level)
ba395927 778{
b026fd28 779 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 780 struct dma_pte *parent, *pte = NULL;
781 int level = agaw_to_level(domain->agaw);
4399c8bf 782 int offset;
ba395927
KA		 783
784 BUG_ON(!domain->pgd);
b026fd28 785 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 786 parent = domain->pgd;
787
ba395927
KA		 788 while (level > 0) {
789 void *tmp_page;
790
b026fd28 791 offset = pfn_level_offset(pfn, level);
ba395927 792 pte = &parent[offset];
4399c8bf 793 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7
YS		 794 break;
795 if (level == target_level)
ba395927
KA		 796 break;
797
19c239ce 798 if (!dma_pte_present(pte)) {
c85994e4
DW		 799 uint64_t pteval;
800
4c923d47 801 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 802
206a73c1 803 if (!tmp_page)
ba395927 804 return NULL;
206a73c1 805
c85994e4 806 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 807 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 808 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
809 /* Someone else set it while we were thinking; use theirs. */
810 free_pgtable_page(tmp_page);
811 } else {
812 dma_pte_addr(pte);
813 domain_flush_cache(domain, pte, sizeof(*pte));
814 }
ba395927 815 }
19c239ce 816 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 817 level--;
818 }
819
ba395927
KA		 820 return pte;
821}
822
6dd9a7c7 823
ba395927 824/* return address's pte at specific level */
90dcfb5e
DW		 825static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
826 unsigned long pfn,
6dd9a7c7 827 int level, int *large_page)
ba395927
KA		 828{
829 struct dma_pte *parent, *pte = NULL;
830 int total = agaw_to_level(domain->agaw);
831 int offset;
832
833 parent = domain->pgd;
834 while (level <= total) {
90dcfb5e 835 offset = pfn_level_offset(pfn, total);
ba395927
KA		 836 pte = &parent[offset];
837 if (level == total)
838 return pte;
839
6dd9a7c7
YS		 840 if (!dma_pte_present(pte)) {
841 *large_page = total;
ba395927 842 break;
6dd9a7c7
YS		 843 }
844
845 if (pte->val & DMA_PTE_LARGE_PAGE) {
846 *large_page = total;
847 return pte;
848 }
849
19c239ce 850 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 851 total--;
852 }
853 return NULL;
854}
855
ba395927 856/* clear last level pte, a tlb flush should be followed */
292827cb 857static int dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 858 unsigned long start_pfn,
859 unsigned long last_pfn)
ba395927 860{
04b18e65 861 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 862 unsigned int large_page = 1;
310a5ab9 863 struct dma_pte *first_pte, *pte;
292827cb 864 int order;
66eae846 865
04b18e65 866 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 867 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 868 BUG_ON(start_pfn > last_pfn);
ba395927 869
04b18e65 870 /* we don't need lock here; nobody else touches the iova range */
59c36286 871 do {
6dd9a7c7
YS		 872 large_page = 1;
873 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 874 if (!pte) {
6dd9a7c7 875 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 876 continue;
877 }
6dd9a7c7 878 do {
310a5ab9 879 dma_clear_pte(pte);
6dd9a7c7 880 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 881 pte++;
75e6bf96
DW		 882 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
883
310a5ab9
DW		 884 domain_flush_cache(domain, first_pte,
885 (void *)pte - (void *)first_pte);
59c36286
DW		 886
887 } while (start_pfn && start_pfn <= last_pfn);
292827cb
AK		 888
889 order = (large_page - 1) * 9;
890 return order;
ba395927
KA		 891}
892
893/* free page table pages. last level pte should already be cleared */
894static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 895 unsigned long start_pfn,
896 unsigned long last_pfn)
ba395927 897{
6660c63a 898 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
f3a0a52f 899 struct dma_pte *first_pte, *pte;
ba395927
KA		 900 int total = agaw_to_level(domain->agaw);
901 int level;
6660c63a 902 unsigned long tmp;
6dd9a7c7 903 int large_page = 2;
ba395927 904
6660c63a
DW		 905 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
906 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 907 BUG_ON(start_pfn > last_pfn);
ba395927 908
f3a0a52f 909 /* We don't need lock here; nobody else touches the iova range */
ba395927
KA		 910 level = 2;
911 while (level <= total) {
6660c63a
DW		 912 tmp = align_to_level(start_pfn, level);
913
f3a0a52f 914 /* If we can't even clear one PTE at this level, we're done */
6660c63a 915 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 916 return;
917
59c36286 918 do {
6dd9a7c7
YS		 919 large_page = level;
920 first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
921 if (large_page > level)
922 level = large_page + 1;
f3a0a52f
DW		 923 if (!pte) {
924 tmp = align_to_level(tmp + 1, level + 1);
925 continue;
926 }
75e6bf96 927 do {
6a43e574
DW		 928 if (dma_pte_present(pte)) {
929 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
930 dma_clear_pte(pte);
931 }
f3a0a52f
DW		 932 pte++;
933 tmp += level_size(level);
75e6bf96
DW		 934 } while (!first_pte_in_page(pte) &&
935 tmp + level_size(level) - 1 <= last_pfn);
936
f3a0a52f
DW		 937 domain_flush_cache(domain, first_pte,
938 (void *)pte - (void *)first_pte);
939
59c36286 940 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
ba395927
KA		 941 level++;
942 }
943 /* free pgd */
d794dc9b 944 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 945 free_pgtable_page(domain->pgd);
946 domain->pgd = NULL;
947 }
948}
949
950/* iommu handling */
951static int iommu_alloc_root_entry(struct intel_iommu *iommu)
952{
953 struct root_entry *root;
954 unsigned long flags;
955
4c923d47 956 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 957 if (!root)
958 return -ENOMEM;
959
5b6985ce 960 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 961
962 spin_lock_irqsave(&iommu->lock, flags);
963 iommu->root_entry = root;
964 spin_unlock_irqrestore(&iommu->lock, flags);
965
966 return 0;
967}
968
ba395927
KA		 969static void iommu_set_root_entry(struct intel_iommu *iommu)
970{
971 void *addr;
c416daa9 972 u32 sts;
ba395927
KA		 973 unsigned long flag;
974
975 addr = iommu->root_entry;
976
1f5b3c3f 977 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 978 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
979
c416daa9 980 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 981
982 /* Make sure hardware complete it */
983 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 984 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 985
1f5b3c3f 986 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 987}
988
989static void iommu_flush_write_buffer(struct intel_iommu *iommu)
990{
991 u32 val;
992 unsigned long flag;
993
9af88143 994 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 995 return;
ba395927 996
1f5b3c3f 997 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 998 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 999
1000 /* Make sure hardware complete it */
1001 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1002 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1003
1f5b3c3f 1004 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1005}
1006
1007/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1008static void __iommu_flush_context(struct intel_iommu *iommu,
1009 u16 did, u16 source_id, u8 function_mask,
1010 u64 type)
ba395927
KA		 1011{
1012 u64 val = 0;
1013 unsigned long flag;
1014
ba395927
KA		 1015 switch (type) {
1016 case DMA_CCMD_GLOBAL_INVL:
1017 val = DMA_CCMD_GLOBAL_INVL;
1018 break;
1019 case DMA_CCMD_DOMAIN_INVL:
1020 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1021 break;
1022 case DMA_CCMD_DEVICE_INVL:
1023 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1024 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1025 break;
1026 default:
1027 BUG();
1028 }
1029 val |= DMA_CCMD_ICC;
1030
1f5b3c3f 1031 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1032 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1033
1034 /* Make sure hardware complete it */
1035 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1036 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1037
1f5b3c3f 1038 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1039}
1040
ba395927 1041/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1042static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1043 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1044{
1045 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1046 u64 val = 0, val_iva = 0;
1047 unsigned long flag;
1048
ba395927
KA		 1049 switch (type) {
1050 case DMA_TLB_GLOBAL_FLUSH:
1051 /* global flush doesn't need set IVA_REG */
1052 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1053 break;
1054 case DMA_TLB_DSI_FLUSH:
1055 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1056 break;
1057 case DMA_TLB_PSI_FLUSH:
1058 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1059 /* Note: always flush non-leaf currently */
1060 val_iva = size_order | addr;
1061 break;
1062 default:
1063 BUG();
1064 }
1065 /* Note: set drain read/write */
1066#if 0
1067 /*
1068 * This is probably to be super secure.. Looks like we can
1069 * ignore it without any impact.
1070 */
1071 if (cap_read_drain(iommu->cap))
1072 val |= DMA_TLB_READ_DRAIN;
1073#endif
1074 if (cap_write_drain(iommu->cap))
1075 val |= DMA_TLB_WRITE_DRAIN;
1076
1f5b3c3f 1077 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1078 /* Note: Only uses first TLB reg currently */
1079 if (val_iva)
1080 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1081 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1082
1083 /* Make sure hardware complete it */
1084 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1085 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1086
1f5b3c3f 1087 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1088
1089 /* check IOTLB invalidation granularity */
1090 if (DMA_TLB_IAIG(val) == 0)
1091 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1092 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1093 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1094 (unsigned long long)DMA_TLB_IIRG(type),
1095 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1096}
1097
93a23a72
YZ		 1098static struct device_domain_info *iommu_support_dev_iotlb(
1099 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1100{
1101 int found = 0;
1102 unsigned long flags;
1103 struct device_domain_info *info;
1104 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1105
1106 if (!ecap_dev_iotlb_support(iommu->ecap))
1107 return NULL;
1108
1109 if (!iommu->qi)
1110 return NULL;
1111
1112 spin_lock_irqsave(&device_domain_lock, flags);
1113 list_for_each_entry(info, &domain->devices, link)
1114 if (info->bus == bus && info->devfn == devfn) {
1115 found = 1;
1116 break;
1117 }
1118 spin_unlock_irqrestore(&device_domain_lock, flags);
1119
1120 if (!found || !info->dev)
1121 return NULL;
1122
1123 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1124 return NULL;
1125
1126 if (!dmar_find_matched_atsr_unit(info->dev))
1127 return NULL;
1128
1129 info->iommu = iommu;
1130
1131 return info;
1132}
1133
1134static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1135{
93a23a72
YZ		 1136 if (!info)
1137 return;
1138
1139 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1140}
1141
1142static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1143{
1144 if (!info->dev || !pci_ats_enabled(info->dev))
1145 return;
1146
1147 pci_disable_ats(info->dev);
1148}
1149
1150static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1151 u64 addr, unsigned mask)
1152{
1153 u16 sid, qdep;
1154 unsigned long flags;
1155 struct device_domain_info *info;
1156
1157 spin_lock_irqsave(&device_domain_lock, flags);
1158 list_for_each_entry(info, &domain->devices, link) {
1159 if (!info->dev || !pci_ats_enabled(info->dev))
1160 continue;
1161
1162 sid = info->bus << 8 | info->devfn;
1163 qdep = pci_ats_queue_depth(info->dev);
1164 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1165 }
1166 spin_unlock_irqrestore(&device_domain_lock, flags);
1167}
1168
1f0ef2aa 1169static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1170 unsigned long pfn, unsigned int pages, int map)
ba395927 1171{
9dd2fe89 1172 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1173 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1174
ba395927
KA		 1175 BUG_ON(pages == 0);
1176
ba395927 1177 /*
9dd2fe89
YZ		 1178 * Fallback to domain selective flush if no PSI support or the size is
1179 * too big.
ba395927
KA		 1180 * PSI requires page size to be 2 ^ x, and the base address is naturally
1181 * aligned to the size
1182 */
9dd2fe89
YZ		 1183 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1184 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1185 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1186 else
1187 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1188 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1189
1190 /*
82653633
NA		 1191 * In caching mode, changes of pages from non-present to present require
1192 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1193 */
82653633 1194 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1195 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1196}
1197
f8bab735 1198static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1199{
1200 u32 pmen;
1201 unsigned long flags;
1202
1f5b3c3f 1203 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1204 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1205 pmen &= ~DMA_PMEN_EPM;
1206 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1207
1208 /* wait for the protected region status bit to clear */
1209 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1210 readl, !(pmen & DMA_PMEN_PRS), pmen);
1211
1f5b3c3f 1212 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1213}
1214
ba395927
KA		 1215static int iommu_enable_translation(struct intel_iommu *iommu)
1216{
1217 u32 sts;
1218 unsigned long flags;
1219
1f5b3c3f 1220 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1221 iommu->gcmd |= DMA_GCMD_TE;
1222 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1223
1224 /* Make sure hardware complete it */
1225 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1226 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1227
1f5b3c3f 1228 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1229 return 0;
1230}
1231
1232static int iommu_disable_translation(struct intel_iommu *iommu)
1233{
1234 u32 sts;
1235 unsigned long flag;
1236
1f5b3c3f 1237 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1238 iommu->gcmd &= ~DMA_GCMD_TE;
1239 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1240
1241 /* Make sure hardware complete it */
1242 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1243 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1244
1f5b3c3f 1245 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1246 return 0;
1247}
1248
3460a6d9 1249
ba395927
KA		 1250static int iommu_init_domains(struct intel_iommu *iommu)
1251{
1252 unsigned long ndomains;
1253 unsigned long nlongs;
1254
1255 ndomains = cap_ndoms(iommu->cap);
68aeb968 1256 pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
680a7524 1257 ndomains);
ba395927
KA		 1258 nlongs = BITS_TO_LONGS(ndomains);
1259
94a91b50
DD		 1260 spin_lock_init(&iommu->lock);
1261
ba395927
KA		 1262 /* TBD: there might be 64K domains,
1263 * consider other allocation for future chip
1264 */
1265 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1266 if (!iommu->domain_ids) {
1267 printk(KERN_ERR "Allocating domain id array failed\n");
1268 return -ENOMEM;
1269 }
1270 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1271 GFP_KERNEL);
1272 if (!iommu->domains) {
1273 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1274 return -ENOMEM;
1275 }
1276
1277 /*
1278 * if Caching mode is set, then invalid translations are tagged
1279 * with domainid 0. Hence we need to pre-allocate it.
1280 */
1281 if (cap_caching_mode(iommu->cap))
1282 set_bit(0, iommu->domain_ids);
1283 return 0;
1284}
ba395927 1285
ba395927
KA		 1286
1287static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1288static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1289
1290void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1291{
1292 struct dmar_domain *domain;
1293 int i;
c7151a8d 1294 unsigned long flags;
ba395927 1295
94a91b50 1296 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1297 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1298 domain = iommu->domains[i];
1299 clear_bit(i, iommu->domain_ids);
1300
1301 spin_lock_irqsave(&domain->iommu_lock, flags);
1302 if (--domain->iommu_count == 0) {
1303 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1304 vm_domain_exit(domain);
1305 else
1306 domain_exit(domain);
1307 }
1308 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1309 }
ba395927
KA		 1310 }
1311
1312 if (iommu->gcmd & DMA_GCMD_TE)
1313 iommu_disable_translation(iommu);
1314
1315 if (iommu->irq) {
dced35ae 1316 irq_set_handler_data(iommu->irq, NULL);
ba395927
KA		 1317 /* This will mask the irq */
1318 free_irq(iommu->irq, iommu);
1319 destroy_irq(iommu->irq);
1320 }
1321
1322 kfree(iommu->domains);
1323 kfree(iommu->domain_ids);
1324
d9630fe9
WH		 1325 g_iommus[iommu->seq_id] = NULL;
1326
1327 /* if all iommus are freed, free g_iommus */
1328 for (i = 0; i < g_num_of_iommus; i++) {
1329 if (g_iommus[i])
1330 break;
1331 }
1332
1333 if (i == g_num_of_iommus)
1334 kfree(g_iommus);
1335
ba395927
KA		 1336 /* free context mapping */
1337 free_context_table(iommu);
ba395927
KA		 1338}
1339
2c2e2c38 1340static struct dmar_domain *alloc_domain(void)
ba395927 1341{
ba395927 1342 struct dmar_domain *domain;
ba395927
KA		 1343
1344 domain = alloc_domain_mem();
1345 if (!domain)
1346 return NULL;
1347
4c923d47 1348 domain->nid = -1;
1b198bb0 1349 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38
FY		 1350 domain->flags = 0;
1351
1352 return domain;
1353}
1354
1355static int iommu_attach_domain(struct dmar_domain *domain,
1356 struct intel_iommu *iommu)
1357{
1358 int num;
1359 unsigned long ndomains;
1360 unsigned long flags;
1361
ba395927
KA		 1362 ndomains = cap_ndoms(iommu->cap);
1363
1364 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1365
ba395927
KA		 1366 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1367 if (num >= ndomains) {
1368 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1369 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1370 return -ENOMEM;
ba395927
KA		 1371 }
1372
ba395927 1373 domain->id = num;
2c2e2c38 1374 set_bit(num, iommu->domain_ids);
1b198bb0 1375 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1376 iommu->domains[num] = domain;
1377 spin_unlock_irqrestore(&iommu->lock, flags);
1378
2c2e2c38 1379 return 0;
ba395927
KA		 1380}
1381
2c2e2c38
FY		 1382static void iommu_detach_domain(struct dmar_domain *domain,
1383 struct intel_iommu *iommu)
ba395927
KA		 1384{
1385 unsigned long flags;
2c2e2c38
FY		 1386 int num, ndomains;
1387 int found = 0;
ba395927 1388
8c11e798 1389 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1390 ndomains = cap_ndoms(iommu->cap);
a45946ab 1391 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1392 if (iommu->domains[num] == domain) {
1393 found = 1;
1394 break;
1395 }
2c2e2c38
FY		 1396 }
1397
1398 if (found) {
1399 clear_bit(num, iommu->domain_ids);
1b198bb0 1400 clear_bit(iommu->seq_id, domain->iommu_bmp);
2c2e2c38
FY		 1401 iommu->domains[num] = NULL;
1402 }
8c11e798 1403 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1404}
1405
1406static struct iova_domain reserved_iova_list;
8a443df4 1407static struct lock_class_key reserved_rbtree_key;
ba395927 1408
51a63e67 1409static int dmar_init_reserved_ranges(void)
ba395927
KA		 1410{
1411 struct pci_dev *pdev = NULL;
1412 struct iova *iova;
1413 int i;
ba395927 1414
f661197e 1415 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1416
8a443df4
MG		 1417 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1418 &reserved_rbtree_key);
1419
ba395927
KA		 1420 /* IOAPIC ranges shouldn't be accessed by DMA */
1421 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1422 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1423 if (!iova) {
ba395927 1424 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1425 return -ENODEV;
1426 }
ba395927
KA		 1427
1428 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1429 for_each_pci_dev(pdev) {
1430 struct resource *r;
1431
1432 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1433 r = &pdev->resource[i];
1434 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1435 continue;
1a4a4551
DW		 1436 iova = reserve_iova(&reserved_iova_list,
1437 IOVA_PFN(r->start),
1438 IOVA_PFN(r->end));
51a63e67 1439 if (!iova) {
ba395927 1440 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1441 return -ENODEV;
1442 }
ba395927
KA		 1443 }
1444 }
51a63e67 1445 return 0;
ba395927
KA		 1446}
1447
1448static void domain_reserve_special_ranges(struct dmar_domain *domain)
1449{
1450 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1451}
1452
1453static inline int guestwidth_to_adjustwidth(int gaw)
1454{
1455 int agaw;
1456 int r = (gaw - 12) % 9;
1457
1458 if (r == 0)
1459 agaw = gaw;
1460 else
1461 agaw = gaw + 9 - r;
1462 if (agaw > 64)
1463 agaw = 64;
1464 return agaw;
1465}
1466
1467static int domain_init(struct dmar_domain *domain, int guest_width)
1468{
1469 struct intel_iommu *iommu;
1470 int adjust_width, agaw;
1471 unsigned long sagaw;
1472
f661197e 1473 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1474 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1475
1476 domain_reserve_special_ranges(domain);
1477
1478 /* calculate AGAW */
8c11e798 1479 iommu = domain_get_iommu(domain);
ba395927
KA		 1480 if (guest_width > cap_mgaw(iommu->cap))
1481 guest_width = cap_mgaw(iommu->cap);
1482 domain->gaw = guest_width;
1483 adjust_width = guestwidth_to_adjustwidth(guest_width);
1484 agaw = width_to_agaw(adjust_width);
1485 sagaw = cap_sagaw(iommu->cap);
1486 if (!test_bit(agaw, &sagaw)) {
1487 /* hardware doesn't support it, choose a bigger one */
1488 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1489 agaw = find_next_bit(&sagaw, 5, agaw);
1490 if (agaw >= 5)
1491 return -ENODEV;
1492 }
1493 domain->agaw = agaw;
1494 INIT_LIST_HEAD(&domain->devices);
1495
8e604097
WH		 1496 if (ecap_coherent(iommu->ecap))
1497 domain->iommu_coherency = 1;
1498 else
1499 domain->iommu_coherency = 0;
1500
58c610bd
SY		 1501 if (ecap_sc_support(iommu->ecap))
1502 domain->iommu_snooping = 1;
1503 else
1504 domain->iommu_snooping = 0;
1505
6dd9a7c7 1506 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1507 domain->iommu_count = 1;
4c923d47 1508 domain->nid = iommu->node;
c7151a8d 1509
ba395927 1510 /* always allocate the top pgd */
4c923d47 1511 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1512 if (!domain->pgd)
1513 return -ENOMEM;
5b6985ce 1514 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1515 return 0;
1516}
1517
1518static void domain_exit(struct dmar_domain *domain)
1519{
2c2e2c38
FY		 1520 struct dmar_drhd_unit *drhd;
1521 struct intel_iommu *iommu;
ba395927
KA		 1522
1523 /* Domain 0 is reserved, so dont process it */
1524 if (!domain)
1525 return;
1526
7b668357
AW		 1527 /* Flush any lazy unmaps that may reference this domain */
1528 if (!intel_iommu_strict)
1529 flush_unmaps_timeout(0);
1530
ba395927
KA		 1531 domain_remove_dev_info(domain);
1532 /* destroy iovas */
1533 put_iova_domain(&domain->iovad);
ba395927
KA		 1534
1535 /* clear ptes */
595badf5 1536 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1537
1538 /* free page tables */
d794dc9b 1539 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1540
2c2e2c38 1541 for_each_active_iommu(iommu, drhd)
1b198bb0 1542 if (test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38
FY		 1543 iommu_detach_domain(domain, iommu);
1544
ba395927
KA		 1545 free_domain_mem(domain);
1546}
1547
4ed0d3e6
FY		 1548static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1549 u8 bus, u8 devfn, int translation)
ba395927
KA		 1550{
1551 struct context_entry *context;
ba395927 1552 unsigned long flags;
5331fe6f 1553 struct intel_iommu *iommu;
ea6606b0
WH		 1554 struct dma_pte *pgd;
1555 unsigned long num;
1556 unsigned long ndomains;
1557 int id;
1558 int agaw;
93a23a72 1559 struct device_domain_info *info = NULL;
ba395927
KA		 1560
1561 pr_debug("Set context mapping for %02x:%02x.%d\n",
1562 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1563
ba395927 1564 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1565 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1566 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1567
276dbf99 1568 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1569 if (!iommu)
1570 return -ENODEV;
1571
ba395927
KA		 1572 context = device_to_context_entry(iommu, bus, devfn);
1573 if (!context)
1574 return -ENOMEM;
1575 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1576 if (context_present(context)) {
ba395927
KA		 1577 spin_unlock_irqrestore(&iommu->lock, flags);
1578 return 0;
1579 }
1580
ea6606b0
WH		 1581 id = domain->id;
1582 pgd = domain->pgd;
1583
2c2e2c38
FY		 1584 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1585 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1586 int found = 0;
1587
1588 /* find an available domain id for this device in iommu */
1589 ndomains = cap_ndoms(iommu->cap);
a45946ab 1590 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1591 if (iommu->domains[num] == domain) {
1592 id = num;
1593 found = 1;
1594 break;
1595 }
ea6606b0
WH		 1596 }
1597
1598 if (found == 0) {
1599 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1600 if (num >= ndomains) {
1601 spin_unlock_irqrestore(&iommu->lock, flags);
1602 printk(KERN_ERR "IOMMU: no free domain ids\n");
1603 return -EFAULT;
1604 }
1605
1606 set_bit(num, iommu->domain_ids);
1607 iommu->domains[num] = domain;
1608 id = num;
1609 }
1610
1611 /* Skip top levels of page tables for
1612 * iommu which has less agaw than default.
1672af11 1613 * Unnecessary for PT mode.
ea6606b0 1614 */
1672af11
CW		 1615 if (translation != CONTEXT_TT_PASS_THROUGH) {
1616 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1617 pgd = phys_to_virt(dma_pte_addr(pgd));
1618 if (!dma_pte_present(pgd)) {
1619 spin_unlock_irqrestore(&iommu->lock, flags);
1620 return -ENOMEM;
1621 }
ea6606b0
WH		 1622 }
1623 }
1624 }
1625
1626 context_set_domain_id(context, id);
4ed0d3e6 1627
93a23a72
YZ		 1628 if (translation != CONTEXT_TT_PASS_THROUGH) {
1629 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1630 translation = info ? CONTEXT_TT_DEV_IOTLB :
1631 CONTEXT_TT_MULTI_LEVEL;
1632 }
4ed0d3e6
FY		 1633 /*
1634 * In pass through mode, AW must be programmed to indicate the largest
1635 * AGAW value supported by hardware. And ASR is ignored by hardware.
1636 */
93a23a72 1637 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1638 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1639 else {
1640 context_set_address_root(context, virt_to_phys(pgd));
1641 context_set_address_width(context, iommu->agaw);
1642 }
4ed0d3e6
FY		 1643
1644 context_set_translation_type(context, translation);
c07e7d21
MM		 1645 context_set_fault_enable(context);
1646 context_set_present(context);
5331fe6f 1647 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1648
4c25a2c1
DW		 1649 /*
1650 * It's a non-present to present mapping. If hardware doesn't cache
1651 * non-present entry we only need to flush the write-buffer. If the
1652 * _does_ cache non-present entries, then it does so in the special
1653 * domain #0, which we have to flush:
1654 */
1655 if (cap_caching_mode(iommu->cap)) {
1656 iommu->flush.flush_context(iommu, 0,
1657 (((u16)bus) << 8) | devfn,
1658 DMA_CCMD_MASK_NOBIT,
1659 DMA_CCMD_DEVICE_INVL);
82653633 1660 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1661 } else {
ba395927 1662 iommu_flush_write_buffer(iommu);
4c25a2c1 1663 }
93a23a72 1664 iommu_enable_dev_iotlb(info);
ba395927 1665 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1666
1667 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1668 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1669 domain->iommu_count++;
4c923d47
SS		 1670 if (domain->iommu_count == 1)
1671 domain->nid = iommu->node;
58c610bd 1672 domain_update_iommu_cap(domain);
c7151a8d
WH		 1673 }
1674 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1675 return 0;
1676}
1677
1678static int
4ed0d3e6
FY		 1679domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1680 int translation)
ba395927
KA		 1681{
1682 int ret;
1683 struct pci_dev *tmp, *parent;
1684
276dbf99 1685 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1686 pdev->bus->number, pdev->devfn,
1687 translation);
ba395927
KA		 1688 if (ret)
1689 return ret;
1690
1691 /* dependent device mapping */
1692 tmp = pci_find_upstream_pcie_bridge(pdev);
1693 if (!tmp)
1694 return 0;
1695 /* Secondary interface's bus number and devfn 0 */
1696 parent = pdev->bus->self;
1697 while (parent != tmp) {
276dbf99
DW		 1698 ret = domain_context_mapping_one(domain,
1699 pci_domain_nr(parent->bus),
1700 parent->bus->number,
4ed0d3e6 1701 parent->devfn, translation);
ba395927
KA		 1702 if (ret)
1703 return ret;
1704 parent = parent->bus->self;
1705 }
45e829ea 1706 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1707 return domain_context_mapping_one(domain,
276dbf99 1708 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1709 tmp->subordinate->number, 0,
1710 translation);
ba395927
KA		 1711 else /* this is a legacy PCI bridge */
1712 return domain_context_mapping_one(domain,
276dbf99
DW		 1713 pci_domain_nr(tmp->bus),
1714 tmp->bus->number,
4ed0d3e6
FY		 1715 tmp->devfn,
1716 translation);
ba395927
KA		 1717}
1718
5331fe6f 1719static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1720{
1721 int ret;
1722 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1723 struct intel_iommu *iommu;
1724
276dbf99
DW		 1725 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1726 pdev->devfn);
5331fe6f
WH		 1727 if (!iommu)
1728 return -ENODEV;
ba395927 1729
276dbf99 1730 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1731 if (!ret)
1732 return ret;
1733 /* dependent device mapping */
1734 tmp = pci_find_upstream_pcie_bridge(pdev);
1735 if (!tmp)
1736 return ret;
1737 /* Secondary interface's bus number and devfn 0 */
1738 parent = pdev->bus->self;
1739 while (parent != tmp) {
8c11e798 1740 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1741 parent->devfn);
ba395927
KA		 1742 if (!ret)
1743 return ret;
1744 parent = parent->bus->self;
1745 }
5f4d91a1 1746 if (pci_is_pcie(tmp))
276dbf99
DW		 1747 return device_context_mapped(iommu, tmp->subordinate->number,
1748 0);
ba395927 1749 else
276dbf99
DW		 1750 return device_context_mapped(iommu, tmp->bus->number,
1751 tmp->devfn);
ba395927
KA		 1752}
1753
f532959b
FY		 1754/* Returns a number of VTD pages, but aligned to MM page size */
1755static inline unsigned long aligned_nrpages(unsigned long host_addr,
1756 size_t size)
1757{
1758 host_addr &= ~PAGE_MASK;
1759 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1760}
1761
6dd9a7c7
YS		 1762/* Return largest possible superpage level for a given mapping */
1763static inline int hardware_largepage_caps(struct dmar_domain *domain,
1764 unsigned long iov_pfn,
1765 unsigned long phy_pfn,
1766 unsigned long pages)
1767{
1768 int support, level = 1;
1769 unsigned long pfnmerge;
1770
1771 support = domain->iommu_superpage;
1772
1773 /* To use a large page, the virtual *and* physical addresses
1774 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1775 of them will mean we have to use smaller pages. So just
1776 merge them and check both at once. */
1777 pfnmerge = iov_pfn | phy_pfn;
1778
1779 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1780 pages >>= VTD_STRIDE_SHIFT;
1781 if (!pages)
1782 break;
1783 pfnmerge >>= VTD_STRIDE_SHIFT;
1784 level++;
1785 support--;
1786 }
1787 return level;
1788}
1789
9051aa02
DW		 1790static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1791 struct scatterlist *sg, unsigned long phys_pfn,
1792 unsigned long nr_pages, int prot)
e1605495
DW		 1793{
1794 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1795 phys_addr_t uninitialized_var(pteval);
e1605495 1796 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1797 unsigned long sg_res;
6dd9a7c7
YS		 1798 unsigned int largepage_lvl = 0;
1799 unsigned long lvl_pages = 0;
e1605495
DW		 1800
1801 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1802
1803 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1804 return -EINVAL;
1805
1806 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1807
9051aa02
DW		 1808 if (sg)
1809 sg_res = 0;
1810 else {
1811 sg_res = nr_pages + 1;
1812 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1813 }
1814
6dd9a7c7 1815 while (nr_pages > 0) {
c85994e4
DW		 1816 uint64_t tmp;
1817
e1605495 1818 if (!sg_res) {
f532959b 1819 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1820 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1821 sg->dma_length = sg->length;
1822 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1823 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1824 }
6dd9a7c7 1825
e1605495 1826 if (!pte) {
6dd9a7c7
YS		 1827 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1828
1829 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1830 if (!pte)
1831 return -ENOMEM;
6dd9a7c7 1832 /* It is large page*/
6491d4d0 1833 if (largepage_lvl > 1) {
6dd9a7c7 1834 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1835 /* Ensure that old small page tables are removed to make room
1836 for superpage, if they exist. */
1837 dma_pte_clear_range(domain, iov_pfn,
1838 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1839 dma_pte_free_pagetable(domain, iov_pfn,
1840 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1841 } else {
6dd9a7c7 1842 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1843 }
6dd9a7c7 1844
e1605495
DW		 1845 }
1846 /* We don't need lock here, nobody else
1847 * touches the iova range
1848 */
7766a3fb 1849 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1850 if (tmp) {
1bf20f0d 1851 static int dumps = 5;
c85994e4
DW		 1852 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1853 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1854 if (dumps) {
1855 dumps--;
1856 debug_dma_dump_mappings(NULL);
1857 }
1858 WARN_ON(1);
1859 }
6dd9a7c7
YS		 1860
1861 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1862
1863 BUG_ON(nr_pages < lvl_pages);
1864 BUG_ON(sg_res < lvl_pages);
1865
1866 nr_pages -= lvl_pages;
1867 iov_pfn += lvl_pages;
1868 phys_pfn += lvl_pages;
1869 pteval += lvl_pages * VTD_PAGE_SIZE;
1870 sg_res -= lvl_pages;
1871
1872 /* If the next PTE would be the first in a new page, then we
1873 need to flush the cache on the entries we've just written.
1874 And then we'll need to recalculate 'pte', so clear it and
1875 let it get set again in the if (!pte) block above.
1876
1877 If we're done (!nr_pages) we need to flush the cache too.
1878
1879 Also if we've been setting superpages, we may need to
1880 recalculate 'pte' and switch back to smaller pages for the
1881 end of the mapping, if the trailing size is not enough to
1882 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1883 pte++;
6dd9a7c7
YS		 1884 if (!nr_pages || first_pte_in_page(pte) ||
1885 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1886 domain_flush_cache(domain, first_pte,
1887 (void *)pte - (void *)first_pte);
1888 pte = NULL;
1889 }
6dd9a7c7
YS		 1890
1891 if (!sg_res && nr_pages)
e1605495
DW		 1892 sg = sg_next(sg);
1893 }
1894 return 0;
1895}
1896
9051aa02
DW		 1897static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1898 struct scatterlist *sg, unsigned long nr_pages,
1899 int prot)
ba395927 1900{
9051aa02
DW		 1901 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1902}
6f6a00e4 1903
9051aa02
DW		 1904static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1905 unsigned long phys_pfn, unsigned long nr_pages,
1906 int prot)
1907{
1908 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1909}
1910
c7151a8d 1911static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1912{
c7151a8d
WH		 1913 if (!iommu)
1914 return;
8c11e798
WH		 1915
1916 clear_context_table(iommu, bus, devfn);
1917 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1918 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1919 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1920}
1921
109b9b04
DW		 1922static inline void unlink_domain_info(struct device_domain_info *info)
1923{
1924 assert_spin_locked(&device_domain_lock);
1925 list_del(&info->link);
1926 list_del(&info->global);
1927 if (info->dev)
1928 info->dev->dev.archdata.iommu = NULL;
1929}
1930
ba395927
KA		 1931static void domain_remove_dev_info(struct dmar_domain *domain)
1932{
1933 struct device_domain_info *info;
1934 unsigned long flags;
c7151a8d 1935 struct intel_iommu *iommu;
ba395927
KA		 1936
1937 spin_lock_irqsave(&device_domain_lock, flags);
1938 while (!list_empty(&domain->devices)) {
1939 info = list_entry(domain->devices.next,
1940 struct device_domain_info, link);
109b9b04 1941 unlink_domain_info(info);
ba395927
KA		 1942 spin_unlock_irqrestore(&device_domain_lock, flags);
1943
93a23a72 1944 iommu_disable_dev_iotlb(info);
276dbf99 1945 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1946 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1947 free_devinfo_mem(info);
1948
1949 spin_lock_irqsave(&device_domain_lock, flags);
1950 }
1951 spin_unlock_irqrestore(&device_domain_lock, flags);
1952}
1953
1954/*
1955 * find_domain
358dd8ac 1956 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1957 */
38717946 1958static struct dmar_domain *
ba395927
KA		 1959find_domain(struct pci_dev *pdev)
1960{
1961 struct device_domain_info *info;
1962
1963 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1964 info = pdev->dev.archdata.iommu;
ba395927
KA		 1965 if (info)
1966 return info->domain;
1967 return NULL;
1968}
1969
ba395927
KA		 1970/* domain is initialized */
1971static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1972{
1973 struct dmar_domain *domain, *found = NULL;
1974 struct intel_iommu *iommu;
1975 struct dmar_drhd_unit *drhd;
1976 struct device_domain_info *info, *tmp;
1977 struct pci_dev *dev_tmp;
1978 unsigned long flags;
1979 int bus = 0, devfn = 0;
276dbf99 1980 int segment;
2c2e2c38 1981 int ret;
ba395927
KA		 1982
1983 domain = find_domain(pdev);
1984 if (domain)
1985 return domain;
1986
276dbf99
DW		 1987 segment = pci_domain_nr(pdev->bus);
1988
ba395927
KA		 1989 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1990 if (dev_tmp) {
5f4d91a1 1991 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1992 bus = dev_tmp->subordinate->number;
1993 devfn = 0;
1994 } else {
1995 bus = dev_tmp->bus->number;
1996 devfn = dev_tmp->devfn;
1997 }
1998 spin_lock_irqsave(&device_domain_lock, flags);
1999 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 2000 if (info->segment == segment &&
2001 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 2002 found = info->domain;
2003 break;
2004 }
2005 }
2006 spin_unlock_irqrestore(&device_domain_lock, flags);
2007 /* pcie-pci bridge already has a domain, uses it */
2008 if (found) {
2009 domain = found;
2010 goto found_domain;
2011 }
2012 }
2013
2c2e2c38
FY		 2014 domain = alloc_domain();
2015 if (!domain)
2016 goto error;
2017
ba395927
KA		 2018 /* Allocate new domain for the device */
2019 drhd = dmar_find_matched_drhd_unit(pdev);
2020 if (!drhd) {
2021 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2022 pci_name(pdev));
d2900bd6 2023 free_domain_mem(domain);
ba395927
KA		 2024 return NULL;
2025 }
2026 iommu = drhd->iommu;
2027
2c2e2c38
FY		 2028 ret = iommu_attach_domain(domain, iommu);
2029 if (ret) {
2fe9723d 2030 free_domain_mem(domain);
ba395927 2031 goto error;
2c2e2c38 2032 }
ba395927
KA		 2033
2034 if (domain_init(domain, gaw)) {
2035 domain_exit(domain);
2036 goto error;
2037 }
2038
2039 /* register pcie-to-pci device */
2040 if (dev_tmp) {
2041 info = alloc_devinfo_mem();
2042 if (!info) {
2043 domain_exit(domain);
2044 goto error;
2045 }
276dbf99 2046 info->segment = segment;
ba395927
KA		 2047 info->bus = bus;
2048 info->devfn = devfn;
2049 info->dev = NULL;
2050 info->domain = domain;
2051 /* This domain is shared by devices under p2p bridge */
3b5410e7 2052 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 2053
2054 /* pcie-to-pci bridge already has a domain, uses it */
2055 found = NULL;
2056 spin_lock_irqsave(&device_domain_lock, flags);
2057 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2058 if (tmp->segment == segment &&
2059 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2060 found = tmp->domain;
2061 break;
2062 }
2063 }
2064 if (found) {
00dfff77 2065 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2066 free_devinfo_mem(info);
2067 domain_exit(domain);
2068 domain = found;
2069 } else {
2070 list_add(&info->link, &domain->devices);
2071 list_add(&info->global, &device_domain_list);
00dfff77 2072 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2073 }
ba395927
KA		 2074 }
2075
2076found_domain:
2077 info = alloc_devinfo_mem();
2078 if (!info)
2079 goto error;
276dbf99 2080 info->segment = segment;
ba395927
KA		 2081 info->bus = pdev->bus->number;
2082 info->devfn = pdev->devfn;
2083 info->dev = pdev;
2084 info->domain = domain;
2085 spin_lock_irqsave(&device_domain_lock, flags);
2086 /* somebody is fast */
2087 found = find_domain(pdev);
2088 if (found != NULL) {
2089 spin_unlock_irqrestore(&device_domain_lock, flags);
2090 if (found != domain) {
2091 domain_exit(domain);
2092 domain = found;
2093 }
2094 free_devinfo_mem(info);
2095 return domain;
2096 }
2097 list_add(&info->link, &domain->devices);
2098 list_add(&info->global, &device_domain_list);
358dd8ac 2099 pdev->dev.archdata.iommu = info;
ba395927
KA		 2100 spin_unlock_irqrestore(&device_domain_lock, flags);
2101 return domain;
2102error:
2103 /* recheck it here, maybe others set it */
2104 return find_domain(pdev);
2105}
2106
2c2e2c38 2107static int iommu_identity_mapping;
e0fc7e0b
DW		 2108#define IDENTMAP_ALL 1
2109#define IDENTMAP_GFX 2
2110#define IDENTMAP_AZALIA 4
2c2e2c38 2111
b213203e
DW		 2112static int iommu_domain_identity_map(struct dmar_domain *domain,
2113 unsigned long long start,
2114 unsigned long long end)
ba395927 2115{
c5395d5c
DW		 2116 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2117 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2118
2119 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2120 dma_to_mm_pfn(last_vpfn))) {
ba395927 2121 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2122 return -ENOMEM;
ba395927
KA		 2123 }
2124
c5395d5c
DW		 2125 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2126 start, end, domain->id);
ba395927
KA		 2127 /*
2128 * RMRR range might have overlap with physical memory range,
2129 * clear it first
2130 */
c5395d5c 2131 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2132
c5395d5c
DW		 2133 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2134 last_vpfn - first_vpfn + 1,
61df7443 2135 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2136}
2137
2138static int iommu_prepare_identity_map(struct pci_dev *pdev,
2139 unsigned long long start,
2140 unsigned long long end)
2141{
2142 struct dmar_domain *domain;
2143 int ret;
2144
c7ab48d2 2145 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2146 if (!domain)
2147 return -ENOMEM;
2148
19943b0e
DW		 2149 /* For _hardware_ passthrough, don't bother. But for software
2150 passthrough, we do it anyway -- it may indicate a memory
2151 range which is reserved in E820, so which didn't get set
2152 up to start with in si_domain */
2153 if (domain == si_domain && hw_pass_through) {
2154 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2155 pci_name(pdev), start, end);
2156 return 0;
2157 }
2158
2159 printk(KERN_INFO
2160 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2161 pci_name(pdev), start, end);
2ff729f5 2162
5595b528
DW		 2163 if (end < start) {
2164 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2165 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2166 dmi_get_system_info(DMI_BIOS_VENDOR),
2167 dmi_get_system_info(DMI_BIOS_VERSION),
2168 dmi_get_system_info(DMI_PRODUCT_VERSION));
2169 ret = -EIO;
2170 goto error;
2171 }
2172
2ff729f5
DW		 2173 if (end >> agaw_to_width(domain->agaw)) {
2174 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2175 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2176 agaw_to_width(domain->agaw),
2177 dmi_get_system_info(DMI_BIOS_VENDOR),
2178 dmi_get_system_info(DMI_BIOS_VERSION),
2179 dmi_get_system_info(DMI_PRODUCT_VERSION));
2180 ret = -EIO;
2181 goto error;
2182 }
19943b0e 2183
b213203e 2184 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2185 if (ret)
2186 goto error;
2187
2188 /* context entry init */
4ed0d3e6 2189 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2190 if (ret)
2191 goto error;
2192
2193 return 0;
2194
2195 error:
ba395927
KA		 2196 domain_exit(domain);
2197 return ret;
ba395927
KA		 2198}
2199
2200static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2201 struct pci_dev *pdev)
2202{
358dd8ac 2203 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2204 return 0;
2205 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2206 rmrr->end_address);
ba395927
KA		 2207}
2208
d3f13810 2209#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2210static inline void iommu_prepare_isa(void)
2211{
2212 struct pci_dev *pdev;
2213 int ret;
2214
2215 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2216 if (!pdev)
2217 return;
2218
c7ab48d2 2219 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2220 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2221
2222 if (ret)
c7ab48d2
DW		 2223 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2224 "floppy might not work\n");
49a0429e
KA		 2225
2226}
2227#else
2228static inline void iommu_prepare_isa(void)
2229{
2230 return;
2231}
d3f13810 2232#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2233
2c2e2c38 2234static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2235
071e1374 2236static int __init si_domain_init(int hw)
2c2e2c38
FY		 2237{
2238 struct dmar_drhd_unit *drhd;
2239 struct intel_iommu *iommu;
c7ab48d2 2240 int nid, ret = 0;
2c2e2c38
FY		 2241
2242 si_domain = alloc_domain();
2243 if (!si_domain)
2244 return -EFAULT;
2245
c7ab48d2 2246 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2247
2248 for_each_active_iommu(iommu, drhd) {
2249 ret = iommu_attach_domain(si_domain, iommu);
2250 if (ret) {
2251 domain_exit(si_domain);
2252 return -EFAULT;
2253 }
2254 }
2255
2256 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2257 domain_exit(si_domain);
2258 return -EFAULT;
2259 }
2260
2261 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2262
19943b0e
DW		 2263 if (hw)
2264 return 0;
2265
c7ab48d2 2266 for_each_online_node(nid) {
5dfe8660
TH		 2267 unsigned long start_pfn, end_pfn;
2268 int i;
2269
2270 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2271 ret = iommu_domain_identity_map(si_domain,
2272 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2273 if (ret)
2274 return ret;
2275 }
c7ab48d2
DW		 2276 }
2277
2c2e2c38
FY		 2278 return 0;
2279}
2280
2281static void domain_remove_one_dev_info(struct dmar_domain *domain,
2282 struct pci_dev *pdev);
2283static int identity_mapping(struct pci_dev *pdev)
2284{
2285 struct device_domain_info *info;
2286
2287 if (likely(!iommu_identity_mapping))
2288 return 0;
2289
cb452a40
MT		 2290 info = pdev->dev.archdata.iommu;
2291 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2292 return (info->domain == si_domain);
2c2e2c38 2293
2c2e2c38
FY		 2294 return 0;
2295}
2296
2297static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2298 struct pci_dev *pdev,
2299 int translation)
2c2e2c38
FY		 2300{
2301 struct device_domain_info *info;
2302 unsigned long flags;
5fe60f4e 2303 int ret;
2c2e2c38
FY		 2304
2305 info = alloc_devinfo_mem();
2306 if (!info)
2307 return -ENOMEM;
2308
2309 info->segment = pci_domain_nr(pdev->bus);
2310 info->bus = pdev->bus->number;
2311 info->devfn = pdev->devfn;
2312 info->dev = pdev;
2313 info->domain = domain;
2314
2315 spin_lock_irqsave(&device_domain_lock, flags);
2316 list_add(&info->link, &domain->devices);
2317 list_add(&info->global, &device_domain_list);
2318 pdev->dev.archdata.iommu = info;
2319 spin_unlock_irqrestore(&device_domain_lock, flags);
2320
e2ad23d0
DW		 2321 ret = domain_context_mapping(domain, pdev, translation);
2322 if (ret) {
2323 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2324 unlink_domain_info(info);
e2ad23d0
DW		 2325 spin_unlock_irqrestore(&device_domain_lock, flags);
2326 free_devinfo_mem(info);
2327 return ret;
2328 }
2329
2c2e2c38
FY		 2330 return 0;
2331}
2332
ea2447f7
TM		 2333static bool device_has_rmrr(struct pci_dev *dev)
2334{
2335 struct dmar_rmrr_unit *rmrr;
2336 int i;
2337
2338 for_each_rmrr_units(rmrr) {
2339 for (i = 0; i < rmrr->devices_cnt; i++) {
2340 /*
2341 * Return TRUE if this RMRR contains the device that
2342 * is passed in.
2343 */
2344 if (rmrr->devices[i] == dev)
2345 return true;
2346 }
2347 }
2348 return false;
2349}
2350
6941af28
DW		 2351static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2352{
ea2447f7
TM		 2353
2354 /*
2355 * We want to prevent any device associated with an RMRR from
2356 * getting placed into the SI Domain. This is done because
2357 * problems exist when devices are moved in and out of domains
2358 * and their respective RMRR info is lost. We exempt USB devices
2359 * from this process due to their usage of RMRRs that are known
2360 * to not be needed after BIOS hand-off to OS.
2361 */
2362 if (device_has_rmrr(pdev) &&
2363 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2364 return 0;
2365
e0fc7e0b
DW		 2366 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2367 return 1;
2368
2369 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2370 return 1;
2371
2372 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2373 return 0;
6941af28 2374
3dfc813d
DW		 2375 /*
2376 * We want to start off with all devices in the 1:1 domain, and
2377 * take them out later if we find they can't access all of memory.
2378 *
2379 * However, we can't do this for PCI devices behind bridges,
2380 * because all PCI devices behind the same bridge will end up
2381 * with the same source-id on their transactions.
2382 *
2383 * Practically speaking, we can't change things around for these
2384 * devices at run-time, because we can't be sure there'll be no
2385 * DMA transactions in flight for any of their siblings.
2386 *
2387 * So PCI devices (unless they're on the root bus) as well as
2388 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2389 * the 1:1 domain, just in _case_ one of their siblings turns out
2390 * not to be able to map all of memory.
2391 */
5f4d91a1 2392 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2393 if (!pci_is_root_bus(pdev->bus))
2394 return 0;
2395 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2396 return 0;
62f87c0e 2397 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2398 return 0;
2399
2400 /*
2401 * At boot time, we don't yet know if devices will be 64-bit capable.
2402 * Assume that they will -- if they turn out not to be, then we can
2403 * take them out of the 1:1 domain later.
2404 */
8fcc5372
CW		 2405 if (!startup) {
2406 /*
2407 * If the device's dma_mask is less than the system's memory
2408 * size then this is not a candidate for identity mapping.
2409 */
2410 u64 dma_mask = pdev->dma_mask;
2411
2412 if (pdev->dev.coherent_dma_mask &&
2413 pdev->dev.coherent_dma_mask < dma_mask)
2414 dma_mask = pdev->dev.coherent_dma_mask;
2415
2416 return dma_mask >= dma_get_required_mask(&pdev->dev);
2417 }
6941af28
DW		 2418
2419 return 1;
2420}
2421
071e1374 2422static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2423{
2c2e2c38
FY		 2424 struct pci_dev *pdev = NULL;
2425 int ret;
2426
19943b0e 2427 ret = si_domain_init(hw);
2c2e2c38
FY		 2428 if (ret)
2429 return -EFAULT;
2430
2c2e2c38 2431 for_each_pci_dev(pdev) {
6941af28 2432 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2433 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2434 hw ? CONTEXT_TT_PASS_THROUGH :
2435 CONTEXT_TT_MULTI_LEVEL);
2436 if (ret) {
2437 /* device not associated with an iommu */
2438 if (ret == -ENODEV)
2439 continue;
62edf5dc 2440 return ret;
eae460b6
MT		 2441 }
2442 pr_info("IOMMU: %s identity mapping for device %s\n",
2443 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2444 }
2c2e2c38
FY		 2445 }
2446
2447 return 0;
2448}
2449
b779260b 2450static int __init init_dmars(void)
ba395927
KA		 2451{
2452 struct dmar_drhd_unit *drhd;
2453 struct dmar_rmrr_unit *rmrr;
2454 struct pci_dev *pdev;
2455 struct intel_iommu *iommu;
9d783ba0 2456 int i, ret;
2c2e2c38 2457
ba395927
KA		 2458 /*
2459 * for each drhd
2460 * allocate root
2461 * initialize and program root entry to not present
2462 * endfor
2463 */
2464 for_each_drhd_unit(drhd) {
5e0d2a6f 2465 /*
2466 * lock not needed as this is only incremented in the single
2467 * threaded kernel __init code path all other access are read
2468 * only
2469 */
1b198bb0
MT		 2470 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2471 g_num_of_iommus++;
2472 continue;
2473 }
2474 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2475 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2476 }
2477
d9630fe9
WH		 2478 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2479 GFP_KERNEL);
2480 if (!g_iommus) {
2481 printk(KERN_ERR "Allocating global iommu array failed\n");
2482 ret = -ENOMEM;
2483 goto error;
2484 }
2485
80b20dd8 2486 deferred_flush = kzalloc(g_num_of_iommus *
2487 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2488 if (!deferred_flush) {
5e0d2a6f 2489 ret = -ENOMEM;
2490 goto error;
2491 }
2492
5e0d2a6f 2493 for_each_drhd_unit(drhd) {
2494 if (drhd->ignored)
2495 continue;
1886e8a9
SS		 2496
2497 iommu = drhd->iommu;
d9630fe9 2498 g_iommus[iommu->seq_id] = iommu;
ba395927 2499
e61d98d8
SS		 2500 ret = iommu_init_domains(iommu);
2501 if (ret)
2502 goto error;
2503
ba395927
KA		 2504 /*
2505 * TBD:
2506 * we could share the same root & context tables
25985edc 2507 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2508 */
2509 ret = iommu_alloc_root_entry(iommu);
2510 if (ret) {
2511 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2512 goto error;
2513 }
4ed0d3e6 2514 if (!ecap_pass_through(iommu->ecap))
19943b0e 2515 hw_pass_through = 0;
ba395927
KA		 2516 }
2517
1531a6a6
SS		 2518 /*
2519 * Start from the sane iommu hardware state.
2520 */
a77b67d4
YS		 2521 for_each_drhd_unit(drhd) {
2522 if (drhd->ignored)
2523 continue;
2524
2525 iommu = drhd->iommu;
1531a6a6
SS		 2526
2527 /*
2528 * If the queued invalidation is already initialized by us
2529 * (for example, while enabling interrupt-remapping) then
2530 * we got the things already rolling from a sane state.
2531 */
2532 if (iommu->qi)
2533 continue;
2534
2535 /*
2536 * Clear any previous faults.
2537 */
2538 dmar_fault(-1, iommu);
2539 /*
2540 * Disable queued invalidation if supported and already enabled
2541 * before OS handover.
2542 */
2543 dmar_disable_qi(iommu);
2544 }
2545
2546 for_each_drhd_unit(drhd) {
2547 if (drhd->ignored)
2548 continue;
2549
2550 iommu = drhd->iommu;
2551
a77b67d4
YS		 2552 if (dmar_enable_qi(iommu)) {
2553 /*
2554 * Queued Invalidate not enabled, use Register Based
2555 * Invalidate
2556 */
2557 iommu->flush.flush_context = __iommu_flush_context;
2558 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2559 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2560 "invalidation\n",
680a7524 2561 iommu->seq_id,
b4e0f9eb 2562 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2563 } else {
2564 iommu->flush.flush_context = qi_flush_context;
2565 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2566 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2567 "invalidation\n",
680a7524 2568 iommu->seq_id,
b4e0f9eb 2569 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2570 }
2571 }
2572
19943b0e 2573 if (iommu_pass_through)
e0fc7e0b
DW		 2574 iommu_identity_mapping |= IDENTMAP_ALL;
2575
d3f13810 2576#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2577 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2578#endif
e0fc7e0b
DW		 2579
2580 check_tylersburg_isoch();
2581
ba395927 2582 /*
19943b0e
DW		 2583 * If pass through is not set or not enabled, setup context entries for
2584 * identity mappings for rmrr, gfx, and isa and may fall back to static
2585 * identity mapping if iommu_identity_mapping is set.
ba395927 2586 */
19943b0e
DW		 2587 if (iommu_identity_mapping) {
2588 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2589 if (ret) {
19943b0e
DW		 2590 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2591 goto error;
ba395927
KA		 2592 }
2593 }
ba395927 2594 /*
19943b0e
DW		 2595 * For each rmrr
2596 * for each dev attached to rmrr
2597 * do
2598 * locate drhd for dev, alloc domain for dev
2599 * allocate free domain
2600 * allocate page table entries for rmrr
2601 * if context not allocated for bus
2602 * allocate and init context
2603 * set present in root table for this bus
2604 * init context with domain, translation etc
2605 * endfor
2606 * endfor
ba395927 2607 */
19943b0e
DW		 2608 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2609 for_each_rmrr_units(rmrr) {
2610 for (i = 0; i < rmrr->devices_cnt; i++) {
2611 pdev = rmrr->devices[i];
2612 /*
2613 * some BIOS lists non-exist devices in DMAR
2614 * table.
2615 */
2616 if (!pdev)
2617 continue;
2618 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2619 if (ret)
2620 printk(KERN_ERR
2621 "IOMMU: mapping reserved region failed\n");
ba395927 2622 }
4ed0d3e6 2623 }
49a0429e 2624
19943b0e
DW		 2625 iommu_prepare_isa();
2626
ba395927
KA		 2627 /*
2628 * for each drhd
2629 * enable fault log
2630 * global invalidate context cache
2631 * global invalidate iotlb
2632 * enable translation
2633 */
2634 for_each_drhd_unit(drhd) {
51a63e67
JC		 2635 if (drhd->ignored) {
2636 /*
2637 * we always have to disable PMRs or DMA may fail on
2638 * this device
2639 */
2640 if (force_on)
2641 iommu_disable_protect_mem_regions(drhd->iommu);
ba395927 2642 continue;
51a63e67 2643 }
ba395927 2644 iommu = drhd->iommu;
ba395927
KA		 2645
2646 iommu_flush_write_buffer(iommu);
2647
3460a6d9
KA		 2648 ret = dmar_set_interrupt(iommu);
2649 if (ret)
2650 goto error;
2651
ba395927
KA		 2652 iommu_set_root_entry(iommu);
2653
4c25a2c1 2654 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2655 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2656
ba395927
KA		 2657 ret = iommu_enable_translation(iommu);
2658 if (ret)
2659 goto error;
b94996c9
DW		 2660
2661 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2662 }
2663
2664 return 0;
2665error:
2666 for_each_drhd_unit(drhd) {
2667 if (drhd->ignored)
2668 continue;
2669 iommu = drhd->iommu;
2670 free_iommu(iommu);
2671 }
d9630fe9 2672 kfree(g_iommus);
ba395927
KA		 2673 return ret;
2674}
2675
5a5e02a6 2676/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2677static struct iova *intel_alloc_iova(struct device *dev,
2678 struct dmar_domain *domain,
2679 unsigned long nrpages, uint64_t dma_mask)
ba395927 2680{
ba395927 2681 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2682 struct iova *iova = NULL;
ba395927 2683
875764de
DW		 2684 /* Restrict dma_mask to the width that the iommu can handle */
2685 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2686
2687 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2688 /*
2689 * First try to allocate an io virtual address in
284901a9 2690 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2691 * from higher range
ba395927 2692 */
875764de
DW		 2693 iova = alloc_iova(&domain->iovad, nrpages,
2694 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2695 if (iova)
2696 return iova;
2697 }
2698 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2699 if (unlikely(!iova)) {
2700 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2701 nrpages, pci_name(pdev));
f76aec76
KA		 2702 return NULL;
2703 }
2704
2705 return iova;
2706}
2707
147202aa 2708static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2709{
2710 struct dmar_domain *domain;
2711 int ret;
2712
2713 domain = get_domain_for_dev(pdev,
2714 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2715 if (!domain) {
2716 printk(KERN_ERR
2717 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2718 return NULL;
ba395927
KA		 2719 }
2720
2721 /* make sure context mapping is ok */
5331fe6f 2722 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2723 ret = domain_context_mapping(domain, pdev,
2724 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2725 if (ret) {
2726 printk(KERN_ERR
2727 "Domain context map for %s failed",
2728 pci_name(pdev));
4fe05bbc 2729 return NULL;
f76aec76 2730 }
ba395927
KA		 2731 }
2732
f76aec76
KA		 2733 return domain;
2734}
2735
147202aa
DW		 2736static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2737{
2738 struct device_domain_info *info;
2739
2740 /* No lock here, assumes no domain exit in normal case */
2741 info = dev->dev.archdata.iommu;
2742 if (likely(info))
2743 return info->domain;
2744
2745 return __get_valid_domain_for_dev(dev);
2746}
2747
2c2e2c38
FY		 2748static int iommu_dummy(struct pci_dev *pdev)
2749{
2750 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2751}
2752
2753/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2754static int iommu_no_mapping(struct device *dev)
2c2e2c38 2755{
73676832 2756 struct pci_dev *pdev;
2c2e2c38
FY		 2757 int found;
2758
73676832
DW		 2759 if (unlikely(dev->bus != &pci_bus_type))
2760 return 1;
2761
2762 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2763 if (iommu_dummy(pdev))
2764 return 1;
2765
2c2e2c38 2766 if (!iommu_identity_mapping)
1e4c64c4 2767 return 0;
2c2e2c38
FY		 2768
2769 found = identity_mapping(pdev);
2770 if (found) {
6941af28 2771 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2772 return 1;
2773 else {
2774 /*
2775 * 32 bit DMA is removed from si_domain and fall back
2776 * to non-identity mapping.
2777 */
2778 domain_remove_one_dev_info(si_domain, pdev);
2779 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2780 pci_name(pdev));
2781 return 0;
2782 }
2783 } else {
2784 /*
2785 * In case of a detached 64 bit DMA device from vm, the device
2786 * is put into si_domain for identity mapping.
2787 */
6941af28 2788 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2789 int ret;
5fe60f4e
DW		 2790 ret = domain_add_dev_info(si_domain, pdev,
2791 hw_pass_through ?
2792 CONTEXT_TT_PASS_THROUGH :
2793 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2794 if (!ret) {
2795 printk(KERN_INFO "64bit %s uses identity mapping\n",
2796 pci_name(pdev));
2797 return 1;
2798 }
2799 }
2800 }
2801
1e4c64c4 2802 return 0;
2c2e2c38
FY		 2803}
2804
bb9e6d65
FT		 2805static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2806 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2807{
2808 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2809 struct dmar_domain *domain;
5b6985ce 2810 phys_addr_t start_paddr;
f76aec76
KA		 2811 struct iova *iova;
2812 int prot = 0;
6865f0d1 2813 int ret;
8c11e798 2814 struct intel_iommu *iommu;
33041ec0 2815 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2816
2817 BUG_ON(dir == DMA_NONE);
2c2e2c38 2818
73676832 2819 if (iommu_no_mapping(hwdev))
6865f0d1 2820 return paddr;
f76aec76
KA		 2821
2822 domain = get_valid_domain_for_dev(pdev);
2823 if (!domain)
2824 return 0;
2825
8c11e798 2826 iommu = domain_get_iommu(domain);
88cb6a74 2827 size = aligned_nrpages(paddr, size);
f76aec76 2828
c681d0ba 2829 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2830 if (!iova)
2831 goto error;
2832
ba395927
KA		 2833 /*
2834 * Check if DMAR supports zero-length reads on write only
2835 * mappings..
2836 */
2837 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2838 !cap_zlr(iommu->cap))
ba395927
KA		 2839 prot |= DMA_PTE_READ;
2840 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2841 prot |= DMA_PTE_WRITE;
2842 /*
6865f0d1 2843 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2844 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2845 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2846 * is not a big problem
2847 */
0ab36de2 2848 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2849 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2850 if (ret)
2851 goto error;
2852
1f0ef2aa
DW		 2853 /* it's a non-present to present mapping. Only flush if caching mode */
2854 if (cap_caching_mode(iommu->cap))
82653633 2855 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2856 else
8c11e798 2857 iommu_flush_write_buffer(iommu);
f76aec76 2858
03d6a246
DW		 2859 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2860 start_paddr += paddr & ~PAGE_MASK;
2861 return start_paddr;
ba395927 2862
ba395927 2863error:
f76aec76
KA		 2864 if (iova)
2865 __free_iova(&domain->iovad, iova);
4cf2e75d 2866 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2867 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2868 return 0;
2869}
2870
ffbbef5c
FT		 2871static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2872 unsigned long offset, size_t size,
2873 enum dma_data_direction dir,
2874 struct dma_attrs *attrs)
bb9e6d65 2875{
ffbbef5c
FT		 2876 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2877 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2878}
2879
5e0d2a6f 2880static void flush_unmaps(void)
2881{
80b20dd8 2882 int i, j;
5e0d2a6f 2883
5e0d2a6f 2884 timer_on = 0;
2885
2886 /* just flush them all */
2887 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2888 struct intel_iommu *iommu = g_iommus[i];
2889 if (!iommu)
2890 continue;
c42d9f32 2891
9dd2fe89
YZ		 2892 if (!deferred_flush[i].next)
2893 continue;
2894
78d5f0f5
NA		 2895 /* In caching mode, global flushes turn emulation expensive */
2896 if (!cap_caching_mode(iommu->cap))
2897 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2898 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2899 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2900 unsigned long mask;
2901 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2902 struct dmar_domain *domain = deferred_flush[i].domain[j];
2903
2904 /* On real hardware multiple invalidations are expensive */
2905 if (cap_caching_mode(iommu->cap))
2906 iommu_flush_iotlb_psi(iommu, domain->id,
2907 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2908 else {
2909 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2910 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2911 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2912 }
93a23a72 2913 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2914 }
9dd2fe89 2915 deferred_flush[i].next = 0;
5e0d2a6f 2916 }
2917
5e0d2a6f 2918 list_size = 0;
5e0d2a6f 2919}
2920
2921static void flush_unmaps_timeout(unsigned long data)
2922{
80b20dd8 2923 unsigned long flags;
2924
2925 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2926 flush_unmaps();
80b20dd8 2927 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2928}
2929
2930static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2931{
2932 unsigned long flags;
80b20dd8 2933 int next, iommu_id;
8c11e798 2934 struct intel_iommu *iommu;
5e0d2a6f 2935
2936 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2937 if (list_size == HIGH_WATER_MARK)
2938 flush_unmaps();
2939
8c11e798
WH		 2940 iommu = domain_get_iommu(dom);
2941 iommu_id = iommu->seq_id;
c42d9f32 2942
80b20dd8 2943 next = deferred_flush[iommu_id].next;
2944 deferred_flush[iommu_id].domain[next] = dom;
2945 deferred_flush[iommu_id].iova[next] = iova;
2946 deferred_flush[iommu_id].next++;
5e0d2a6f 2947
2948 if (!timer_on) {
2949 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2950 timer_on = 1;
2951 }
2952 list_size++;
2953 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2954}
2955
ffbbef5c
FT		 2956static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2957 size_t size, enum dma_data_direction dir,
2958 struct dma_attrs *attrs)
ba395927 2959{
ba395927 2960 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2961 struct dmar_domain *domain;
d794dc9b 2962 unsigned long start_pfn, last_pfn;
ba395927 2963 struct iova *iova;
8c11e798 2964 struct intel_iommu *iommu;
ba395927 2965
73676832 2966 if (iommu_no_mapping(dev))
f76aec76 2967 return;
2c2e2c38 2968
ba395927
KA		 2969 domain = find_domain(pdev);
2970 BUG_ON(!domain);
2971
8c11e798
WH		 2972 iommu = domain_get_iommu(domain);
2973
ba395927 2974 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2975 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2976 (unsigned long long)dev_addr))
ba395927 2977 return;
ba395927 2978
d794dc9b
DW		 2979 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2980 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2981
d794dc9b
DW		 2982 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2983 pci_name(pdev), start_pfn, last_pfn);
ba395927 2984
f76aec76 2985 /* clear the whole page */
d794dc9b
DW		 2986 dma_pte_clear_range(domain, start_pfn, last_pfn);
2987
f76aec76 2988 /* free page tables */
d794dc9b
DW		 2989 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2990
5e0d2a6f 2991 if (intel_iommu_strict) {
03d6a246 2992 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2993 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2994 /* free iova */
2995 __free_iova(&domain->iovad, iova);
2996 } else {
2997 add_unmap(domain, iova);
2998 /*
2999 * queue up the release of the unmap to save the 1/6th of the
3000 * cpu used up by the iotlb flush operation...
3001 */
5e0d2a6f 3002 }
ba395927
KA		 3003}
3004
d7ab5c46 3005static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 3006 dma_addr_t *dma_handle, gfp_t flags,
3007 struct dma_attrs *attrs)
ba395927
KA		 3008{
3009 void *vaddr;
3010 int order;
3011
5b6985ce 3012 size = PAGE_ALIGN(size);
ba395927 3013 order = get_order(size);
e8bb910d
AW		 3014
3015 if (!iommu_no_mapping(hwdev))
3016 flags &= ~(GFP_DMA | GFP_DMA32);
3017 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3018 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3019 flags |= GFP_DMA;
3020 else
3021 flags |= GFP_DMA32;
3022 }
ba395927
KA		 3023
3024 vaddr = (void *)__get_free_pages(flags, order);
3025 if (!vaddr)
3026 return NULL;
3027 memset(vaddr, 0, size);
3028
bb9e6d65
FT		 3029 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3030 DMA_BIDIRECTIONAL,
3031 hwdev->coherent_dma_mask);
ba395927
KA		 3032 if (*dma_handle)
3033 return vaddr;
3034 free_pages((unsigned long)vaddr, order);
3035 return NULL;
3036}
3037
d7ab5c46 3038static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 3039 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3040{
3041 int order;
3042
5b6985ce 3043 size = PAGE_ALIGN(size);
ba395927
KA		 3044 order = get_order(size);
3045
0db9b7ae 3046 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3047 free_pages((unsigned long)vaddr, order);
3048}
3049
d7ab5c46
FT		 3050static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3051 int nelems, enum dma_data_direction dir,
3052 struct dma_attrs *attrs)
ba395927 3053{
ba395927
KA		 3054 struct pci_dev *pdev = to_pci_dev(hwdev);
3055 struct dmar_domain *domain;
d794dc9b 3056 unsigned long start_pfn, last_pfn;
f76aec76 3057 struct iova *iova;
8c11e798 3058 struct intel_iommu *iommu;
ba395927 3059
73676832 3060 if (iommu_no_mapping(hwdev))
ba395927
KA		 3061 return;
3062
3063 domain = find_domain(pdev);
8c11e798
WH		 3064 BUG_ON(!domain);
3065
3066 iommu = domain_get_iommu(domain);
ba395927 3067
c03ab37c 3068 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3069 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3070 (unsigned long long)sglist[0].dma_address))
f76aec76 3071 return;
f76aec76 3072
d794dc9b
DW		 3073 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3074 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 3075
3076 /* clear the whole page */
d794dc9b
DW		 3077 dma_pte_clear_range(domain, start_pfn, last_pfn);
3078
f76aec76 3079 /* free page tables */
d794dc9b 3080 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 3081
acea0018
DW		 3082 if (intel_iommu_strict) {
3083 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 3084 last_pfn - start_pfn + 1, 0);
acea0018
DW		 3085 /* free iova */
3086 __free_iova(&domain->iovad, iova);
3087 } else {
3088 add_unmap(domain, iova);
3089 /*
3090 * queue up the release of the unmap to save the 1/6th of the
3091 * cpu used up by the iotlb flush operation...
3092 */
3093 }
ba395927
KA		 3094}
3095
ba395927 3096static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3097 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3098{
3099 int i;
c03ab37c 3100 struct scatterlist *sg;
ba395927 3101
c03ab37c 3102 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3103 BUG_ON(!sg_page(sg));
4cf2e75d 3104 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3105 sg->dma_length = sg->length;
ba395927
KA		 3106 }
3107 return nelems;
3108}
3109
d7ab5c46
FT		 3110static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3111 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3112{
ba395927 3113 int i;
ba395927
KA		 3114 struct pci_dev *pdev = to_pci_dev(hwdev);
3115 struct dmar_domain *domain;
f76aec76
KA		 3116 size_t size = 0;
3117 int prot = 0;
f76aec76
KA		 3118 struct iova *iova = NULL;
3119 int ret;
c03ab37c 3120 struct scatterlist *sg;
b536d24d 3121 unsigned long start_vpfn;
8c11e798 3122 struct intel_iommu *iommu;
ba395927
KA		 3123
3124 BUG_ON(dir == DMA_NONE);
73676832 3125 if (iommu_no_mapping(hwdev))
c03ab37c 3126 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3127
f76aec76
KA		 3128 domain = get_valid_domain_for_dev(pdev);
3129 if (!domain)
3130 return 0;
3131
8c11e798
WH		 3132 iommu = domain_get_iommu(domain);
3133
b536d24d 3134 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3135 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3136
5a5e02a6
DW		 3137 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3138 pdev->dma_mask);
f76aec76 3139 if (!iova) {
c03ab37c 3140 sglist->dma_length = 0;
f76aec76
KA		 3141 return 0;
3142 }
3143
3144 /*
3145 * Check if DMAR supports zero-length reads on write only
3146 * mappings..
3147 */
3148 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3149 !cap_zlr(iommu->cap))
f76aec76
KA		 3150 prot |= DMA_PTE_READ;
3151 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3152 prot |= DMA_PTE_WRITE;
3153
b536d24d 3154 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3155
f532959b 3156 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3157 if (unlikely(ret)) {
3158 /* clear the page */
3159 dma_pte_clear_range(domain, start_vpfn,
3160 start_vpfn + size - 1);
3161 /* free page tables */
3162 dma_pte_free_pagetable(domain, start_vpfn,
3163 start_vpfn + size - 1);
3164 /* free iova */
3165 __free_iova(&domain->iovad, iova);
3166 return 0;
ba395927
KA		 3167 }
3168
1f0ef2aa
DW		 3169 /* it's a non-present to present mapping. Only flush if caching mode */
3170 if (cap_caching_mode(iommu->cap))
82653633 3171 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3172 else
8c11e798 3173 iommu_flush_write_buffer(iommu);
1f0ef2aa 3174
ba395927
KA		 3175 return nelems;
3176}
3177
dfb805e8
FT		 3178static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3179{
3180 return !dma_addr;
3181}
3182
160c1d8e 3183struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3184 .alloc = intel_alloc_coherent,
3185 .free = intel_free_coherent,
ba395927
KA		 3186 .map_sg = intel_map_sg,
3187 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3188 .map_page = intel_map_page,
3189 .unmap_page = intel_unmap_page,
dfb805e8 3190 .mapping_error = intel_mapping_error,
ba395927
KA		 3191};
3192
3193static inline int iommu_domain_cache_init(void)
3194{
3195 int ret = 0;
3196
3197 iommu_domain_cache = kmem_cache_create("iommu_domain",
3198 sizeof(struct dmar_domain),
3199 0,
3200 SLAB_HWCACHE_ALIGN,
3201
3202 NULL);
3203 if (!iommu_domain_cache) {
3204 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3205 ret = -ENOMEM;
3206 }
3207
3208 return ret;
3209}
3210
3211static inline int iommu_devinfo_cache_init(void)
3212{
3213 int ret = 0;
3214
3215 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3216 sizeof(struct device_domain_info),
3217 0,
3218 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3219 NULL);
3220 if (!iommu_devinfo_cache) {
3221 printk(KERN_ERR "Couldn't create devinfo cache\n");
3222 ret = -ENOMEM;
3223 }
3224
3225 return ret;
3226}
3227
3228static inline int iommu_iova_cache_init(void)
3229{
3230 int ret = 0;
3231
3232 iommu_iova_cache = kmem_cache_create("iommu_iova",
3233 sizeof(struct iova),
3234 0,
3235 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3236 NULL);
3237 if (!iommu_iova_cache) {
3238 printk(KERN_ERR "Couldn't create iova cache\n");
3239 ret = -ENOMEM;
3240 }
3241
3242 return ret;
3243}
3244
3245static int __init iommu_init_mempool(void)
3246{
3247 int ret;
3248 ret = iommu_iova_cache_init();
3249 if (ret)
3250 return ret;
3251
3252 ret = iommu_domain_cache_init();
3253 if (ret)
3254 goto domain_error;
3255
3256 ret = iommu_devinfo_cache_init();
3257 if (!ret)
3258 return ret;
3259
3260 kmem_cache_destroy(iommu_domain_cache);
3261domain_error:
3262 kmem_cache_destroy(iommu_iova_cache);
3263
3264 return -ENOMEM;
3265}
3266
3267static void __init iommu_exit_mempool(void)
3268{
3269 kmem_cache_destroy(iommu_devinfo_cache);
3270 kmem_cache_destroy(iommu_domain_cache);
3271 kmem_cache_destroy(iommu_iova_cache);
3272
3273}
3274
556ab45f
DW		 3275static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3276{
3277 struct dmar_drhd_unit *drhd;
3278 u32 vtbar;
3279 int rc;
3280
3281 /* We know that this device on this chipset has its own IOMMU.
3282 * If we find it under a different IOMMU, then the BIOS is lying
3283 * to us. Hope that the IOMMU for this device is actually
3284 * disabled, and it needs no translation...
3285 */
3286 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3287 if (rc) {
3288 /* "can't" happen */
3289 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3290 return;
3291 }
3292 vtbar &= 0xffff0000;
3293
3294 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3295 drhd = dmar_find_matched_drhd_unit(pdev);
3296 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3297 TAINT_FIRMWARE_WORKAROUND,
3298 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3299 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3300}
3301DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3302
ba395927
KA		 3303static void __init init_no_remapping_devices(void)
3304{
3305 struct dmar_drhd_unit *drhd;
3306
3307 for_each_drhd_unit(drhd) {
3308 if (!drhd->include_all) {
3309 int i;
3310 for (i = 0; i < drhd->devices_cnt; i++)
3311 if (drhd->devices[i] != NULL)
3312 break;
3313 /* ignore DMAR unit if no pci devices exist */
3314 if (i == drhd->devices_cnt)
3315 drhd->ignored = 1;
3316 }
3317 }
3318
ba395927
KA		 3319 for_each_drhd_unit(drhd) {
3320 int i;
3321 if (drhd->ignored || drhd->include_all)
3322 continue;
3323
3324 for (i = 0; i < drhd->devices_cnt; i++)
3325 if (drhd->devices[i] &&
c0771df8 3326 !IS_GFX_DEVICE(drhd->devices[i]))
ba395927
KA		 3327 break;
3328
3329 if (i < drhd->devices_cnt)
3330 continue;
3331
c0771df8
DW		 3332 /* This IOMMU has *only* gfx devices. Either bypass it or
3333 set the gfx_mapped flag, as appropriate */
3334 if (dmar_map_gfx) {
3335 intel_iommu_gfx_mapped = 1;
3336 } else {
3337 drhd->ignored = 1;
3338 for (i = 0; i < drhd->devices_cnt; i++) {
3339 if (!drhd->devices[i])
3340 continue;
3341 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3342 }
ba395927
KA		 3343 }
3344 }
3345}
3346
f59c7b69
FY		 3347#ifdef CONFIG_SUSPEND
3348static int init_iommu_hw(void)
3349{
3350 struct dmar_drhd_unit *drhd;
3351 struct intel_iommu *iommu = NULL;
3352
3353 for_each_active_iommu(iommu, drhd)
3354 if (iommu->qi)
3355 dmar_reenable_qi(iommu);
3356
b779260b
JC		 3357 for_each_iommu(iommu, drhd) {
3358 if (drhd->ignored) {
3359 /*
3360 * we always have to disable PMRs or DMA may fail on
3361 * this device
3362 */
3363 if (force_on)
3364 iommu_disable_protect_mem_regions(iommu);
3365 continue;
3366 }
3367
f59c7b69
FY		 3368 iommu_flush_write_buffer(iommu);
3369
3370 iommu_set_root_entry(iommu);
3371
3372 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3373 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3374 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3375 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3376 if (iommu_enable_translation(iommu))
3377 return 1;
b94996c9 3378 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3379 }
3380
3381 return 0;
3382}
3383
3384static void iommu_flush_all(void)
3385{
3386 struct dmar_drhd_unit *drhd;
3387 struct intel_iommu *iommu;
3388
3389 for_each_active_iommu(iommu, drhd) {
3390 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3391 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3392 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3393 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3394 }
3395}
3396
134fac3f 3397static int iommu_suspend(void)
f59c7b69
FY		 3398{
3399 struct dmar_drhd_unit *drhd;
3400 struct intel_iommu *iommu = NULL;
3401 unsigned long flag;
3402
3403 for_each_active_iommu(iommu, drhd) {
3404 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3405 GFP_ATOMIC);
3406 if (!iommu->iommu_state)
3407 goto nomem;
3408 }
3409
3410 iommu_flush_all();
3411
3412 for_each_active_iommu(iommu, drhd) {
3413 iommu_disable_translation(iommu);
3414
1f5b3c3f 3415 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3416
3417 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3418 readl(iommu->reg + DMAR_FECTL_REG);
3419 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3420 readl(iommu->reg + DMAR_FEDATA_REG);
3421 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3422 readl(iommu->reg + DMAR_FEADDR_REG);
3423 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3424 readl(iommu->reg + DMAR_FEUADDR_REG);
3425
1f5b3c3f 3426 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3427 }
3428 return 0;
3429
3430nomem:
3431 for_each_active_iommu(iommu, drhd)
3432 kfree(iommu->iommu_state);
3433
3434 return -ENOMEM;
3435}
3436
134fac3f 3437static void iommu_resume(void)
f59c7b69
FY		 3438{
3439 struct dmar_drhd_unit *drhd;
3440 struct intel_iommu *iommu = NULL;
3441 unsigned long flag;
3442
3443 if (init_iommu_hw()) {
b779260b
JC		 3444 if (force_on)
3445 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3446 else
3447 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3448 return;
f59c7b69
FY		 3449 }
3450
3451 for_each_active_iommu(iommu, drhd) {
3452
1f5b3c3f 3453 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3454
3455 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3456 iommu->reg + DMAR_FECTL_REG);
3457 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3458 iommu->reg + DMAR_FEDATA_REG);
3459 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3460 iommu->reg + DMAR_FEADDR_REG);
3461 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3462 iommu->reg + DMAR_FEUADDR_REG);
3463
1f5b3c3f 3464 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3465 }
3466
3467 for_each_active_iommu(iommu, drhd)
3468 kfree(iommu->iommu_state);
f59c7b69
FY		 3469}
3470
134fac3f 3471static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3472 .resume = iommu_resume,
3473 .suspend = iommu_suspend,
3474};
3475
134fac3f 3476static void __init init_iommu_pm_ops(void)
f59c7b69 3477{
134fac3f 3478 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3479}
3480
3481#else
99592ba4 3482static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3483#endif /* CONFIG_PM */
3484
318fe7df
SS		 3485LIST_HEAD(dmar_rmrr_units);
3486
3487static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3488{
3489 list_add(&rmrr->list, &dmar_rmrr_units);
3490}
3491
3492
3493int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3494{
3495 struct acpi_dmar_reserved_memory *rmrr;
3496 struct dmar_rmrr_unit *rmrru;
3497
3498 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3499 if (!rmrru)
3500 return -ENOMEM;
3501
3502 rmrru->hdr = header;
3503 rmrr = (struct acpi_dmar_reserved_memory *)header;
3504 rmrru->base_address = rmrr->base_address;
3505 rmrru->end_address = rmrr->end_address;
3506
3507 dmar_register_rmrr_unit(rmrru);
3508 return 0;
3509}
3510
3511static int __init
3512rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3513{
3514 struct acpi_dmar_reserved_memory *rmrr;
3515 int ret;
3516
3517 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3518 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3519 ((void *)rmrr) + rmrr->header.length,
3520 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3521
3522 if (ret || (rmrru->devices_cnt == 0)) {
3523 list_del(&rmrru->list);
3524 kfree(rmrru);
3525 }
3526 return ret;
3527}
3528
3529static LIST_HEAD(dmar_atsr_units);
3530
3531int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3532{
3533 struct acpi_dmar_atsr *atsr;
3534 struct dmar_atsr_unit *atsru;
3535
3536 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3537 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3538 if (!atsru)
3539 return -ENOMEM;
3540
3541 atsru->hdr = hdr;
3542 atsru->include_all = atsr->flags & 0x1;
3543
3544 list_add(&atsru->list, &dmar_atsr_units);
3545
3546 return 0;
3547}
3548
3549static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3550{
3551 int rc;
3552 struct acpi_dmar_atsr *atsr;
3553
3554 if (atsru->include_all)
3555 return 0;
3556
3557 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3558 rc = dmar_parse_dev_scope((void *)(atsr + 1),
3559 (void *)atsr + atsr->header.length,
3560 &atsru->devices_cnt, &atsru->devices,
3561 atsr->segment);
3562 if (rc || !atsru->devices_cnt) {
3563 list_del(&atsru->list);
3564 kfree(atsru);
3565 }
3566
3567 return rc;
3568}
3569
3570int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3571{
3572 int i;
3573 struct pci_bus *bus;
3574 struct acpi_dmar_atsr *atsr;
3575 struct dmar_atsr_unit *atsru;
3576
3577 dev = pci_physfn(dev);
3578
3579 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3580 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3581 if (atsr->segment == pci_domain_nr(dev->bus))
3582 goto found;
3583 }
3584
3585 return 0;
3586
3587found:
3588 for (bus = dev->bus; bus; bus = bus->parent) {
3589 struct pci_dev *bridge = bus->self;
3590
3591 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3592 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df
SS		 3593 return 0;
3594
62f87c0e 3595 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
318fe7df
SS		 3596 for (i = 0; i < atsru->devices_cnt; i++)
3597 if (atsru->devices[i] == bridge)
3598 return 1;
3599 break;
3600 }
3601 }
3602
3603 if (atsru->include_all)
3604 return 1;
3605
3606 return 0;
3607}
3608
c8f369ab 3609int __init dmar_parse_rmrr_atsr_dev(void)
318fe7df
SS		 3610{
3611 struct dmar_rmrr_unit *rmrr, *rmrr_n;
3612 struct dmar_atsr_unit *atsr, *atsr_n;
3613 int ret = 0;
3614
3615 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3616 ret = rmrr_parse_dev(rmrr);
3617 if (ret)
3618 return ret;
3619 }
3620
3621 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3622 ret = atsr_parse_dev(atsr);
3623 if (ret)
3624 return ret;
3625 }
3626
3627 return ret;
3628}
3629
99dcaded
FY		 3630/*
3631 * Here we only respond to action of unbound device from driver.
3632 *
3633 * Added device is not attached to its DMAR domain here yet. That will happen
3634 * when mapping the device to iova.
3635 */
3636static int device_notifier(struct notifier_block *nb,
3637 unsigned long action, void *data)
3638{
3639 struct device *dev = data;
3640 struct pci_dev *pdev = to_pci_dev(dev);
3641 struct dmar_domain *domain;
3642
44cd613c
DW		 3643 if (iommu_no_mapping(dev))
3644 return 0;
3645
99dcaded
FY		 3646 domain = find_domain(pdev);
3647 if (!domain)
3648 return 0;
3649
a97590e5 3650 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3651 domain_remove_one_dev_info(domain, pdev);
3652
a97590e5
AW		 3653 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3654 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3655 list_empty(&domain->devices))
3656 domain_exit(domain);
3657 }
3658
99dcaded
FY		 3659 return 0;
3660}
3661
3662static struct notifier_block device_nb = {
3663 .notifier_call = device_notifier,
3664};
3665
ba395927
KA		 3666int __init intel_iommu_init(void)
3667{
3668 int ret = 0;
3a93c841 3669 struct dmar_drhd_unit *drhd;
ba395927 3670
a59b50e9
JC		 3671 /* VT-d is required for a TXT/tboot launch, so enforce that */
3672 force_on = tboot_force_iommu();
3673
3674 if (dmar_table_init()) {
3675 if (force_on)
3676 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3677 return -ENODEV;
a59b50e9 3678 }
ba395927 3679
3a93c841
TI		 3680 /*
3681 * Disable translation if already enabled prior to OS handover.
3682 */
3683 for_each_drhd_unit(drhd) {
3684 struct intel_iommu *iommu;
3685
3686 if (drhd->ignored)
3687 continue;
3688
3689 iommu = drhd->iommu;
3690 if (iommu->gcmd & DMA_GCMD_TE)
3691 iommu_disable_translation(iommu);
3692 }
3693
c2c7286a 3694 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3695 if (force_on)
3696 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3697 return -ENODEV;
a59b50e9 3698 }
1886e8a9 3699
75f1cdf1 3700 if (no_iommu || dmar_disabled)
2ae21010
SS		 3701 return -ENODEV;
3702
51a63e67
JC		 3703 if (iommu_init_mempool()) {
3704 if (force_on)
3705 panic("tboot: Failed to initialize iommu memory\n");
3706 return -ENODEV;
3707 }
3708
318fe7df
SS		 3709 if (list_empty(&dmar_rmrr_units))
3710 printk(KERN_INFO "DMAR: No RMRR found\n");
3711
3712 if (list_empty(&dmar_atsr_units))
3713 printk(KERN_INFO "DMAR: No ATSR found\n");
3714
51a63e67
JC		 3715 if (dmar_init_reserved_ranges()) {
3716 if (force_on)
3717 panic("tboot: Failed to reserve iommu ranges\n");
3718 return -ENODEV;
3719 }
ba395927
KA		 3720
3721 init_no_remapping_devices();
3722
b779260b 3723 ret = init_dmars();
ba395927 3724 if (ret) {
a59b50e9
JC		 3725 if (force_on)
3726 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3727 printk(KERN_ERR "IOMMU: dmar init failed\n");
3728 put_iova_domain(&reserved_iova_list);
3729 iommu_exit_mempool();
3730 return ret;
3731 }
3732 printk(KERN_INFO
3733 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3734
5e0d2a6f 3735 init_timer(&unmap_timer);
75f1cdf1
FT		 3736#ifdef CONFIG_SWIOTLB
3737 swiotlb = 0;
3738#endif
19943b0e 3739 dma_ops = &intel_dma_ops;
4ed0d3e6 3740
134fac3f 3741 init_iommu_pm_ops();
a8bcbb0d 3742
4236d97d 3743 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
a8bcbb0d 3744
99dcaded
FY		 3745 bus_register_notifier(&pci_bus_type, &device_nb);
3746
8bc1f85c
ED		 3747 intel_iommu_enabled = 1;
3748
ba395927
KA		 3749 return 0;
3750}
e820482c 3751
3199aa6b
HW		 3752static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3753 struct pci_dev *pdev)
3754{
3755 struct pci_dev *tmp, *parent;
3756
3757 if (!iommu || !pdev)
3758 return;
3759
3760 /* dependent device detach */
3761 tmp = pci_find_upstream_pcie_bridge(pdev);
3762 /* Secondary interface's bus number and devfn 0 */
3763 if (tmp) {
3764 parent = pdev->bus->self;
3765 while (parent != tmp) {
3766 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3767 parent->devfn);
3199aa6b
HW		 3768 parent = parent->bus->self;
3769 }
45e829ea 3770 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3771 iommu_detach_dev(iommu,
3772 tmp->subordinate->number, 0);
3773 else /* this is a legacy PCI bridge */
276dbf99
DW		 3774 iommu_detach_dev(iommu, tmp->bus->number,
3775 tmp->devfn);
3199aa6b
HW		 3776 }
3777}
3778
2c2e2c38 3779static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3780 struct pci_dev *pdev)
3781{
3782 struct device_domain_info *info;
3783 struct intel_iommu *iommu;
3784 unsigned long flags;
3785 int found = 0;
3786 struct list_head *entry, *tmp;
3787
276dbf99
DW		 3788 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3789 pdev->devfn);
c7151a8d
WH		 3790 if (!iommu)
3791 return;
3792
3793 spin_lock_irqsave(&device_domain_lock, flags);
3794 list_for_each_safe(entry, tmp, &domain->devices) {
3795 info = list_entry(entry, struct device_domain_info, link);
8519dc44
MH		 3796 if (info->segment == pci_domain_nr(pdev->bus) &&
3797 info->bus == pdev->bus->number &&
c7151a8d 3798 info->devfn == pdev->devfn) {
109b9b04 3799 unlink_domain_info(info);
c7151a8d
WH		 3800 spin_unlock_irqrestore(&device_domain_lock, flags);
3801
93a23a72 3802 iommu_disable_dev_iotlb(info);
c7151a8d 3803 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3804 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3805 free_devinfo_mem(info);
3806
3807 spin_lock_irqsave(&device_domain_lock, flags);
3808
3809 if (found)
3810 break;
3811 else
3812 continue;
3813 }
3814
3815 /* if there is no other devices under the same iommu
3816 * owned by this domain, clear this iommu in iommu_bmp
3817 * update iommu count and coherency
3818 */
276dbf99
DW		 3819 if (iommu == device_to_iommu(info->segment, info->bus,
3820 info->devfn))
c7151a8d
WH		 3821 found = 1;
3822 }
3823
3e7abe25
RD		 3824 spin_unlock_irqrestore(&device_domain_lock, flags);
3825
c7151a8d
WH		 3826 if (found == 0) {
3827 unsigned long tmp_flags;
3828 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 3829 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 3830 domain->iommu_count--;
58c610bd 3831 domain_update_iommu_cap(domain);
c7151a8d 3832 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3833
9b4554b2
AW		 3834 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3835 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3836 spin_lock_irqsave(&iommu->lock, tmp_flags);
3837 clear_bit(domain->id, iommu->domain_ids);
3838 iommu->domains[domain->id] = NULL;
3839 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3840 }
c7151a8d 3841 }
c7151a8d
WH		 3842}
3843
3844static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3845{
3846 struct device_domain_info *info;
3847 struct intel_iommu *iommu;
3848 unsigned long flags1, flags2;
3849
3850 spin_lock_irqsave(&device_domain_lock, flags1);
3851 while (!list_empty(&domain->devices)) {
3852 info = list_entry(domain->devices.next,
3853 struct device_domain_info, link);
109b9b04 3854 unlink_domain_info(info);
c7151a8d
WH		 3855 spin_unlock_irqrestore(&device_domain_lock, flags1);
3856
93a23a72 3857 iommu_disable_dev_iotlb(info);
276dbf99 3858 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3859 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3860 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3861
3862 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3863 * and capabilities
c7151a8d
WH		 3864 */
3865 spin_lock_irqsave(&domain->iommu_lock, flags2);
3866 if (test_and_clear_bit(iommu->seq_id,
1b198bb0 3867 domain->iommu_bmp)) {
c7151a8d 3868 domain->iommu_count--;
58c610bd 3869 domain_update_iommu_cap(domain);
c7151a8d
WH		 3870 }
3871 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3872
3873 free_devinfo_mem(info);
3874 spin_lock_irqsave(&device_domain_lock, flags1);
3875 }
3876 spin_unlock_irqrestore(&device_domain_lock, flags1);
3877}
3878
5e98c4b1
WH		 3879/* domain id for virtual machine, it won't be set in context */
3880static unsigned long vm_domid;
3881
3882static struct dmar_domain *iommu_alloc_vm_domain(void)
3883{
3884 struct dmar_domain *domain;
3885
3886 domain = alloc_domain_mem();
3887 if (!domain)
3888 return NULL;
3889
3890 domain->id = vm_domid++;
4c923d47 3891 domain->nid = -1;
1b198bb0 3892 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
5e98c4b1
WH		 3893 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3894
3895 return domain;
3896}
3897
2c2e2c38 3898static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3899{
3900 int adjust_width;
3901
3902 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3903 spin_lock_init(&domain->iommu_lock);
3904
3905 domain_reserve_special_ranges(domain);
3906
3907 /* calculate AGAW */
3908 domain->gaw = guest_width;
3909 adjust_width = guestwidth_to_adjustwidth(guest_width);
3910 domain->agaw = width_to_agaw(adjust_width);
3911
3912 INIT_LIST_HEAD(&domain->devices);
3913
3914 domain->iommu_count = 0;
3915 domain->iommu_coherency = 0;
c5b15255 3916 domain->iommu_snooping = 0;
6dd9a7c7 3917 domain->iommu_superpage = 0;
fe40f1e0 3918 domain->max_addr = 0;
4c923d47 3919 domain->nid = -1;
5e98c4b1
WH		 3920
3921 /* always allocate the top pgd */
4c923d47 3922 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3923 if (!domain->pgd)
3924 return -ENOMEM;
3925 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3926 return 0;
3927}
3928
3929static void iommu_free_vm_domain(struct dmar_domain *domain)
3930{
3931 unsigned long flags;
3932 struct dmar_drhd_unit *drhd;
3933 struct intel_iommu *iommu;
3934 unsigned long i;
3935 unsigned long ndomains;
3936
3937 for_each_drhd_unit(drhd) {
3938 if (drhd->ignored)
3939 continue;
3940 iommu = drhd->iommu;
3941
3942 ndomains = cap_ndoms(iommu->cap);
a45946ab 3943 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3944 if (iommu->domains[i] == domain) {
3945 spin_lock_irqsave(&iommu->lock, flags);
3946 clear_bit(i, iommu->domain_ids);
3947 iommu->domains[i] = NULL;
3948 spin_unlock_irqrestore(&iommu->lock, flags);
3949 break;
3950 }
5e98c4b1
WH		 3951 }
3952 }
3953}
3954
3955static void vm_domain_exit(struct dmar_domain *domain)
3956{
5e98c4b1
WH		 3957 /* Domain 0 is reserved, so dont process it */
3958 if (!domain)
3959 return;
3960
3961 vm_domain_remove_all_dev_info(domain);
3962 /* destroy iovas */
3963 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3964
3965 /* clear ptes */
595badf5 3966 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3967
3968 /* free page tables */
d794dc9b 3969 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3970
3971 iommu_free_vm_domain(domain);
3972 free_domain_mem(domain);
3973}
3974
5d450806 3975static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3976{
5d450806 3977 struct dmar_domain *dmar_domain;
38717946 3978
5d450806
JR		 3979 dmar_domain = iommu_alloc_vm_domain();
3980 if (!dmar_domain) {
38717946 3981 printk(KERN_ERR
5d450806
JR		 3982 "intel_iommu_domain_init: dmar_domain == NULL\n");
3983 return -ENOMEM;
38717946 3984 }
2c2e2c38 3985 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3986 printk(KERN_ERR
5d450806
JR		 3987 "intel_iommu_domain_init() failed\n");
3988 vm_domain_exit(dmar_domain);
3989 return -ENOMEM;
38717946 3990 }
8140a95d 3991 domain_update_iommu_cap(dmar_domain);
5d450806 3992 domain->priv = dmar_domain;
faa3d6f5 3993
8a0e715b
JR		 3994 domain->geometry.aperture_start = 0;
3995 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3996 domain->geometry.force_aperture = true;
3997
5d450806 3998 return 0;
38717946 3999}
38717946 4000
5d450806 4001static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4002{
5d450806
JR		 4003 struct dmar_domain *dmar_domain = domain->priv;
4004
4005 domain->priv = NULL;
4006 vm_domain_exit(dmar_domain);
38717946 4007}
38717946 4008
4c5478c9
JR		 4009static int intel_iommu_attach_device(struct iommu_domain *domain,
4010 struct device *dev)
38717946 4011{
4c5478c9
JR		 4012 struct dmar_domain *dmar_domain = domain->priv;
4013 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 4014 struct intel_iommu *iommu;
4015 int addr_width;
faa3d6f5
WH		 4016
4017 /* normally pdev is not mapped */
4018 if (unlikely(domain_context_mapped(pdev))) {
4019 struct dmar_domain *old_domain;
4020
4021 old_domain = find_domain(pdev);
4022 if (old_domain) {
2c2e2c38
FY		 4023 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4024 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4025 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 4026 else
4027 domain_remove_dev_info(old_domain);
4028 }
4029 }
4030
276dbf99
DW		 4031 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4032 pdev->devfn);
fe40f1e0
WH		 4033 if (!iommu)
4034 return -ENODEV;
4035
4036 /* check if this iommu agaw is sufficient for max mapped address */
4037 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4038 if (addr_width > cap_mgaw(iommu->cap))
4039 addr_width = cap_mgaw(iommu->cap);
4040
4041 if (dmar_domain->max_addr > (1LL << addr_width)) {
4042 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4043 "sufficient for the mapped address (%llx)\n",
a99c47a2 4044 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4045 return -EFAULT;
4046 }
a99c47a2
TL		 4047 dmar_domain->gaw = addr_width;
4048
4049 /*
4050 * Knock out extra levels of page tables if necessary
4051 */
4052 while (iommu->agaw < dmar_domain->agaw) {
4053 struct dma_pte *pte;
4054
4055 pte = dmar_domain->pgd;
4056 if (dma_pte_present(pte)) {
25cbff16
SY		 4057 dmar_domain->pgd = (struct dma_pte *)
4058 phys_to_virt(dma_pte_addr(pte));
7a661013 4059 free_pgtable_page(pte);
a99c47a2
TL		 4060 }
4061 dmar_domain->agaw--;
4062 }
fe40f1e0 4063
5fe60f4e 4064 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4065}
38717946 4066
4c5478c9
JR		 4067static void intel_iommu_detach_device(struct iommu_domain *domain,
4068 struct device *dev)
38717946 4069{
4c5478c9
JR		 4070 struct dmar_domain *dmar_domain = domain->priv;
4071 struct pci_dev *pdev = to_pci_dev(dev);
4072
2c2e2c38 4073 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4074}
c7151a8d 4075
b146a1c9
JR		 4076static int intel_iommu_map(struct iommu_domain *domain,
4077 unsigned long iova, phys_addr_t hpa,
5009065d 4078 size_t size, int iommu_prot)
faa3d6f5 4079{
dde57a21 4080 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4081 u64 max_addr;
dde57a21 4082 int prot = 0;
faa3d6f5 4083 int ret;
fe40f1e0 4084
dde57a21
JR		 4085 if (iommu_prot & IOMMU_READ)
4086 prot |= DMA_PTE_READ;
4087 if (iommu_prot & IOMMU_WRITE)
4088 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4089 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4090 prot |= DMA_PTE_SNP;
dde57a21 4091
163cc52c 4092 max_addr = iova + size;
dde57a21 4093 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4094 u64 end;
4095
4096 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4097 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4098 if (end < max_addr) {
8954da1f 4099 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4100 "sufficient for the mapped address (%llx)\n",
8954da1f 4101 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4102 return -EFAULT;
4103 }
dde57a21 4104 dmar_domain->max_addr = max_addr;
fe40f1e0 4105 }
ad051221
DW		 4106 /* Round up size to next multiple of PAGE_SIZE, if it and
4107 the low bits of hpa would take us onto the next page */
88cb6a74 4108 size = aligned_nrpages(hpa, size);
ad051221
DW		 4109 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4110 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4111 return ret;
38717946 4112}
38717946 4113
5009065d
OBC		 4114static size_t intel_iommu_unmap(struct iommu_domain *domain,
4115 unsigned long iova, size_t size)
38717946 4116{
dde57a21 4117 struct dmar_domain *dmar_domain = domain->priv;
292827cb 4118 int order;
4b99d352 4119
292827cb 4120 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
163cc52c 4121 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 4122
163cc52c
DW		 4123 if (dmar_domain->max_addr == iova + size)
4124 dmar_domain->max_addr = iova;
b146a1c9 4125
5009065d 4126 return PAGE_SIZE << order;
38717946 4127}
38717946 4128
d14d6577 4129static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4130 dma_addr_t iova)
38717946 4131{
d14d6577 4132 struct dmar_domain *dmar_domain = domain->priv;
38717946 4133 struct dma_pte *pte;
faa3d6f5 4134 u64 phys = 0;
38717946 4135
6dd9a7c7 4136 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 4137 if (pte)
faa3d6f5 4138 phys = dma_pte_addr(pte);
38717946 4139
faa3d6f5 4140 return phys;
38717946 4141}
a8bcbb0d 4142
dbb9fd86
SY		 4143static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4144 unsigned long cap)
4145{
4146 struct dmar_domain *dmar_domain = domain->priv;
4147
4148 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4149 return dmar_domain->iommu_snooping;
323f99cb 4150 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4151 return irq_remapping_enabled;
dbb9fd86
SY		 4152
4153 return 0;
4154}
4155
783f157b 4156#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4157
abdfdde2
AW		 4158static int intel_iommu_add_device(struct device *dev)
4159{
4160 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4161 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4162 struct iommu_group *group;
4163 int ret;
70ae6f0d 4164
abdfdde2
AW		 4165 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4166 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4167 return -ENODEV;
4168
4169 bridge = pci_find_upstream_pcie_bridge(pdev);
4170 if (bridge) {
abdfdde2
AW		 4171 if (pci_is_pcie(bridge))
4172 dma_pdev = pci_get_domain_bus_and_slot(
4173 pci_domain_nr(pdev->bus),
4174 bridge->subordinate->number, 0);
3da4af0a 4175 if (!dma_pdev)
abdfdde2
AW		 4176 dma_pdev = pci_dev_get(bridge);
4177 } else
4178 dma_pdev = pci_dev_get(pdev);
4179
a4ff1fc2 4180 /* Account for quirked devices */
783f157b
AW		 4181 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4182
a4ff1fc2
AW		 4183 /*
4184 * If it's a multifunction device that does not support our
c14d2690
AW		 4185 * required ACS flags, add to the same group as lowest numbered
4186 * function that also does not suport the required ACS flags.
a4ff1fc2 4187 */
783f157b 4188 if (dma_pdev->multifunction &&
c14d2690
AW		 4189 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4190 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4191
4192 for (i = 0; i < 8; i++) {
4193 struct pci_dev *tmp;
4194
4195 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4196 if (!tmp)
4197 continue;
4198
4199 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4200 swap_pci_ref(&dma_pdev, tmp);
4201 break;
4202 }
4203 pci_dev_put(tmp);
4204 }
4205 }
783f157b 4206
a4ff1fc2
AW		 4207 /*
4208 * Devices on the root bus go through the iommu. If that's not us,
4209 * find the next upstream device and test ACS up to the root bus.
4210 * Finding the next device may require skipping virtual buses.
4211 */
783f157b 4212 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4213 struct pci_bus *bus = dma_pdev->bus;
4214
4215 while (!bus->self) {
4216 if (!pci_is_root_bus(bus))
4217 bus = bus->parent;
4218 else
4219 goto root_bus;
4220 }
4221
4222 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4223 break;
4224
a4ff1fc2 4225 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4226 }
4227
a4ff1fc2 4228root_bus:
abdfdde2
AW		 4229 group = iommu_group_get(&dma_pdev->dev);
4230 pci_dev_put(dma_pdev);
4231 if (!group) {
4232 group = iommu_group_alloc();
4233 if (IS_ERR(group))
4234 return PTR_ERR(group);
70ae6f0d
AW		 4235 }
4236
abdfdde2 4237 ret = iommu_group_add_device(group, dev);
bcb71abe 4238
abdfdde2
AW		 4239 iommu_group_put(group);
4240 return ret;
4241}
70ae6f0d 4242
abdfdde2
AW		 4243static void intel_iommu_remove_device(struct device *dev)
4244{
4245 iommu_group_remove_device(dev);
70ae6f0d
AW		 4246}
4247
a8bcbb0d
JR		 4248static struct iommu_ops intel_iommu_ops = {
4249 .domain_init = intel_iommu_domain_init,
4250 .domain_destroy = intel_iommu_domain_destroy,
4251 .attach_dev = intel_iommu_attach_device,
4252 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4253 .map = intel_iommu_map,
4254 .unmap = intel_iommu_unmap,
a8bcbb0d 4255 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4256 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4257 .add_device = intel_iommu_add_device,
4258 .remove_device = intel_iommu_remove_device,
6d1c56a9 4259 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4260};
9af88143 4261
9452618e
DV		 4262static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4263{
4264 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4265 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4266 dmar_map_gfx = 0;
4267}
4268
4269DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4270DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4271DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4272DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4273DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4274DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4275DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4276
d34d6517 4277static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4278{
4279 /*
4280 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4281 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4282 */
4283 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4284 rwbf_quirk = 1;
4285}
4286
4287DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4288DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4289DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4290DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4291DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4292DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4293DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4294
eecfd57f
AJ		 4295#define GGC 0x52
4296#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4297#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4298#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4299#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4300#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4301#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4302#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4303#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4304
d34d6517 4305static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4306{
4307 unsigned short ggc;
4308
eecfd57f 4309 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4310 return;
4311
eecfd57f 4312 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4313 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4314 dmar_map_gfx = 0;
6fbcfb3e
DW		 4315 } else if (dmar_map_gfx) {
4316 /* we have to ensure the gfx device is idle before we flush */
4317 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4318 intel_iommu_strict = 1;
4319 }
9eecabcb
DW		 4320}
4321DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4322DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4323DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4324DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4325
e0fc7e0b
DW		 4326/* On Tylersburg chipsets, some BIOSes have been known to enable the
4327 ISOCH DMAR unit for the Azalia sound device, but not give it any
4328 TLB entries, which causes it to deadlock. Check for that. We do
4329 this in a function called from init_dmars(), instead of in a PCI
4330 quirk, because we don't want to print the obnoxious "BIOS broken"
4331 message if VT-d is actually disabled.
4332*/
4333static void __init check_tylersburg_isoch(void)
4334{
4335 struct pci_dev *pdev;
4336 uint32_t vtisochctrl;
4337
4338 /* If there's no Azalia in the system anyway, forget it. */
4339 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4340 if (!pdev)
4341 return;
4342 pci_dev_put(pdev);
4343
4344 /* System Management Registers. Might be hidden, in which case
4345 we can't do the sanity check. But that's OK, because the
4346 known-broken BIOSes _don't_ actually hide it, so far. */
4347 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4348 if (!pdev)
4349 return;
4350
4351 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4352 pci_dev_put(pdev);
4353 return;
4354 }
4355
4356 pci_dev_put(pdev);
4357
4358 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4359 if (vtisochctrl & 1)
4360 return;
4361
4362 /* Drop all bits other than the number of TLB entries */
4363 vtisochctrl &= 0x1c;
4364
4365 /* If we have the recommended number of TLB entries (16), fine. */
4366 if (vtisochctrl == 0x10)
4367 return;
4368
4369 /* Zero TLB entries? You get to ride the short bus to school. */
4370 if (!vtisochctrl) {
4371 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4372 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4373 dmi_get_system_info(DMI_BIOS_VENDOR),
4374 dmi_get_system_info(DMI_BIOS_VERSION),
4375 dmi_get_system_info(DMI_PRODUCT_VERSION));
4376 iommu_identity_mapping |= IDENTMAP_AZALIA;
4377 return;
4378 }
4379
4380 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4381 vtisochctrl);
4382}
Linux 6.x block layer and io_uring tree(s)