projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branches 'dma-debug/next', 'amd-iommu/command-cleanups', 'amd-iommu/ats' and...
[linux-2.6-block.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
134fac3f 39#include <linux/syscore_ops.h>
69575d38 40#include <linux/tboot.h>
adb2fe02 41#include <linux/dmi.h>
5cdede24 42#include <linux/pci-ats.h>
ba395927 43#include <asm/cacheflush.h>
46a7fa27 44#include <asm/iommu.h>
ba395927
KA		 45#include "pci.h"
46
5b6985ce
FY		 47#define ROOT_SIZE VTD_PAGE_SIZE
48#define CONTEXT_SIZE VTD_PAGE_SIZE
49
ba395927
KA		 50#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
51#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 52#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 53
54#define IOAPIC_RANGE_START (0xfee00000)
55#define IOAPIC_RANGE_END (0xfeefffff)
56#define IOVA_START_ADDR (0x1000)
57
58#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
59
4ed0d3e6
FY		 60#define MAX_AGAW_WIDTH 64
61
2ebe3151
DW		 62#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
63#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
64
65/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
66 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
67#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
68 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
69#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 70
f27be03b 71#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 72#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 73#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 74
df08cdc7
AM		 75/* page table handling */
76#define LEVEL_STRIDE (9)
77#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
78
79static inline int agaw_to_level(int agaw)
80{
81 return agaw + 2;
82}
83
84static inline int agaw_to_width(int agaw)
85{
86 return 30 + agaw * LEVEL_STRIDE;
87}
88
89static inline int width_to_agaw(int width)
90{
91 return (width - 30) / LEVEL_STRIDE;
92}
93
94static inline unsigned int level_to_offset_bits(int level)
95{
96 return (level - 1) * LEVEL_STRIDE;
97}
98
99static inline int pfn_level_offset(unsigned long pfn, int level)
100{
101 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
102}
103
104static inline unsigned long level_mask(int level)
105{
106 return -1UL << level_to_offset_bits(level);
107}
108
109static inline unsigned long level_size(int level)
110{
111 return 1UL << level_to_offset_bits(level);
112}
113
114static inline unsigned long align_to_level(unsigned long pfn, int level)
115{
116 return (pfn + level_size(level) - 1) & level_mask(level);
117}
fd18de50 118
dd4e8319
DW		 119/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
120 are never going to work. */
121static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
122{
123 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
124}
125
126static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
127{
128 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
129}
130static inline unsigned long page_to_dma_pfn(struct page *pg)
131{
132 return mm_to_dma_pfn(page_to_pfn(pg));
133}
134static inline unsigned long virt_to_dma_pfn(void *p)
135{
136 return page_to_dma_pfn(virt_to_page(p));
137}
138
d9630fe9
WH		 139/* global iommu list, set NULL for ignored DMAR units */
140static struct intel_iommu **g_iommus;
141
e0fc7e0b 142static void __init check_tylersburg_isoch(void);
9af88143
DW		 143static int rwbf_quirk;
144
46b08e1a
MM		 145/*
146 * 0: Present
147 * 1-11: Reserved
148 * 12-63: Context Ptr (12 - (haw-1))
149 * 64-127: Reserved
150 */
151struct root_entry {
152 u64 val;
153 u64 rsvd1;
154};
155#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
156static inline bool root_present(struct root_entry *root)
157{
158 return (root->val & 1);
159}
160static inline void set_root_present(struct root_entry *root)
161{
162 root->val |= 1;
163}
164static inline void set_root_value(struct root_entry *root, unsigned long value)
165{
166 root->val |= value & VTD_PAGE_MASK;
167}
168
169static inline struct context_entry *
170get_context_addr_from_root(struct root_entry *root)
171{
172 return (struct context_entry *)
173 (root_present(root)?phys_to_virt(
174 root->val & VTD_PAGE_MASK) :
175 NULL);
176}
177
7a8fc25e
MM		 178/*
179 * low 64 bits:
180 * 0: present
181 * 1: fault processing disable
182 * 2-3: translation type
183 * 12-63: address space root
184 * high 64 bits:
185 * 0-2: address width
186 * 3-6: aval
187 * 8-23: domain id
188 */
189struct context_entry {
190 u64 lo;
191 u64 hi;
192};
c07e7d21
MM		 193
194static inline bool context_present(struct context_entry *context)
195{
196 return (context->lo & 1);
197}
198static inline void context_set_present(struct context_entry *context)
199{
200 context->lo |= 1;
201}
202
203static inline void context_set_fault_enable(struct context_entry *context)
204{
205 context->lo &= (((u64)-1) << 2) | 1;
206}
207
c07e7d21
MM		 208static inline void context_set_translation_type(struct context_entry *context,
209 unsigned long value)
210{
211 context->lo &= (((u64)-1) << 4) | 3;
212 context->lo |= (value & 3) << 2;
213}
214
215static inline void context_set_address_root(struct context_entry *context,
216 unsigned long value)
217{
218 context->lo |= value & VTD_PAGE_MASK;
219}
220
221static inline void context_set_address_width(struct context_entry *context,
222 unsigned long value)
223{
224 context->hi |= value & 7;
225}
226
227static inline void context_set_domain_id(struct context_entry *context,
228 unsigned long value)
229{
230 context->hi |= (value & ((1 << 16) - 1)) << 8;
231}
232
233static inline void context_clear_entry(struct context_entry *context)
234{
235 context->lo = 0;
236 context->hi = 0;
237}
7a8fc25e 238
622ba12a
MM		 239/*
240 * 0: readable
241 * 1: writable
242 * 2-6: reserved
243 * 7: super page
9cf06697
SY		 244 * 8-10: available
245 * 11: snoop behavior
622ba12a
MM		 246 * 12-63: Host physcial address
247 */
248struct dma_pte {
249 u64 val;
250};
622ba12a 251
19c239ce
MM		 252static inline void dma_clear_pte(struct dma_pte *pte)
253{
254 pte->val = 0;
255}
256
257static inline void dma_set_pte_readable(struct dma_pte *pte)
258{
259 pte->val |= DMA_PTE_READ;
260}
261
262static inline void dma_set_pte_writable(struct dma_pte *pte)
263{
264 pte->val |= DMA_PTE_WRITE;
265}
266
9cf06697
SY		 267static inline void dma_set_pte_snp(struct dma_pte *pte)
268{
269 pte->val |= DMA_PTE_SNP;
270}
271
19c239ce
MM		 272static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
273{
274 pte->val = (pte->val & ~3) | (prot & 3);
275}
276
277static inline u64 dma_pte_addr(struct dma_pte *pte)
278{
c85994e4
DW		 279#ifdef CONFIG_64BIT
280 return pte->val & VTD_PAGE_MASK;
281#else
282 /* Must have a full atomic 64-bit read */
1a8bd481 283 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 284#endif
19c239ce
MM		 285}
286
dd4e8319 287static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 288{
dd4e8319 289 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 290}
291
292static inline bool dma_pte_present(struct dma_pte *pte)
293{
294 return (pte->val & 3) != 0;
295}
622ba12a 296
75e6bf96
DW		 297static inline int first_pte_in_page(struct dma_pte *pte)
298{
299 return !((unsigned long)pte & ~VTD_PAGE_MASK);
300}
301
2c2e2c38
FY		 302/*
303 * This domain is a statically identity mapping domain.
304 * 1. This domain creats a static 1:1 mapping to all usable memory.
305 * 2. It maps to each iommu if successful.
306 * 3. Each iommu mapps to this domain if successful.
307 */
19943b0e
DW		 308static struct dmar_domain *si_domain;
309static int hw_pass_through = 1;
2c2e2c38 310
3b5410e7 311/* devices under the same p2p bridge are owned in one domain */
cdc7b837 312#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 313
1ce28feb
WH		 314/* domain represents a virtual machine, more than one devices
315 * across iommus may be owned in one domain, e.g. kvm guest.
316 */
317#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
318
2c2e2c38
FY		 319/* si_domain contains mulitple devices */
320#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
321
99126f7c
MM		 322struct dmar_domain {
323 int id; /* domain id */
4c923d47 324 int nid; /* node id */
8c11e798 325 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 326
327 struct list_head devices; /* all devices' list */
328 struct iova_domain iovad; /* iova's that belong to this domain */
329
330 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 331 int gaw; /* max guest address width */
332
333 /* adjusted guest address width, 0 is level 2 30-bit */
334 int agaw;
335
3b5410e7 336 int flags; /* flags to find out type of domain */
8e604097
WH		 337
338 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 339 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d
WH		 340 int iommu_count; /* reference count of iommu */
341 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 342 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 343};
344
a647dacb
MM		 345/* PCI domain-device relationship */
346struct device_domain_info {
347 struct list_head link; /* link to domain siblings */
348 struct list_head global; /* link to global list */
276dbf99
DW		 349 int segment; /* PCI domain */
350 u8 bus; /* PCI bus number */
a647dacb 351 u8 devfn; /* PCI devfn number */
45e829ea 352 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 353 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 354 struct dmar_domain *domain; /* pointer to domain */
355};
356
5e0d2a6f 357static void flush_unmaps_timeout(unsigned long data);
358
359DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
360
80b20dd8 361#define HIGH_WATER_MARK 250
362struct deferred_flush_tables {
363 int next;
364 struct iova *iova[HIGH_WATER_MARK];
365 struct dmar_domain *domain[HIGH_WATER_MARK];
366};
367
368static struct deferred_flush_tables *deferred_flush;
369
5e0d2a6f 370/* bitmap for indexing intel_iommus */
5e0d2a6f 371static int g_num_of_iommus;
372
373static DEFINE_SPINLOCK(async_umap_flush_lock);
374static LIST_HEAD(unmaps_to_do);
375
376static int timer_on;
377static long list_size;
5e0d2a6f 378
ba395927
KA		 379static void domain_remove_dev_info(struct dmar_domain *domain);
380
0cd5c3c8
KM		 381#ifdef CONFIG_DMAR_DEFAULT_ON
382int dmar_disabled = 0;
383#else
384int dmar_disabled = 1;
385#endif /*CONFIG_DMAR_DEFAULT_ON*/
386
2d9e667e 387static int dmar_map_gfx = 1;
7d3b03ce 388static int dmar_forcedac;
5e0d2a6f 389static int intel_iommu_strict;
ba395927
KA		 390
391#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
392static DEFINE_SPINLOCK(device_domain_lock);
393static LIST_HEAD(device_domain_list);
394
a8bcbb0d
JR		 395static struct iommu_ops intel_iommu_ops;
396
ba395927
KA		 397static int __init intel_iommu_setup(char *str)
398{
399 if (!str)
400 return -EINVAL;
401 while (*str) {
0cd5c3c8
KM		 402 if (!strncmp(str, "on", 2)) {
403 dmar_disabled = 0;
404 printk(KERN_INFO "Intel-IOMMU: enabled\n");
405 } else if (!strncmp(str, "off", 3)) {
ba395927 406 dmar_disabled = 1;
0cd5c3c8 407 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 408 } else if (!strncmp(str, "igfx_off", 8)) {
409 dmar_map_gfx = 0;
410 printk(KERN_INFO
411 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 412 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 413 printk(KERN_INFO
7d3b03ce
KA		 414 "Intel-IOMMU: Forcing DAC for PCI devices\n");
415 dmar_forcedac = 1;
5e0d2a6f 416 } else if (!strncmp(str, "strict", 6)) {
417 printk(KERN_INFO
418 "Intel-IOMMU: disable batched IOTLB flush\n");
419 intel_iommu_strict = 1;
ba395927
KA		 420 }
421
422 str += strcspn(str, ",");
423 while (*str == ',')
424 str++;
425 }
426 return 0;
427}
428__setup("intel_iommu=", intel_iommu_setup);
429
430static struct kmem_cache *iommu_domain_cache;
431static struct kmem_cache *iommu_devinfo_cache;
432static struct kmem_cache *iommu_iova_cache;
433
4c923d47 434static inline void *alloc_pgtable_page(int node)
eb3fa7cb 435{
4c923d47
SS		 436 struct page *page;
437 void *vaddr = NULL;
eb3fa7cb 438
4c923d47
SS		 439 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
440 if (page)
441 vaddr = page_address(page);
eb3fa7cb 442 return vaddr;
ba395927
KA		 443}
444
445static inline void free_pgtable_page(void *vaddr)
446{
447 free_page((unsigned long)vaddr);
448}
449
450static inline void *alloc_domain_mem(void)
451{
354bb65e 452 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 453}
454
38717946 455static void free_domain_mem(void *vaddr)
ba395927
KA		 456{
457 kmem_cache_free(iommu_domain_cache, vaddr);
458}
459
460static inline void * alloc_devinfo_mem(void)
461{
354bb65e 462 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 463}
464
465static inline void free_devinfo_mem(void *vaddr)
466{
467 kmem_cache_free(iommu_devinfo_cache, vaddr);
468}
469
470struct iova *alloc_iova_mem(void)
471{
354bb65e 472 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 473}
474
475void free_iova_mem(struct iova *iova)
476{
477 kmem_cache_free(iommu_iova_cache, iova);
478}
479
1b573683 480
4ed0d3e6 481static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 482{
483 unsigned long sagaw;
484 int agaw = -1;
485
486 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 487 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 488 agaw >= 0; agaw--) {
489 if (test_bit(agaw, &sagaw))
490 break;
491 }
492
493 return agaw;
494}
495
4ed0d3e6
FY		 496/*
497 * Calculate max SAGAW for each iommu.
498 */
499int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
500{
501 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
502}
503
504/*
505 * calculate agaw for each iommu.
506 * "SAGAW" may be different across iommus, use a default agaw, and
507 * get a supported less agaw for iommus that don't support the default agaw.
508 */
509int iommu_calculate_agaw(struct intel_iommu *iommu)
510{
511 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
512}
513
2c2e2c38 514/* This functionin only returns single iommu in a domain */
8c11e798
WH		 515static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
516{
517 int iommu_id;
518
2c2e2c38 519 /* si_domain and vm domain should not get here. */
1ce28feb 520 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 521 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 522
8c11e798
WH		 523 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
524 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
525 return NULL;
526
527 return g_iommus[iommu_id];
528}
529
8e604097
WH		 530static void domain_update_iommu_coherency(struct dmar_domain *domain)
531{
532 int i;
533
534 domain->iommu_coherency = 1;
535
a45946ab 536 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 537 if (!ecap_coherent(g_iommus[i]->ecap)) {
538 domain->iommu_coherency = 0;
539 break;
540 }
8e604097
WH		 541 }
542}
543
58c610bd
SY		 544static void domain_update_iommu_snooping(struct dmar_domain *domain)
545{
546 int i;
547
548 domain->iommu_snooping = 1;
549
a45946ab 550 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 551 if (!ecap_sc_support(g_iommus[i]->ecap)) {
552 domain->iommu_snooping = 0;
553 break;
554 }
58c610bd
SY		 555 }
556}
557
558/* Some capabilities may be different across iommus */
559static void domain_update_iommu_cap(struct dmar_domain *domain)
560{
561 domain_update_iommu_coherency(domain);
562 domain_update_iommu_snooping(domain);
563}
564
276dbf99 565static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 566{
567 struct dmar_drhd_unit *drhd = NULL;
568 int i;
569
570 for_each_drhd_unit(drhd) {
571 if (drhd->ignored)
572 continue;
276dbf99
DW		 573 if (segment != drhd->segment)
574 continue;
c7151a8d 575
924b6231 576 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 577 if (drhd->devices[i] &&
578 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 579 drhd->devices[i]->devfn == devfn)
580 return drhd->iommu;
4958c5dc
DW		 581 if (drhd->devices[i] &&
582 drhd->devices[i]->subordinate &&
924b6231
DW		 583 drhd->devices[i]->subordinate->number <= bus &&
584 drhd->devices[i]->subordinate->subordinate >= bus)
585 return drhd->iommu;
586 }
c7151a8d
WH		 587
588 if (drhd->include_all)
589 return drhd->iommu;
590 }
591
592 return NULL;
593}
594
5331fe6f
WH		 595static void domain_flush_cache(struct dmar_domain *domain,
596 void *addr, int size)
597{
598 if (!domain->iommu_coherency)
599 clflush_cache_range(addr, size);
600}
601
ba395927
KA		 602/* Gets context entry for a given bus and devfn */
603static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
604 u8 bus, u8 devfn)
605{
606 struct root_entry *root;
607 struct context_entry *context;
608 unsigned long phy_addr;
609 unsigned long flags;
610
611 spin_lock_irqsave(&iommu->lock, flags);
612 root = &iommu->root_entry[bus];
613 context = get_context_addr_from_root(root);
614 if (!context) {
4c923d47
SS		 615 context = (struct context_entry *)
616 alloc_pgtable_page(iommu->node);
ba395927
KA		 617 if (!context) {
618 spin_unlock_irqrestore(&iommu->lock, flags);
619 return NULL;
620 }
5b6985ce 621 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 622 phy_addr = virt_to_phys((void *)context);
623 set_root_value(root, phy_addr);
624 set_root_present(root);
625 __iommu_flush_cache(iommu, root, sizeof(*root));
626 }
627 spin_unlock_irqrestore(&iommu->lock, flags);
628 return &context[devfn];
629}
630
631static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
632{
633 struct root_entry *root;
634 struct context_entry *context;
635 int ret;
636 unsigned long flags;
637
638 spin_lock_irqsave(&iommu->lock, flags);
639 root = &iommu->root_entry[bus];
640 context = get_context_addr_from_root(root);
641 if (!context) {
642 ret = 0;
643 goto out;
644 }
c07e7d21 645 ret = context_present(&context[devfn]);
ba395927
KA		 646out:
647 spin_unlock_irqrestore(&iommu->lock, flags);
648 return ret;
649}
650
651static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
652{
653 struct root_entry *root;
654 struct context_entry *context;
655 unsigned long flags;
656
657 spin_lock_irqsave(&iommu->lock, flags);
658 root = &iommu->root_entry[bus];
659 context = get_context_addr_from_root(root);
660 if (context) {
c07e7d21 661 context_clear_entry(&context[devfn]);
ba395927
KA		 662 __iommu_flush_cache(iommu, &context[devfn], \
663 sizeof(*context));
664 }
665 spin_unlock_irqrestore(&iommu->lock, flags);
666}
667
668static void free_context_table(struct intel_iommu *iommu)
669{
670 struct root_entry *root;
671 int i;
672 unsigned long flags;
673 struct context_entry *context;
674
675 spin_lock_irqsave(&iommu->lock, flags);
676 if (!iommu->root_entry) {
677 goto out;
678 }
679 for (i = 0; i < ROOT_ENTRY_NR; i++) {
680 root = &iommu->root_entry[i];
681 context = get_context_addr_from_root(root);
682 if (context)
683 free_pgtable_page(context);
684 }
685 free_pgtable_page(iommu->root_entry);
686 iommu->root_entry = NULL;
687out:
688 spin_unlock_irqrestore(&iommu->lock, flags);
689}
690
b026fd28
DW		 691static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
692 unsigned long pfn)
ba395927 693{
b026fd28 694 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 695 struct dma_pte *parent, *pte = NULL;
696 int level = agaw_to_level(domain->agaw);
697 int offset;
ba395927
KA		 698
699 BUG_ON(!domain->pgd);
b026fd28 700 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 701 parent = domain->pgd;
702
ba395927
KA		 703 while (level > 0) {
704 void *tmp_page;
705
b026fd28 706 offset = pfn_level_offset(pfn, level);
ba395927
KA		 707 pte = &parent[offset];
708 if (level == 1)
709 break;
710
19c239ce 711 if (!dma_pte_present(pte)) {
c85994e4
DW		 712 uint64_t pteval;
713
4c923d47 714 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 715
206a73c1 716 if (!tmp_page)
ba395927 717 return NULL;
206a73c1 718
c85994e4 719 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 720 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 721 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
722 /* Someone else set it while we were thinking; use theirs. */
723 free_pgtable_page(tmp_page);
724 } else {
725 dma_pte_addr(pte);
726 domain_flush_cache(domain, pte, sizeof(*pte));
727 }
ba395927 728 }
19c239ce 729 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 730 level--;
731 }
732
ba395927
KA		 733 return pte;
734}
735
736/* return address's pte at specific level */
90dcfb5e
DW		 737static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
738 unsigned long pfn,
739 int level)
ba395927
KA		 740{
741 struct dma_pte *parent, *pte = NULL;
742 int total = agaw_to_level(domain->agaw);
743 int offset;
744
745 parent = domain->pgd;
746 while (level <= total) {
90dcfb5e 747 offset = pfn_level_offset(pfn, total);
ba395927
KA		 748 pte = &parent[offset];
749 if (level == total)
750 return pte;
751
19c239ce 752 if (!dma_pte_present(pte))
ba395927 753 break;
19c239ce 754 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 755 total--;
756 }
757 return NULL;
758}
759
ba395927 760/* clear last level pte, a tlb flush should be followed */
595badf5
DW		 761static void dma_pte_clear_range(struct dmar_domain *domain,
762 unsigned long start_pfn,
763 unsigned long last_pfn)
ba395927 764{
04b18e65 765 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
310a5ab9 766 struct dma_pte *first_pte, *pte;
66eae846 767
04b18e65 768 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 769 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 770 BUG_ON(start_pfn > last_pfn);
ba395927 771
04b18e65 772 /* we don't need lock here; nobody else touches the iova range */
59c36286 773 do {
310a5ab9
DW		 774 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
775 if (!pte) {
776 start_pfn = align_to_level(start_pfn + 1, 2);
777 continue;
778 }
75e6bf96 779 do {
310a5ab9
DW		 780 dma_clear_pte(pte);
781 start_pfn++;
782 pte++;
75e6bf96
DW		 783 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
784
310a5ab9
DW		 785 domain_flush_cache(domain, first_pte,
786 (void *)pte - (void *)first_pte);
59c36286
DW		 787
788 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 789}
790
791/* free page table pages. last level pte should already be cleared */
792static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 793 unsigned long start_pfn,
794 unsigned long last_pfn)
ba395927 795{
6660c63a 796 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
f3a0a52f 797 struct dma_pte *first_pte, *pte;
ba395927
KA		 798 int total = agaw_to_level(domain->agaw);
799 int level;
6660c63a 800 unsigned long tmp;
ba395927 801
6660c63a
DW		 802 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
803 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 804 BUG_ON(start_pfn > last_pfn);
ba395927 805
f3a0a52f 806 /* We don't need lock here; nobody else touches the iova range */
ba395927
KA		 807 level = 2;
808 while (level <= total) {
6660c63a
DW		 809 tmp = align_to_level(start_pfn, level);
810
f3a0a52f 811 /* If we can't even clear one PTE at this level, we're done */
6660c63a 812 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 813 return;
814
59c36286 815 do {
f3a0a52f
DW		 816 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
817 if (!pte) {
818 tmp = align_to_level(tmp + 1, level + 1);
819 continue;
820 }
75e6bf96 821 do {
6a43e574
DW		 822 if (dma_pte_present(pte)) {
823 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
824 dma_clear_pte(pte);
825 }
f3a0a52f
DW		 826 pte++;
827 tmp += level_size(level);
75e6bf96
DW		 828 } while (!first_pte_in_page(pte) &&
829 tmp + level_size(level) - 1 <= last_pfn);
830
f3a0a52f
DW		 831 domain_flush_cache(domain, first_pte,
832 (void *)pte - (void *)first_pte);
833
59c36286 834 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
ba395927
KA		 835 level++;
836 }
837 /* free pgd */
d794dc9b 838 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 839 free_pgtable_page(domain->pgd);
840 domain->pgd = NULL;
841 }
842}
843
844/* iommu handling */
845static int iommu_alloc_root_entry(struct intel_iommu *iommu)
846{
847 struct root_entry *root;
848 unsigned long flags;
849
4c923d47 850 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 851 if (!root)
852 return -ENOMEM;
853
5b6985ce 854 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 855
856 spin_lock_irqsave(&iommu->lock, flags);
857 iommu->root_entry = root;
858 spin_unlock_irqrestore(&iommu->lock, flags);
859
860 return 0;
861}
862
ba395927
KA		 863static void iommu_set_root_entry(struct intel_iommu *iommu)
864{
865 void *addr;
c416daa9 866 u32 sts;
ba395927
KA		 867 unsigned long flag;
868
869 addr = iommu->root_entry;
870
871 spin_lock_irqsave(&iommu->register_lock, flag);
872 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
873
c416daa9 874 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 875
876 /* Make sure hardware complete it */
877 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 878 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 879
880 spin_unlock_irqrestore(&iommu->register_lock, flag);
881}
882
883static void iommu_flush_write_buffer(struct intel_iommu *iommu)
884{
885 u32 val;
886 unsigned long flag;
887
9af88143 888 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 889 return;
ba395927
KA		 890
891 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 892 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 893
894 /* Make sure hardware complete it */
895 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 896 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 897
898 spin_unlock_irqrestore(&iommu->register_lock, flag);
899}
900
901/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 902static void __iommu_flush_context(struct intel_iommu *iommu,
903 u16 did, u16 source_id, u8 function_mask,
904 u64 type)
ba395927
KA		 905{
906 u64 val = 0;
907 unsigned long flag;
908
ba395927
KA		 909 switch (type) {
910 case DMA_CCMD_GLOBAL_INVL:
911 val = DMA_CCMD_GLOBAL_INVL;
912 break;
913 case DMA_CCMD_DOMAIN_INVL:
914 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
915 break;
916 case DMA_CCMD_DEVICE_INVL:
917 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
918 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
919 break;
920 default:
921 BUG();
922 }
923 val |= DMA_CCMD_ICC;
924
925 spin_lock_irqsave(&iommu->register_lock, flag);
926 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
927
928 /* Make sure hardware complete it */
929 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
930 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
931
932 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 933}
934
ba395927 935/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 936static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
937 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 938{
939 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
940 u64 val = 0, val_iva = 0;
941 unsigned long flag;
942
ba395927
KA		 943 switch (type) {
944 case DMA_TLB_GLOBAL_FLUSH:
945 /* global flush doesn't need set IVA_REG */
946 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
947 break;
948 case DMA_TLB_DSI_FLUSH:
949 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
950 break;
951 case DMA_TLB_PSI_FLUSH:
952 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
953 /* Note: always flush non-leaf currently */
954 val_iva = size_order | addr;
955 break;
956 default:
957 BUG();
958 }
959 /* Note: set drain read/write */
960#if 0
961 /*
962 * This is probably to be super secure.. Looks like we can
963 * ignore it without any impact.
964 */
965 if (cap_read_drain(iommu->cap))
966 val |= DMA_TLB_READ_DRAIN;
967#endif
968 if (cap_write_drain(iommu->cap))
969 val |= DMA_TLB_WRITE_DRAIN;
970
971 spin_lock_irqsave(&iommu->register_lock, flag);
972 /* Note: Only uses first TLB reg currently */
973 if (val_iva)
974 dmar_writeq(iommu->reg + tlb_offset, val_iva);
975 dmar_writeq(iommu->reg + tlb_offset + 8, val);
976
977 /* Make sure hardware complete it */
978 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
979 dmar_readq, (!(val & DMA_TLB_IVT)), val);
980
981 spin_unlock_irqrestore(&iommu->register_lock, flag);
982
983 /* check IOTLB invalidation granularity */
984 if (DMA_TLB_IAIG(val) == 0)
985 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
986 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
987 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 988 (unsigned long long)DMA_TLB_IIRG(type),
989 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 990}
991
93a23a72
YZ		 992static struct device_domain_info *iommu_support_dev_iotlb(
993 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
994{
995 int found = 0;
996 unsigned long flags;
997 struct device_domain_info *info;
998 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
999
1000 if (!ecap_dev_iotlb_support(iommu->ecap))
1001 return NULL;
1002
1003 if (!iommu->qi)
1004 return NULL;
1005
1006 spin_lock_irqsave(&device_domain_lock, flags);
1007 list_for_each_entry(info, &domain->devices, link)
1008 if (info->bus == bus && info->devfn == devfn) {
1009 found = 1;
1010 break;
1011 }
1012 spin_unlock_irqrestore(&device_domain_lock, flags);
1013
1014 if (!found || !info->dev)
1015 return NULL;
1016
1017 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1018 return NULL;
1019
1020 if (!dmar_find_matched_atsr_unit(info->dev))
1021 return NULL;
1022
1023 info->iommu = iommu;
1024
1025 return info;
1026}
1027
1028static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1029{
93a23a72
YZ		 1030 if (!info)
1031 return;
1032
1033 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1034}
1035
1036static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1037{
1038 if (!info->dev || !pci_ats_enabled(info->dev))
1039 return;
1040
1041 pci_disable_ats(info->dev);
1042}
1043
1044static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1045 u64 addr, unsigned mask)
1046{
1047 u16 sid, qdep;
1048 unsigned long flags;
1049 struct device_domain_info *info;
1050
1051 spin_lock_irqsave(&device_domain_lock, flags);
1052 list_for_each_entry(info, &domain->devices, link) {
1053 if (!info->dev || !pci_ats_enabled(info->dev))
1054 continue;
1055
1056 sid = info->bus << 8 | info->devfn;
1057 qdep = pci_ats_queue_depth(info->dev);
1058 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1059 }
1060 spin_unlock_irqrestore(&device_domain_lock, flags);
1061}
1062
1f0ef2aa 1063static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1064 unsigned long pfn, unsigned int pages, int map)
ba395927 1065{
9dd2fe89 1066 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1067 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1068
ba395927
KA		 1069 BUG_ON(pages == 0);
1070
ba395927 1071 /*
9dd2fe89
YZ		 1072 * Fallback to domain selective flush if no PSI support or the size is
1073 * too big.
ba395927
KA		 1074 * PSI requires page size to be 2 ^ x, and the base address is naturally
1075 * aligned to the size
1076 */
9dd2fe89
YZ		 1077 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1078 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1079 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1080 else
1081 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1082 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1083
1084 /*
82653633
NA		 1085 * In caching mode, changes of pages from non-present to present require
1086 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1087 */
82653633 1088 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1089 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1090}
1091
f8bab735 1092static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1093{
1094 u32 pmen;
1095 unsigned long flags;
1096
1097 spin_lock_irqsave(&iommu->register_lock, flags);
1098 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1099 pmen &= ~DMA_PMEN_EPM;
1100 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1101
1102 /* wait for the protected region status bit to clear */
1103 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1104 readl, !(pmen & DMA_PMEN_PRS), pmen);
1105
1106 spin_unlock_irqrestore(&iommu->register_lock, flags);
1107}
1108
ba395927
KA		 1109static int iommu_enable_translation(struct intel_iommu *iommu)
1110{
1111 u32 sts;
1112 unsigned long flags;
1113
1114 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1115 iommu->gcmd |= DMA_GCMD_TE;
1116 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1117
1118 /* Make sure hardware complete it */
1119 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1120 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1121
ba395927
KA		 1122 spin_unlock_irqrestore(&iommu->register_lock, flags);
1123 return 0;
1124}
1125
1126static int iommu_disable_translation(struct intel_iommu *iommu)
1127{
1128 u32 sts;
1129 unsigned long flag;
1130
1131 spin_lock_irqsave(&iommu->register_lock, flag);
1132 iommu->gcmd &= ~DMA_GCMD_TE;
1133 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1134
1135 /* Make sure hardware complete it */
1136 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1137 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1138
1139 spin_unlock_irqrestore(&iommu->register_lock, flag);
1140 return 0;
1141}
1142
3460a6d9 1143
ba395927
KA		 1144static int iommu_init_domains(struct intel_iommu *iommu)
1145{
1146 unsigned long ndomains;
1147 unsigned long nlongs;
1148
1149 ndomains = cap_ndoms(iommu->cap);
680a7524
YL		 1150 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1151 ndomains);
ba395927
KA		 1152 nlongs = BITS_TO_LONGS(ndomains);
1153
94a91b50
DD		 1154 spin_lock_init(&iommu->lock);
1155
ba395927
KA		 1156 /* TBD: there might be 64K domains,
1157 * consider other allocation for future chip
1158 */
1159 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1160 if (!iommu->domain_ids) {
1161 printk(KERN_ERR "Allocating domain id array failed\n");
1162 return -ENOMEM;
1163 }
1164 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1165 GFP_KERNEL);
1166 if (!iommu->domains) {
1167 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1168 return -ENOMEM;
1169 }
1170
1171 /*
1172 * if Caching mode is set, then invalid translations are tagged
1173 * with domainid 0. Hence we need to pre-allocate it.
1174 */
1175 if (cap_caching_mode(iommu->cap))
1176 set_bit(0, iommu->domain_ids);
1177 return 0;
1178}
ba395927 1179
ba395927
KA		 1180
1181static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1182static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1183
1184void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1185{
1186 struct dmar_domain *domain;
1187 int i;
c7151a8d 1188 unsigned long flags;
ba395927 1189
94a91b50 1190 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1191 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1192 domain = iommu->domains[i];
1193 clear_bit(i, iommu->domain_ids);
1194
1195 spin_lock_irqsave(&domain->iommu_lock, flags);
1196 if (--domain->iommu_count == 0) {
1197 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1198 vm_domain_exit(domain);
1199 else
1200 domain_exit(domain);
1201 }
1202 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1203 }
ba395927
KA		 1204 }
1205
1206 if (iommu->gcmd & DMA_GCMD_TE)
1207 iommu_disable_translation(iommu);
1208
1209 if (iommu->irq) {
dced35ae 1210 irq_set_handler_data(iommu->irq, NULL);
ba395927
KA		 1211 /* This will mask the irq */
1212 free_irq(iommu->irq, iommu);
1213 destroy_irq(iommu->irq);
1214 }
1215
1216 kfree(iommu->domains);
1217 kfree(iommu->domain_ids);
1218
d9630fe9
WH		 1219 g_iommus[iommu->seq_id] = NULL;
1220
1221 /* if all iommus are freed, free g_iommus */
1222 for (i = 0; i < g_num_of_iommus; i++) {
1223 if (g_iommus[i])
1224 break;
1225 }
1226
1227 if (i == g_num_of_iommus)
1228 kfree(g_iommus);
1229
ba395927
KA		 1230 /* free context mapping */
1231 free_context_table(iommu);
ba395927
KA		 1232}
1233
2c2e2c38 1234static struct dmar_domain *alloc_domain(void)
ba395927 1235{
ba395927 1236 struct dmar_domain *domain;
ba395927
KA		 1237
1238 domain = alloc_domain_mem();
1239 if (!domain)
1240 return NULL;
1241
4c923d47 1242 domain->nid = -1;
2c2e2c38
FY		 1243 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1244 domain->flags = 0;
1245
1246 return domain;
1247}
1248
1249static int iommu_attach_domain(struct dmar_domain *domain,
1250 struct intel_iommu *iommu)
1251{
1252 int num;
1253 unsigned long ndomains;
1254 unsigned long flags;
1255
ba395927
KA		 1256 ndomains = cap_ndoms(iommu->cap);
1257
1258 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1259
ba395927
KA		 1260 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1261 if (num >= ndomains) {
1262 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1263 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1264 return -ENOMEM;
ba395927
KA		 1265 }
1266
ba395927 1267 domain->id = num;
2c2e2c38 1268 set_bit(num, iommu->domain_ids);
8c11e798 1269 set_bit(iommu->seq_id, &domain->iommu_bmp);
ba395927
KA		 1270 iommu->domains[num] = domain;
1271 spin_unlock_irqrestore(&iommu->lock, flags);
1272
2c2e2c38 1273 return 0;
ba395927
KA		 1274}
1275
2c2e2c38
FY		 1276static void iommu_detach_domain(struct dmar_domain *domain,
1277 struct intel_iommu *iommu)
ba395927
KA		 1278{
1279 unsigned long flags;
2c2e2c38
FY		 1280 int num, ndomains;
1281 int found = 0;
ba395927 1282
8c11e798 1283 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1284 ndomains = cap_ndoms(iommu->cap);
a45946ab 1285 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1286 if (iommu->domains[num] == domain) {
1287 found = 1;
1288 break;
1289 }
2c2e2c38
FY		 1290 }
1291
1292 if (found) {
1293 clear_bit(num, iommu->domain_ids);
1294 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1295 iommu->domains[num] = NULL;
1296 }
8c11e798 1297 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1298}
1299
1300static struct iova_domain reserved_iova_list;
8a443df4 1301static struct lock_class_key reserved_rbtree_key;
ba395927 1302
51a63e67 1303static int dmar_init_reserved_ranges(void)
ba395927
KA		 1304{
1305 struct pci_dev *pdev = NULL;
1306 struct iova *iova;
1307 int i;
ba395927 1308
f661197e 1309 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1310
8a443df4
MG		 1311 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1312 &reserved_rbtree_key);
1313
ba395927
KA		 1314 /* IOAPIC ranges shouldn't be accessed by DMA */
1315 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1316 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1317 if (!iova) {
ba395927 1318 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1319 return -ENODEV;
1320 }
ba395927
KA		 1321
1322 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1323 for_each_pci_dev(pdev) {
1324 struct resource *r;
1325
1326 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1327 r = &pdev->resource[i];
1328 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1329 continue;
1a4a4551
DW		 1330 iova = reserve_iova(&reserved_iova_list,
1331 IOVA_PFN(r->start),
1332 IOVA_PFN(r->end));
51a63e67 1333 if (!iova) {
ba395927 1334 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1335 return -ENODEV;
1336 }
ba395927
KA		 1337 }
1338 }
51a63e67 1339 return 0;
ba395927
KA		 1340}
1341
1342static void domain_reserve_special_ranges(struct dmar_domain *domain)
1343{
1344 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1345}
1346
1347static inline int guestwidth_to_adjustwidth(int gaw)
1348{
1349 int agaw;
1350 int r = (gaw - 12) % 9;
1351
1352 if (r == 0)
1353 agaw = gaw;
1354 else
1355 agaw = gaw + 9 - r;
1356 if (agaw > 64)
1357 agaw = 64;
1358 return agaw;
1359}
1360
1361static int domain_init(struct dmar_domain *domain, int guest_width)
1362{
1363 struct intel_iommu *iommu;
1364 int adjust_width, agaw;
1365 unsigned long sagaw;
1366
f661197e 1367 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1368 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1369
1370 domain_reserve_special_ranges(domain);
1371
1372 /* calculate AGAW */
8c11e798 1373 iommu = domain_get_iommu(domain);
ba395927
KA		 1374 if (guest_width > cap_mgaw(iommu->cap))
1375 guest_width = cap_mgaw(iommu->cap);
1376 domain->gaw = guest_width;
1377 adjust_width = guestwidth_to_adjustwidth(guest_width);
1378 agaw = width_to_agaw(adjust_width);
1379 sagaw = cap_sagaw(iommu->cap);
1380 if (!test_bit(agaw, &sagaw)) {
1381 /* hardware doesn't support it, choose a bigger one */
1382 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1383 agaw = find_next_bit(&sagaw, 5, agaw);
1384 if (agaw >= 5)
1385 return -ENODEV;
1386 }
1387 domain->agaw = agaw;
1388 INIT_LIST_HEAD(&domain->devices);
1389
8e604097
WH		 1390 if (ecap_coherent(iommu->ecap))
1391 domain->iommu_coherency = 1;
1392 else
1393 domain->iommu_coherency = 0;
1394
58c610bd
SY		 1395 if (ecap_sc_support(iommu->ecap))
1396 domain->iommu_snooping = 1;
1397 else
1398 domain->iommu_snooping = 0;
1399
c7151a8d 1400 domain->iommu_count = 1;
4c923d47 1401 domain->nid = iommu->node;
c7151a8d 1402
ba395927 1403 /* always allocate the top pgd */
4c923d47 1404 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1405 if (!domain->pgd)
1406 return -ENOMEM;
5b6985ce 1407 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1408 return 0;
1409}
1410
1411static void domain_exit(struct dmar_domain *domain)
1412{
2c2e2c38
FY		 1413 struct dmar_drhd_unit *drhd;
1414 struct intel_iommu *iommu;
ba395927
KA		 1415
1416 /* Domain 0 is reserved, so dont process it */
1417 if (!domain)
1418 return;
1419
1420 domain_remove_dev_info(domain);
1421 /* destroy iovas */
1422 put_iova_domain(&domain->iovad);
ba395927
KA		 1423
1424 /* clear ptes */
595badf5 1425 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1426
1427 /* free page tables */
d794dc9b 1428 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1429
2c2e2c38
FY		 1430 for_each_active_iommu(iommu, drhd)
1431 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1432 iommu_detach_domain(domain, iommu);
1433
ba395927
KA		 1434 free_domain_mem(domain);
1435}
1436
4ed0d3e6
FY		 1437static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1438 u8 bus, u8 devfn, int translation)
ba395927
KA		 1439{
1440 struct context_entry *context;
ba395927 1441 unsigned long flags;
5331fe6f 1442 struct intel_iommu *iommu;
ea6606b0
WH		 1443 struct dma_pte *pgd;
1444 unsigned long num;
1445 unsigned long ndomains;
1446 int id;
1447 int agaw;
93a23a72 1448 struct device_domain_info *info = NULL;
ba395927
KA		 1449
1450 pr_debug("Set context mapping for %02x:%02x.%d\n",
1451 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1452
ba395927 1453 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1454 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1455 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1456
276dbf99 1457 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1458 if (!iommu)
1459 return -ENODEV;
1460
ba395927
KA		 1461 context = device_to_context_entry(iommu, bus, devfn);
1462 if (!context)
1463 return -ENOMEM;
1464 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1465 if (context_present(context)) {
ba395927
KA		 1466 spin_unlock_irqrestore(&iommu->lock, flags);
1467 return 0;
1468 }
1469
ea6606b0
WH		 1470 id = domain->id;
1471 pgd = domain->pgd;
1472
2c2e2c38
FY		 1473 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1474 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1475 int found = 0;
1476
1477 /* find an available domain id for this device in iommu */
1478 ndomains = cap_ndoms(iommu->cap);
a45946ab 1479 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1480 if (iommu->domains[num] == domain) {
1481 id = num;
1482 found = 1;
1483 break;
1484 }
ea6606b0
WH		 1485 }
1486
1487 if (found == 0) {
1488 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1489 if (num >= ndomains) {
1490 spin_unlock_irqrestore(&iommu->lock, flags);
1491 printk(KERN_ERR "IOMMU: no free domain ids\n");
1492 return -EFAULT;
1493 }
1494
1495 set_bit(num, iommu->domain_ids);
1496 iommu->domains[num] = domain;
1497 id = num;
1498 }
1499
1500 /* Skip top levels of page tables for
1501 * iommu which has less agaw than default.
1672af11 1502 * Unnecessary for PT mode.
ea6606b0 1503 */
1672af11
CW		 1504 if (translation != CONTEXT_TT_PASS_THROUGH) {
1505 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1506 pgd = phys_to_virt(dma_pte_addr(pgd));
1507 if (!dma_pte_present(pgd)) {
1508 spin_unlock_irqrestore(&iommu->lock, flags);
1509 return -ENOMEM;
1510 }
ea6606b0
WH		 1511 }
1512 }
1513 }
1514
1515 context_set_domain_id(context, id);
4ed0d3e6 1516
93a23a72
YZ		 1517 if (translation != CONTEXT_TT_PASS_THROUGH) {
1518 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1519 translation = info ? CONTEXT_TT_DEV_IOTLB :
1520 CONTEXT_TT_MULTI_LEVEL;
1521 }
4ed0d3e6
FY		 1522 /*
1523 * In pass through mode, AW must be programmed to indicate the largest
1524 * AGAW value supported by hardware. And ASR is ignored by hardware.
1525 */
93a23a72 1526 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1527 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1528 else {
1529 context_set_address_root(context, virt_to_phys(pgd));
1530 context_set_address_width(context, iommu->agaw);
1531 }
4ed0d3e6
FY		 1532
1533 context_set_translation_type(context, translation);
c07e7d21
MM		 1534 context_set_fault_enable(context);
1535 context_set_present(context);
5331fe6f 1536 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1537
4c25a2c1
DW		 1538 /*
1539 * It's a non-present to present mapping. If hardware doesn't cache
1540 * non-present entry we only need to flush the write-buffer. If the
1541 * _does_ cache non-present entries, then it does so in the special
1542 * domain #0, which we have to flush:
1543 */
1544 if (cap_caching_mode(iommu->cap)) {
1545 iommu->flush.flush_context(iommu, 0,
1546 (((u16)bus) << 8) | devfn,
1547 DMA_CCMD_MASK_NOBIT,
1548 DMA_CCMD_DEVICE_INVL);
82653633 1549 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1550 } else {
ba395927 1551 iommu_flush_write_buffer(iommu);
4c25a2c1 1552 }
93a23a72 1553 iommu_enable_dev_iotlb(info);
ba395927 1554 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1555
1556 spin_lock_irqsave(&domain->iommu_lock, flags);
1557 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1558 domain->iommu_count++;
4c923d47
SS		 1559 if (domain->iommu_count == 1)
1560 domain->nid = iommu->node;
58c610bd 1561 domain_update_iommu_cap(domain);
c7151a8d
WH		 1562 }
1563 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1564 return 0;
1565}
1566
1567static int
4ed0d3e6
FY		 1568domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1569 int translation)
ba395927
KA		 1570{
1571 int ret;
1572 struct pci_dev *tmp, *parent;
1573
276dbf99 1574 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1575 pdev->bus->number, pdev->devfn,
1576 translation);
ba395927
KA		 1577 if (ret)
1578 return ret;
1579
1580 /* dependent device mapping */
1581 tmp = pci_find_upstream_pcie_bridge(pdev);
1582 if (!tmp)
1583 return 0;
1584 /* Secondary interface's bus number and devfn 0 */
1585 parent = pdev->bus->self;
1586 while (parent != tmp) {
276dbf99
DW		 1587 ret = domain_context_mapping_one(domain,
1588 pci_domain_nr(parent->bus),
1589 parent->bus->number,
4ed0d3e6 1590 parent->devfn, translation);
ba395927
KA		 1591 if (ret)
1592 return ret;
1593 parent = parent->bus->self;
1594 }
45e829ea 1595 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1596 return domain_context_mapping_one(domain,
276dbf99 1597 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1598 tmp->subordinate->number, 0,
1599 translation);
ba395927
KA		 1600 else /* this is a legacy PCI bridge */
1601 return domain_context_mapping_one(domain,
276dbf99
DW		 1602 pci_domain_nr(tmp->bus),
1603 tmp->bus->number,
4ed0d3e6
FY		 1604 tmp->devfn,
1605 translation);
ba395927
KA		 1606}
1607
5331fe6f 1608static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1609{
1610 int ret;
1611 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1612 struct intel_iommu *iommu;
1613
276dbf99
DW		 1614 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1615 pdev->devfn);
5331fe6f
WH		 1616 if (!iommu)
1617 return -ENODEV;
ba395927 1618
276dbf99 1619 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1620 if (!ret)
1621 return ret;
1622 /* dependent device mapping */
1623 tmp = pci_find_upstream_pcie_bridge(pdev);
1624 if (!tmp)
1625 return ret;
1626 /* Secondary interface's bus number and devfn 0 */
1627 parent = pdev->bus->self;
1628 while (parent != tmp) {
8c11e798 1629 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1630 parent->devfn);
ba395927
KA		 1631 if (!ret)
1632 return ret;
1633 parent = parent->bus->self;
1634 }
5f4d91a1 1635 if (pci_is_pcie(tmp))
276dbf99
DW		 1636 return device_context_mapped(iommu, tmp->subordinate->number,
1637 0);
ba395927 1638 else
276dbf99
DW		 1639 return device_context_mapped(iommu, tmp->bus->number,
1640 tmp->devfn);
ba395927
KA		 1641}
1642
f532959b
FY		 1643/* Returns a number of VTD pages, but aligned to MM page size */
1644static inline unsigned long aligned_nrpages(unsigned long host_addr,
1645 size_t size)
1646{
1647 host_addr &= ~PAGE_MASK;
1648 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1649}
1650
9051aa02
DW		 1651static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1652 struct scatterlist *sg, unsigned long phys_pfn,
1653 unsigned long nr_pages, int prot)
e1605495
DW		 1654{
1655 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1656 phys_addr_t uninitialized_var(pteval);
e1605495 1657 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1658 unsigned long sg_res;
e1605495
DW		 1659
1660 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1661
1662 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1663 return -EINVAL;
1664
1665 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1666
9051aa02
DW		 1667 if (sg)
1668 sg_res = 0;
1669 else {
1670 sg_res = nr_pages + 1;
1671 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1672 }
1673
e1605495 1674 while (nr_pages--) {
c85994e4
DW		 1675 uint64_t tmp;
1676
e1605495 1677 if (!sg_res) {
f532959b 1678 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1679 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1680 sg->dma_length = sg->length;
1681 pteval = page_to_phys(sg_page(sg)) | prot;
1682 }
1683 if (!pte) {
1684 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1685 if (!pte)
1686 return -ENOMEM;
1687 }
1688 /* We don't need lock here, nobody else
1689 * touches the iova range
1690 */
7766a3fb 1691 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1692 if (tmp) {
1bf20f0d 1693 static int dumps = 5;
c85994e4
DW		 1694 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1695 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1696 if (dumps) {
1697 dumps--;
1698 debug_dma_dump_mappings(NULL);
1699 }
1700 WARN_ON(1);
1701 }
e1605495 1702 pte++;
75e6bf96 1703 if (!nr_pages || first_pte_in_page(pte)) {
e1605495
DW		 1704 domain_flush_cache(domain, first_pte,
1705 (void *)pte - (void *)first_pte);
1706 pte = NULL;
1707 }
1708 iov_pfn++;
1709 pteval += VTD_PAGE_SIZE;
1710 sg_res--;
1711 if (!sg_res)
1712 sg = sg_next(sg);
1713 }
1714 return 0;
1715}
1716
9051aa02
DW		 1717static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1718 struct scatterlist *sg, unsigned long nr_pages,
1719 int prot)
ba395927 1720{
9051aa02
DW		 1721 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1722}
6f6a00e4 1723
9051aa02
DW		 1724static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1725 unsigned long phys_pfn, unsigned long nr_pages,
1726 int prot)
1727{
1728 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1729}
1730
c7151a8d 1731static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1732{
c7151a8d
WH		 1733 if (!iommu)
1734 return;
8c11e798
WH		 1735
1736 clear_context_table(iommu, bus, devfn);
1737 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1738 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1739 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1740}
1741
1742static void domain_remove_dev_info(struct dmar_domain *domain)
1743{
1744 struct device_domain_info *info;
1745 unsigned long flags;
c7151a8d 1746 struct intel_iommu *iommu;
ba395927
KA		 1747
1748 spin_lock_irqsave(&device_domain_lock, flags);
1749 while (!list_empty(&domain->devices)) {
1750 info = list_entry(domain->devices.next,
1751 struct device_domain_info, link);
1752 list_del(&info->link);
1753 list_del(&info->global);
1754 if (info->dev)
358dd8ac 1755 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1756 spin_unlock_irqrestore(&device_domain_lock, flags);
1757
93a23a72 1758 iommu_disable_dev_iotlb(info);
276dbf99 1759 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1760 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1761 free_devinfo_mem(info);
1762
1763 spin_lock_irqsave(&device_domain_lock, flags);
1764 }
1765 spin_unlock_irqrestore(&device_domain_lock, flags);
1766}
1767
1768/*
1769 * find_domain
358dd8ac 1770 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1771 */
38717946 1772static struct dmar_domain *
ba395927
KA		 1773find_domain(struct pci_dev *pdev)
1774{
1775 struct device_domain_info *info;
1776
1777 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1778 info = pdev->dev.archdata.iommu;
ba395927
KA		 1779 if (info)
1780 return info->domain;
1781 return NULL;
1782}
1783
ba395927
KA		 1784/* domain is initialized */
1785static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1786{
1787 struct dmar_domain *domain, *found = NULL;
1788 struct intel_iommu *iommu;
1789 struct dmar_drhd_unit *drhd;
1790 struct device_domain_info *info, *tmp;
1791 struct pci_dev *dev_tmp;
1792 unsigned long flags;
1793 int bus = 0, devfn = 0;
276dbf99 1794 int segment;
2c2e2c38 1795 int ret;
ba395927
KA		 1796
1797 domain = find_domain(pdev);
1798 if (domain)
1799 return domain;
1800
276dbf99
DW		 1801 segment = pci_domain_nr(pdev->bus);
1802
ba395927
KA		 1803 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1804 if (dev_tmp) {
5f4d91a1 1805 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1806 bus = dev_tmp->subordinate->number;
1807 devfn = 0;
1808 } else {
1809 bus = dev_tmp->bus->number;
1810 devfn = dev_tmp->devfn;
1811 }
1812 spin_lock_irqsave(&device_domain_lock, flags);
1813 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1814 if (info->segment == segment &&
1815 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1816 found = info->domain;
1817 break;
1818 }
1819 }
1820 spin_unlock_irqrestore(&device_domain_lock, flags);
1821 /* pcie-pci bridge already has a domain, uses it */
1822 if (found) {
1823 domain = found;
1824 goto found_domain;
1825 }
1826 }
1827
2c2e2c38
FY		 1828 domain = alloc_domain();
1829 if (!domain)
1830 goto error;
1831
ba395927
KA		 1832 /* Allocate new domain for the device */
1833 drhd = dmar_find_matched_drhd_unit(pdev);
1834 if (!drhd) {
1835 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1836 pci_name(pdev));
1837 return NULL;
1838 }
1839 iommu = drhd->iommu;
1840
2c2e2c38
FY		 1841 ret = iommu_attach_domain(domain, iommu);
1842 if (ret) {
2fe9723d 1843 free_domain_mem(domain);
ba395927 1844 goto error;
2c2e2c38 1845 }
ba395927
KA		 1846
1847 if (domain_init(domain, gaw)) {
1848 domain_exit(domain);
1849 goto error;
1850 }
1851
1852 /* register pcie-to-pci device */
1853 if (dev_tmp) {
1854 info = alloc_devinfo_mem();
1855 if (!info) {
1856 domain_exit(domain);
1857 goto error;
1858 }
276dbf99 1859 info->segment = segment;
ba395927
KA		 1860 info->bus = bus;
1861 info->devfn = devfn;
1862 info->dev = NULL;
1863 info->domain = domain;
1864 /* This domain is shared by devices under p2p bridge */
3b5410e7 1865 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1866
1867 /* pcie-to-pci bridge already has a domain, uses it */
1868 found = NULL;
1869 spin_lock_irqsave(&device_domain_lock, flags);
1870 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 1871 if (tmp->segment == segment &&
1872 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 1873 found = tmp->domain;
1874 break;
1875 }
1876 }
1877 if (found) {
00dfff77 1878 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1879 free_devinfo_mem(info);
1880 domain_exit(domain);
1881 domain = found;
1882 } else {
1883 list_add(&info->link, &domain->devices);
1884 list_add(&info->global, &device_domain_list);
00dfff77 1885 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 1886 }
ba395927
KA		 1887 }
1888
1889found_domain:
1890 info = alloc_devinfo_mem();
1891 if (!info)
1892 goto error;
276dbf99 1893 info->segment = segment;
ba395927
KA		 1894 info->bus = pdev->bus->number;
1895 info->devfn = pdev->devfn;
1896 info->dev = pdev;
1897 info->domain = domain;
1898 spin_lock_irqsave(&device_domain_lock, flags);
1899 /* somebody is fast */
1900 found = find_domain(pdev);
1901 if (found != NULL) {
1902 spin_unlock_irqrestore(&device_domain_lock, flags);
1903 if (found != domain) {
1904 domain_exit(domain);
1905 domain = found;
1906 }
1907 free_devinfo_mem(info);
1908 return domain;
1909 }
1910 list_add(&info->link, &domain->devices);
1911 list_add(&info->global, &device_domain_list);
358dd8ac 1912 pdev->dev.archdata.iommu = info;
ba395927
KA		 1913 spin_unlock_irqrestore(&device_domain_lock, flags);
1914 return domain;
1915error:
1916 /* recheck it here, maybe others set it */
1917 return find_domain(pdev);
1918}
1919
2c2e2c38 1920static int iommu_identity_mapping;
e0fc7e0b
DW		 1921#define IDENTMAP_ALL 1
1922#define IDENTMAP_GFX 2
1923#define IDENTMAP_AZALIA 4
2c2e2c38 1924
b213203e
DW		 1925static int iommu_domain_identity_map(struct dmar_domain *domain,
1926 unsigned long long start,
1927 unsigned long long end)
ba395927 1928{
c5395d5c
DW		 1929 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1930 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1931
1932 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1933 dma_to_mm_pfn(last_vpfn))) {
ba395927 1934 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 1935 return -ENOMEM;
ba395927
KA		 1936 }
1937
c5395d5c
DW		 1938 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1939 start, end, domain->id);
ba395927
KA		 1940 /*
1941 * RMRR range might have overlap with physical memory range,
1942 * clear it first
1943 */
c5395d5c 1944 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 1945
c5395d5c
DW		 1946 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1947 last_vpfn - first_vpfn + 1,
61df7443 1948 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 1949}
1950
1951static int iommu_prepare_identity_map(struct pci_dev *pdev,
1952 unsigned long long start,
1953 unsigned long long end)
1954{
1955 struct dmar_domain *domain;
1956 int ret;
1957
c7ab48d2 1958 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 1959 if (!domain)
1960 return -ENOMEM;
1961
19943b0e
DW		 1962 /* For _hardware_ passthrough, don't bother. But for software
1963 passthrough, we do it anyway -- it may indicate a memory
1964 range which is reserved in E820, so which didn't get set
1965 up to start with in si_domain */
1966 if (domain == si_domain && hw_pass_through) {
1967 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1968 pci_name(pdev), start, end);
1969 return 0;
1970 }
1971
1972 printk(KERN_INFO
1973 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1974 pci_name(pdev), start, end);
2ff729f5 1975
5595b528
DW		 1976 if (end < start) {
1977 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1978 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1979 dmi_get_system_info(DMI_BIOS_VENDOR),
1980 dmi_get_system_info(DMI_BIOS_VERSION),
1981 dmi_get_system_info(DMI_PRODUCT_VERSION));
1982 ret = -EIO;
1983 goto error;
1984 }
1985
2ff729f5
DW		 1986 if (end >> agaw_to_width(domain->agaw)) {
1987 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1988 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1989 agaw_to_width(domain->agaw),
1990 dmi_get_system_info(DMI_BIOS_VENDOR),
1991 dmi_get_system_info(DMI_BIOS_VERSION),
1992 dmi_get_system_info(DMI_PRODUCT_VERSION));
1993 ret = -EIO;
1994 goto error;
1995 }
19943b0e 1996
b213203e 1997 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 1998 if (ret)
1999 goto error;
2000
2001 /* context entry init */
4ed0d3e6 2002 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2003 if (ret)
2004 goto error;
2005
2006 return 0;
2007
2008 error:
ba395927
KA		 2009 domain_exit(domain);
2010 return ret;
ba395927
KA		 2011}
2012
2013static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2014 struct pci_dev *pdev)
2015{
358dd8ac 2016 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2017 return 0;
2018 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2019 rmrr->end_address + 1);
2020}
2021
49a0429e
KA		 2022#ifdef CONFIG_DMAR_FLOPPY_WA
2023static inline void iommu_prepare_isa(void)
2024{
2025 struct pci_dev *pdev;
2026 int ret;
2027
2028 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2029 if (!pdev)
2030 return;
2031
c7ab48d2 2032 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
49a0429e
KA		 2033 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2034
2035 if (ret)
c7ab48d2
DW		 2036 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2037 "floppy might not work\n");
49a0429e
KA		 2038
2039}
2040#else
2041static inline void iommu_prepare_isa(void)
2042{
2043 return;
2044}
2045#endif /* !CONFIG_DMAR_FLPY_WA */
2046
2c2e2c38 2047static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2
DW		 2048
2049static int __init si_domain_work_fn(unsigned long start_pfn,
2050 unsigned long end_pfn, void *datax)
2051{
2052 int *ret = datax;
2053
2054 *ret = iommu_domain_identity_map(si_domain,
2055 (uint64_t)start_pfn << PAGE_SHIFT,
2056 (uint64_t)end_pfn << PAGE_SHIFT);
2057 return *ret;
2058
2059}
2060
071e1374 2061static int __init si_domain_init(int hw)
2c2e2c38
FY		 2062{
2063 struct dmar_drhd_unit *drhd;
2064 struct intel_iommu *iommu;
c7ab48d2 2065 int nid, ret = 0;
2c2e2c38
FY		 2066
2067 si_domain = alloc_domain();
2068 if (!si_domain)
2069 return -EFAULT;
2070
c7ab48d2 2071 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2072
2073 for_each_active_iommu(iommu, drhd) {
2074 ret = iommu_attach_domain(si_domain, iommu);
2075 if (ret) {
2076 domain_exit(si_domain);
2077 return -EFAULT;
2078 }
2079 }
2080
2081 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2082 domain_exit(si_domain);
2083 return -EFAULT;
2084 }
2085
2086 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2087
19943b0e
DW		 2088 if (hw)
2089 return 0;
2090
c7ab48d2
DW		 2091 for_each_online_node(nid) {
2092 work_with_active_regions(nid, si_domain_work_fn, &ret);
2093 if (ret)
2094 return ret;
2095 }
2096
2c2e2c38
FY		 2097 return 0;
2098}
2099
2100static void domain_remove_one_dev_info(struct dmar_domain *domain,
2101 struct pci_dev *pdev);
2102static int identity_mapping(struct pci_dev *pdev)
2103{
2104 struct device_domain_info *info;
2105
2106 if (likely(!iommu_identity_mapping))
2107 return 0;
2108
2109
2110 list_for_each_entry(info, &si_domain->devices, link)
2111 if (info->dev == pdev)
2112 return 1;
2113 return 0;
2114}
2115
2116static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2117 struct pci_dev *pdev,
2118 int translation)
2c2e2c38
FY		 2119{
2120 struct device_domain_info *info;
2121 unsigned long flags;
5fe60f4e 2122 int ret;
2c2e2c38
FY		 2123
2124 info = alloc_devinfo_mem();
2125 if (!info)
2126 return -ENOMEM;
2127
5fe60f4e
DW		 2128 ret = domain_context_mapping(domain, pdev, translation);
2129 if (ret) {
2130 free_devinfo_mem(info);
2131 return ret;
2132 }
2133
2c2e2c38
FY		 2134 info->segment = pci_domain_nr(pdev->bus);
2135 info->bus = pdev->bus->number;
2136 info->devfn = pdev->devfn;
2137 info->dev = pdev;
2138 info->domain = domain;
2139
2140 spin_lock_irqsave(&device_domain_lock, flags);
2141 list_add(&info->link, &domain->devices);
2142 list_add(&info->global, &device_domain_list);
2143 pdev->dev.archdata.iommu = info;
2144 spin_unlock_irqrestore(&device_domain_lock, flags);
2145
2146 return 0;
2147}
2148
6941af28
DW		 2149static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2150{
e0fc7e0b
DW		 2151 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2152 return 1;
2153
2154 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2155 return 1;
2156
2157 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2158 return 0;
6941af28 2159
3dfc813d
DW		 2160 /*
2161 * We want to start off with all devices in the 1:1 domain, and
2162 * take them out later if we find they can't access all of memory.
2163 *
2164 * However, we can't do this for PCI devices behind bridges,
2165 * because all PCI devices behind the same bridge will end up
2166 * with the same source-id on their transactions.
2167 *
2168 * Practically speaking, we can't change things around for these
2169 * devices at run-time, because we can't be sure there'll be no
2170 * DMA transactions in flight for any of their siblings.
2171 *
2172 * So PCI devices (unless they're on the root bus) as well as
2173 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2174 * the 1:1 domain, just in _case_ one of their siblings turns out
2175 * not to be able to map all of memory.
2176 */
5f4d91a1 2177 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2178 if (!pci_is_root_bus(pdev->bus))
2179 return 0;
2180 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2181 return 0;
2182 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2183 return 0;
2184
2185 /*
2186 * At boot time, we don't yet know if devices will be 64-bit capable.
2187 * Assume that they will -- if they turn out not to be, then we can
2188 * take them out of the 1:1 domain later.
2189 */
6941af28
DW		 2190 if (!startup)
2191 return pdev->dma_mask > DMA_BIT_MASK(32);
2192
2193 return 1;
2194}
2195
071e1374 2196static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2197{
2c2e2c38
FY		 2198 struct pci_dev *pdev = NULL;
2199 int ret;
2200
19943b0e 2201 ret = si_domain_init(hw);
2c2e2c38
FY		 2202 if (ret)
2203 return -EFAULT;
2204
2c2e2c38 2205 for_each_pci_dev(pdev) {
6941af28 2206 if (iommu_should_identity_map(pdev, 1)) {
19943b0e
DW		 2207 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2208 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2209
5fe60f4e 2210 ret = domain_add_dev_info(si_domain, pdev,
19943b0e 2211 hw ? CONTEXT_TT_PASS_THROUGH :
62edf5dc
DW		 2212 CONTEXT_TT_MULTI_LEVEL);
2213 if (ret)
2214 return ret;
62edf5dc 2215 }
2c2e2c38
FY		 2216 }
2217
2218 return 0;
2219}
2220
51a63e67 2221static int __init init_dmars(int force_on)
ba395927
KA		 2222{
2223 struct dmar_drhd_unit *drhd;
2224 struct dmar_rmrr_unit *rmrr;
2225 struct pci_dev *pdev;
2226 struct intel_iommu *iommu;
9d783ba0 2227 int i, ret;
2c2e2c38 2228
ba395927
KA		 2229 /*
2230 * for each drhd
2231 * allocate root
2232 * initialize and program root entry to not present
2233 * endfor
2234 */
2235 for_each_drhd_unit(drhd) {
5e0d2a6f 2236 g_num_of_iommus++;
2237 /*
2238 * lock not needed as this is only incremented in the single
2239 * threaded kernel __init code path all other access are read
2240 * only
2241 */
2242 }
2243
d9630fe9
WH		 2244 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2245 GFP_KERNEL);
2246 if (!g_iommus) {
2247 printk(KERN_ERR "Allocating global iommu array failed\n");
2248 ret = -ENOMEM;
2249 goto error;
2250 }
2251
80b20dd8 2252 deferred_flush = kzalloc(g_num_of_iommus *
2253 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2254 if (!deferred_flush) {
5e0d2a6f 2255 ret = -ENOMEM;
2256 goto error;
2257 }
2258
5e0d2a6f 2259 for_each_drhd_unit(drhd) {
2260 if (drhd->ignored)
2261 continue;
1886e8a9
SS		 2262
2263 iommu = drhd->iommu;
d9630fe9 2264 g_iommus[iommu->seq_id] = iommu;
ba395927 2265
e61d98d8
SS		 2266 ret = iommu_init_domains(iommu);
2267 if (ret)
2268 goto error;
2269
ba395927
KA		 2270 /*
2271 * TBD:
2272 * we could share the same root & context tables
25985edc 2273 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2274 */
2275 ret = iommu_alloc_root_entry(iommu);
2276 if (ret) {
2277 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2278 goto error;
2279 }
4ed0d3e6 2280 if (!ecap_pass_through(iommu->ecap))
19943b0e 2281 hw_pass_through = 0;
ba395927
KA		 2282 }
2283
1531a6a6
SS		 2284 /*
2285 * Start from the sane iommu hardware state.
2286 */
a77b67d4
YS		 2287 for_each_drhd_unit(drhd) {
2288 if (drhd->ignored)
2289 continue;
2290
2291 iommu = drhd->iommu;
1531a6a6
SS		 2292
2293 /*
2294 * If the queued invalidation is already initialized by us
2295 * (for example, while enabling interrupt-remapping) then
2296 * we got the things already rolling from a sane state.
2297 */
2298 if (iommu->qi)
2299 continue;
2300
2301 /*
2302 * Clear any previous faults.
2303 */
2304 dmar_fault(-1, iommu);
2305 /*
2306 * Disable queued invalidation if supported and already enabled
2307 * before OS handover.
2308 */
2309 dmar_disable_qi(iommu);
2310 }
2311
2312 for_each_drhd_unit(drhd) {
2313 if (drhd->ignored)
2314 continue;
2315
2316 iommu = drhd->iommu;
2317
a77b67d4
YS		 2318 if (dmar_enable_qi(iommu)) {
2319 /*
2320 * Queued Invalidate not enabled, use Register Based
2321 * Invalidate
2322 */
2323 iommu->flush.flush_context = __iommu_flush_context;
2324 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2325 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2326 "invalidation\n",
680a7524 2327 iommu->seq_id,
b4e0f9eb 2328 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2329 } else {
2330 iommu->flush.flush_context = qi_flush_context;
2331 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2332 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2333 "invalidation\n",
680a7524 2334 iommu->seq_id,
b4e0f9eb 2335 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2336 }
2337 }
2338
19943b0e 2339 if (iommu_pass_through)
e0fc7e0b
DW		 2340 iommu_identity_mapping |= IDENTMAP_ALL;
2341
19943b0e 2342#ifdef CONFIG_DMAR_BROKEN_GFX_WA
e0fc7e0b 2343 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2344#endif
e0fc7e0b
DW		 2345
2346 check_tylersburg_isoch();
2347
ba395927 2348 /*
19943b0e
DW		 2349 * If pass through is not set or not enabled, setup context entries for
2350 * identity mappings for rmrr, gfx, and isa and may fall back to static
2351 * identity mapping if iommu_identity_mapping is set.
ba395927 2352 */
19943b0e
DW		 2353 if (iommu_identity_mapping) {
2354 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2355 if (ret) {
19943b0e
DW		 2356 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2357 goto error;
ba395927
KA		 2358 }
2359 }
ba395927 2360 /*
19943b0e
DW		 2361 * For each rmrr
2362 * for each dev attached to rmrr
2363 * do
2364 * locate drhd for dev, alloc domain for dev
2365 * allocate free domain
2366 * allocate page table entries for rmrr
2367 * if context not allocated for bus
2368 * allocate and init context
2369 * set present in root table for this bus
2370 * init context with domain, translation etc
2371 * endfor
2372 * endfor
ba395927 2373 */
19943b0e
DW		 2374 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2375 for_each_rmrr_units(rmrr) {
2376 for (i = 0; i < rmrr->devices_cnt; i++) {
2377 pdev = rmrr->devices[i];
2378 /*
2379 * some BIOS lists non-exist devices in DMAR
2380 * table.
2381 */
2382 if (!pdev)
2383 continue;
2384 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2385 if (ret)
2386 printk(KERN_ERR
2387 "IOMMU: mapping reserved region failed\n");
ba395927 2388 }
4ed0d3e6 2389 }
49a0429e 2390
19943b0e
DW		 2391 iommu_prepare_isa();
2392
ba395927
KA		 2393 /*
2394 * for each drhd
2395 * enable fault log
2396 * global invalidate context cache
2397 * global invalidate iotlb
2398 * enable translation
2399 */
2400 for_each_drhd_unit(drhd) {
51a63e67
JC		 2401 if (drhd->ignored) {
2402 /*
2403 * we always have to disable PMRs or DMA may fail on
2404 * this device
2405 */
2406 if (force_on)
2407 iommu_disable_protect_mem_regions(drhd->iommu);
ba395927 2408 continue;
51a63e67 2409 }
ba395927 2410 iommu = drhd->iommu;
ba395927
KA		 2411
2412 iommu_flush_write_buffer(iommu);
2413
3460a6d9
KA		 2414 ret = dmar_set_interrupt(iommu);
2415 if (ret)
2416 goto error;
2417
ba395927
KA		 2418 iommu_set_root_entry(iommu);
2419
4c25a2c1 2420 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2421 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2422
ba395927
KA		 2423 ret = iommu_enable_translation(iommu);
2424 if (ret)
2425 goto error;
b94996c9
DW		 2426
2427 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2428 }
2429
2430 return 0;
2431error:
2432 for_each_drhd_unit(drhd) {
2433 if (drhd->ignored)
2434 continue;
2435 iommu = drhd->iommu;
2436 free_iommu(iommu);
2437 }
d9630fe9 2438 kfree(g_iommus);
ba395927
KA		 2439 return ret;
2440}
2441
5a5e02a6 2442/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2443static struct iova *intel_alloc_iova(struct device *dev,
2444 struct dmar_domain *domain,
2445 unsigned long nrpages, uint64_t dma_mask)
ba395927 2446{
ba395927 2447 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2448 struct iova *iova = NULL;
ba395927 2449
875764de
DW		 2450 /* Restrict dma_mask to the width that the iommu can handle */
2451 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2452
2453 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2454 /*
2455 * First try to allocate an io virtual address in
284901a9 2456 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2457 * from higher range
ba395927 2458 */
875764de
DW		 2459 iova = alloc_iova(&domain->iovad, nrpages,
2460 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2461 if (iova)
2462 return iova;
2463 }
2464 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2465 if (unlikely(!iova)) {
2466 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2467 nrpages, pci_name(pdev));
f76aec76
KA		 2468 return NULL;
2469 }
2470
2471 return iova;
2472}
2473
147202aa 2474static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2475{
2476 struct dmar_domain *domain;
2477 int ret;
2478
2479 domain = get_domain_for_dev(pdev,
2480 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2481 if (!domain) {
2482 printk(KERN_ERR
2483 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2484 return NULL;
ba395927
KA		 2485 }
2486
2487 /* make sure context mapping is ok */
5331fe6f 2488 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2489 ret = domain_context_mapping(domain, pdev,
2490 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2491 if (ret) {
2492 printk(KERN_ERR
2493 "Domain context map for %s failed",
2494 pci_name(pdev));
4fe05bbc 2495 return NULL;
f76aec76 2496 }
ba395927
KA		 2497 }
2498
f76aec76
KA		 2499 return domain;
2500}
2501
147202aa
DW		 2502static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2503{
2504 struct device_domain_info *info;
2505
2506 /* No lock here, assumes no domain exit in normal case */
2507 info = dev->dev.archdata.iommu;
2508 if (likely(info))
2509 return info->domain;
2510
2511 return __get_valid_domain_for_dev(dev);
2512}
2513
2c2e2c38
FY		 2514static int iommu_dummy(struct pci_dev *pdev)
2515{
2516 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2517}
2518
2519/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2520static int iommu_no_mapping(struct device *dev)
2c2e2c38 2521{
73676832 2522 struct pci_dev *pdev;
2c2e2c38
FY		 2523 int found;
2524
73676832
DW		 2525 if (unlikely(dev->bus != &pci_bus_type))
2526 return 1;
2527
2528 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2529 if (iommu_dummy(pdev))
2530 return 1;
2531
2c2e2c38 2532 if (!iommu_identity_mapping)
1e4c64c4 2533 return 0;
2c2e2c38
FY		 2534
2535 found = identity_mapping(pdev);
2536 if (found) {
6941af28 2537 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2538 return 1;
2539 else {
2540 /*
2541 * 32 bit DMA is removed from si_domain and fall back
2542 * to non-identity mapping.
2543 */
2544 domain_remove_one_dev_info(si_domain, pdev);
2545 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2546 pci_name(pdev));
2547 return 0;
2548 }
2549 } else {
2550 /*
2551 * In case of a detached 64 bit DMA device from vm, the device
2552 * is put into si_domain for identity mapping.
2553 */
6941af28 2554 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2555 int ret;
5fe60f4e
DW		 2556 ret = domain_add_dev_info(si_domain, pdev,
2557 hw_pass_through ?
2558 CONTEXT_TT_PASS_THROUGH :
2559 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2560 if (!ret) {
2561 printk(KERN_INFO "64bit %s uses identity mapping\n",
2562 pci_name(pdev));
2563 return 1;
2564 }
2565 }
2566 }
2567
1e4c64c4 2568 return 0;
2c2e2c38
FY		 2569}
2570
bb9e6d65
FT		 2571static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2572 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2573{
2574 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2575 struct dmar_domain *domain;
5b6985ce 2576 phys_addr_t start_paddr;
f76aec76
KA		 2577 struct iova *iova;
2578 int prot = 0;
6865f0d1 2579 int ret;
8c11e798 2580 struct intel_iommu *iommu;
33041ec0 2581 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2582
2583 BUG_ON(dir == DMA_NONE);
2c2e2c38 2584
73676832 2585 if (iommu_no_mapping(hwdev))
6865f0d1 2586 return paddr;
f76aec76
KA		 2587
2588 domain = get_valid_domain_for_dev(pdev);
2589 if (!domain)
2590 return 0;
2591
8c11e798 2592 iommu = domain_get_iommu(domain);
88cb6a74 2593 size = aligned_nrpages(paddr, size);
f76aec76 2594
5a5e02a6
DW		 2595 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2596 pdev->dma_mask);
f76aec76
KA		 2597 if (!iova)
2598 goto error;
2599
ba395927
KA		 2600 /*
2601 * Check if DMAR supports zero-length reads on write only
2602 * mappings..
2603 */
2604 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2605 !cap_zlr(iommu->cap))
ba395927
KA		 2606 prot |= DMA_PTE_READ;
2607 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2608 prot |= DMA_PTE_WRITE;
2609 /*
6865f0d1 2610 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2611 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2612 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2613 * is not a big problem
2614 */
0ab36de2 2615 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2616 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2617 if (ret)
2618 goto error;
2619
1f0ef2aa
DW		 2620 /* it's a non-present to present mapping. Only flush if caching mode */
2621 if (cap_caching_mode(iommu->cap))
82653633 2622 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2623 else
8c11e798 2624 iommu_flush_write_buffer(iommu);
f76aec76 2625
03d6a246
DW		 2626 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2627 start_paddr += paddr & ~PAGE_MASK;
2628 return start_paddr;
ba395927 2629
ba395927 2630error:
f76aec76
KA		 2631 if (iova)
2632 __free_iova(&domain->iovad, iova);
4cf2e75d 2633 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2634 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2635 return 0;
2636}
2637
ffbbef5c
FT		 2638static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2639 unsigned long offset, size_t size,
2640 enum dma_data_direction dir,
2641 struct dma_attrs *attrs)
bb9e6d65 2642{
ffbbef5c
FT		 2643 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2644 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2645}
2646
5e0d2a6f 2647static void flush_unmaps(void)
2648{
80b20dd8 2649 int i, j;
5e0d2a6f 2650
5e0d2a6f 2651 timer_on = 0;
2652
2653 /* just flush them all */
2654 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2655 struct intel_iommu *iommu = g_iommus[i];
2656 if (!iommu)
2657 continue;
c42d9f32 2658
9dd2fe89
YZ		 2659 if (!deferred_flush[i].next)
2660 continue;
2661
78d5f0f5
NA		 2662 /* In caching mode, global flushes turn emulation expensive */
2663 if (!cap_caching_mode(iommu->cap))
2664 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2665 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2666 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2667 unsigned long mask;
2668 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2669 struct dmar_domain *domain = deferred_flush[i].domain[j];
2670
2671 /* On real hardware multiple invalidations are expensive */
2672 if (cap_caching_mode(iommu->cap))
2673 iommu_flush_iotlb_psi(iommu, domain->id,
2674 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2675 else {
2676 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2677 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2678 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2679 }
93a23a72 2680 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2681 }
9dd2fe89 2682 deferred_flush[i].next = 0;
5e0d2a6f 2683 }
2684
5e0d2a6f 2685 list_size = 0;
5e0d2a6f 2686}
2687
2688static void flush_unmaps_timeout(unsigned long data)
2689{
80b20dd8 2690 unsigned long flags;
2691
2692 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2693 flush_unmaps();
80b20dd8 2694 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2695}
2696
2697static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2698{
2699 unsigned long flags;
80b20dd8 2700 int next, iommu_id;
8c11e798 2701 struct intel_iommu *iommu;
5e0d2a6f 2702
2703 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2704 if (list_size == HIGH_WATER_MARK)
2705 flush_unmaps();
2706
8c11e798
WH		 2707 iommu = domain_get_iommu(dom);
2708 iommu_id = iommu->seq_id;
c42d9f32 2709
80b20dd8 2710 next = deferred_flush[iommu_id].next;
2711 deferred_flush[iommu_id].domain[next] = dom;
2712 deferred_flush[iommu_id].iova[next] = iova;
2713 deferred_flush[iommu_id].next++;
5e0d2a6f 2714
2715 if (!timer_on) {
2716 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2717 timer_on = 1;
2718 }
2719 list_size++;
2720 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2721}
2722
ffbbef5c
FT		 2723static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2724 size_t size, enum dma_data_direction dir,
2725 struct dma_attrs *attrs)
ba395927 2726{
ba395927 2727 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2728 struct dmar_domain *domain;
d794dc9b 2729 unsigned long start_pfn, last_pfn;
ba395927 2730 struct iova *iova;
8c11e798 2731 struct intel_iommu *iommu;
ba395927 2732
73676832 2733 if (iommu_no_mapping(dev))
f76aec76 2734 return;
2c2e2c38 2735
ba395927
KA		 2736 domain = find_domain(pdev);
2737 BUG_ON(!domain);
2738
8c11e798
WH		 2739 iommu = domain_get_iommu(domain);
2740
ba395927 2741 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2742 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2743 (unsigned long long)dev_addr))
ba395927 2744 return;
ba395927 2745
d794dc9b
DW		 2746 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2747 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2748
d794dc9b
DW		 2749 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2750 pci_name(pdev), start_pfn, last_pfn);
ba395927 2751
f76aec76 2752 /* clear the whole page */
d794dc9b
DW		 2753 dma_pte_clear_range(domain, start_pfn, last_pfn);
2754
f76aec76 2755 /* free page tables */
d794dc9b
DW		 2756 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2757
5e0d2a6f 2758 if (intel_iommu_strict) {
03d6a246 2759 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2760 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2761 /* free iova */
2762 __free_iova(&domain->iovad, iova);
2763 } else {
2764 add_unmap(domain, iova);
2765 /*
2766 * queue up the release of the unmap to save the 1/6th of the
2767 * cpu used up by the iotlb flush operation...
2768 */
5e0d2a6f 2769 }
ba395927
KA		 2770}
2771
d7ab5c46
FT		 2772static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2773 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2774{
2775 void *vaddr;
2776 int order;
2777
5b6985ce 2778 size = PAGE_ALIGN(size);
ba395927 2779 order = get_order(size);
e8bb910d
AW		 2780
2781 if (!iommu_no_mapping(hwdev))
2782 flags &= ~(GFP_DMA | GFP_DMA32);
2783 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2784 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2785 flags |= GFP_DMA;
2786 else
2787 flags |= GFP_DMA32;
2788 }
ba395927
KA		 2789
2790 vaddr = (void *)__get_free_pages(flags, order);
2791 if (!vaddr)
2792 return NULL;
2793 memset(vaddr, 0, size);
2794
bb9e6d65
FT		 2795 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2796 DMA_BIDIRECTIONAL,
2797 hwdev->coherent_dma_mask);
ba395927
KA		 2798 if (*dma_handle)
2799 return vaddr;
2800 free_pages((unsigned long)vaddr, order);
2801 return NULL;
2802}
2803
d7ab5c46
FT		 2804static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2805 dma_addr_t dma_handle)
ba395927
KA		 2806{
2807 int order;
2808
5b6985ce 2809 size = PAGE_ALIGN(size);
ba395927
KA		 2810 order = get_order(size);
2811
0db9b7ae 2812 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 2813 free_pages((unsigned long)vaddr, order);
2814}
2815
d7ab5c46
FT		 2816static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2817 int nelems, enum dma_data_direction dir,
2818 struct dma_attrs *attrs)
ba395927 2819{
ba395927
KA		 2820 struct pci_dev *pdev = to_pci_dev(hwdev);
2821 struct dmar_domain *domain;
d794dc9b 2822 unsigned long start_pfn, last_pfn;
f76aec76 2823 struct iova *iova;
8c11e798 2824 struct intel_iommu *iommu;
ba395927 2825
73676832 2826 if (iommu_no_mapping(hwdev))
ba395927
KA		 2827 return;
2828
2829 domain = find_domain(pdev);
8c11e798
WH		 2830 BUG_ON(!domain);
2831
2832 iommu = domain_get_iommu(domain);
ba395927 2833
c03ab37c 2834 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 2835 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2836 (unsigned long long)sglist[0].dma_address))
f76aec76 2837 return;
f76aec76 2838
d794dc9b
DW		 2839 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2840 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 2841
2842 /* clear the whole page */
d794dc9b
DW		 2843 dma_pte_clear_range(domain, start_pfn, last_pfn);
2844
f76aec76 2845 /* free page tables */
d794dc9b 2846 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 2847
acea0018
DW		 2848 if (intel_iommu_strict) {
2849 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2850 last_pfn - start_pfn + 1, 0);
acea0018
DW		 2851 /* free iova */
2852 __free_iova(&domain->iovad, iova);
2853 } else {
2854 add_unmap(domain, iova);
2855 /*
2856 * queue up the release of the unmap to save the 1/6th of the
2857 * cpu used up by the iotlb flush operation...
2858 */
2859 }
ba395927
KA		 2860}
2861
ba395927 2862static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2863 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2864{
2865 int i;
c03ab37c 2866 struct scatterlist *sg;
ba395927 2867
c03ab37c 2868 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2869 BUG_ON(!sg_page(sg));
4cf2e75d 2870 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 2871 sg->dma_length = sg->length;
ba395927
KA		 2872 }
2873 return nelems;
2874}
2875
d7ab5c46
FT		 2876static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2877 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 2878{
ba395927 2879 int i;
ba395927
KA		 2880 struct pci_dev *pdev = to_pci_dev(hwdev);
2881 struct dmar_domain *domain;
f76aec76
KA		 2882 size_t size = 0;
2883 int prot = 0;
f76aec76
KA		 2884 struct iova *iova = NULL;
2885 int ret;
c03ab37c 2886 struct scatterlist *sg;
b536d24d 2887 unsigned long start_vpfn;
8c11e798 2888 struct intel_iommu *iommu;
ba395927
KA		 2889
2890 BUG_ON(dir == DMA_NONE);
73676832 2891 if (iommu_no_mapping(hwdev))
c03ab37c 2892 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 2893
f76aec76
KA		 2894 domain = get_valid_domain_for_dev(pdev);
2895 if (!domain)
2896 return 0;
2897
8c11e798
WH		 2898 iommu = domain_get_iommu(domain);
2899
b536d24d 2900 for_each_sg(sglist, sg, nelems, i)
88cb6a74 2901 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 2902
5a5e02a6
DW		 2903 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2904 pdev->dma_mask);
f76aec76 2905 if (!iova) {
c03ab37c 2906 sglist->dma_length = 0;
f76aec76
KA		 2907 return 0;
2908 }
2909
2910 /*
2911 * Check if DMAR supports zero-length reads on write only
2912 * mappings..
2913 */
2914 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2915 !cap_zlr(iommu->cap))
f76aec76
KA		 2916 prot |= DMA_PTE_READ;
2917 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2918 prot |= DMA_PTE_WRITE;
2919
b536d24d 2920 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 2921
f532959b 2922 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 2923 if (unlikely(ret)) {
2924 /* clear the page */
2925 dma_pte_clear_range(domain, start_vpfn,
2926 start_vpfn + size - 1);
2927 /* free page tables */
2928 dma_pte_free_pagetable(domain, start_vpfn,
2929 start_vpfn + size - 1);
2930 /* free iova */
2931 __free_iova(&domain->iovad, iova);
2932 return 0;
ba395927
KA		 2933 }
2934
1f0ef2aa
DW		 2935 /* it's a non-present to present mapping. Only flush if caching mode */
2936 if (cap_caching_mode(iommu->cap))
82653633 2937 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 2938 else
8c11e798 2939 iommu_flush_write_buffer(iommu);
1f0ef2aa 2940
ba395927
KA		 2941 return nelems;
2942}
2943
dfb805e8
FT		 2944static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2945{
2946 return !dma_addr;
2947}
2948
160c1d8e 2949struct dma_map_ops intel_dma_ops = {
ba395927
KA		 2950 .alloc_coherent = intel_alloc_coherent,
2951 .free_coherent = intel_free_coherent,
ba395927
KA		 2952 .map_sg = intel_map_sg,
2953 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 2954 .map_page = intel_map_page,
2955 .unmap_page = intel_unmap_page,
dfb805e8 2956 .mapping_error = intel_mapping_error,
ba395927
KA		 2957};
2958
2959static inline int iommu_domain_cache_init(void)
2960{
2961 int ret = 0;
2962
2963 iommu_domain_cache = kmem_cache_create("iommu_domain",
2964 sizeof(struct dmar_domain),
2965 0,
2966 SLAB_HWCACHE_ALIGN,
2967
2968 NULL);
2969 if (!iommu_domain_cache) {
2970 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2971 ret = -ENOMEM;
2972 }
2973
2974 return ret;
2975}
2976
2977static inline int iommu_devinfo_cache_init(void)
2978{
2979 int ret = 0;
2980
2981 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2982 sizeof(struct device_domain_info),
2983 0,
2984 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2985 NULL);
2986 if (!iommu_devinfo_cache) {
2987 printk(KERN_ERR "Couldn't create devinfo cache\n");
2988 ret = -ENOMEM;
2989 }
2990
2991 return ret;
2992}
2993
2994static inline int iommu_iova_cache_init(void)
2995{
2996 int ret = 0;
2997
2998 iommu_iova_cache = kmem_cache_create("iommu_iova",
2999 sizeof(struct iova),
3000 0,
3001 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3002 NULL);
3003 if (!iommu_iova_cache) {
3004 printk(KERN_ERR "Couldn't create iova cache\n");
3005 ret = -ENOMEM;
3006 }
3007
3008 return ret;
3009}
3010
3011static int __init iommu_init_mempool(void)
3012{
3013 int ret;
3014 ret = iommu_iova_cache_init();
3015 if (ret)
3016 return ret;
3017
3018 ret = iommu_domain_cache_init();
3019 if (ret)
3020 goto domain_error;
3021
3022 ret = iommu_devinfo_cache_init();
3023 if (!ret)
3024 return ret;
3025
3026 kmem_cache_destroy(iommu_domain_cache);
3027domain_error:
3028 kmem_cache_destroy(iommu_iova_cache);
3029
3030 return -ENOMEM;
3031}
3032
3033static void __init iommu_exit_mempool(void)
3034{
3035 kmem_cache_destroy(iommu_devinfo_cache);
3036 kmem_cache_destroy(iommu_domain_cache);
3037 kmem_cache_destroy(iommu_iova_cache);
3038
3039}
3040
556ab45f
DW		 3041static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3042{
3043 struct dmar_drhd_unit *drhd;
3044 u32 vtbar;
3045 int rc;
3046
3047 /* We know that this device on this chipset has its own IOMMU.
3048 * If we find it under a different IOMMU, then the BIOS is lying
3049 * to us. Hope that the IOMMU for this device is actually
3050 * disabled, and it needs no translation...
3051 */
3052 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3053 if (rc) {
3054 /* "can't" happen */
3055 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3056 return;
3057 }
3058 vtbar &= 0xffff0000;
3059
3060 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3061 drhd = dmar_find_matched_drhd_unit(pdev);
3062 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3063 TAINT_FIRMWARE_WORKAROUND,
3064 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3065 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3066}
3067DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3068
ba395927
KA		 3069static void __init init_no_remapping_devices(void)
3070{
3071 struct dmar_drhd_unit *drhd;
3072
3073 for_each_drhd_unit(drhd) {
3074 if (!drhd->include_all) {
3075 int i;
3076 for (i = 0; i < drhd->devices_cnt; i++)
3077 if (drhd->devices[i] != NULL)
3078 break;
3079 /* ignore DMAR unit if no pci devices exist */
3080 if (i == drhd->devices_cnt)
3081 drhd->ignored = 1;
3082 }
3083 }
3084
3085 if (dmar_map_gfx)
3086 return;
3087
3088 for_each_drhd_unit(drhd) {
3089 int i;
3090 if (drhd->ignored || drhd->include_all)
3091 continue;
3092
3093 for (i = 0; i < drhd->devices_cnt; i++)
3094 if (drhd->devices[i] &&
3095 !IS_GFX_DEVICE(drhd->devices[i]))
3096 break;
3097
3098 if (i < drhd->devices_cnt)
3099 continue;
3100
3101 /* bypass IOMMU if it is just for gfx devices */
3102 drhd->ignored = 1;
3103 for (i = 0; i < drhd->devices_cnt; i++) {
3104 if (!drhd->devices[i])
3105 continue;
358dd8ac 3106 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3107 }
3108 }
3109}
3110
f59c7b69
FY		 3111#ifdef CONFIG_SUSPEND
3112static int init_iommu_hw(void)
3113{
3114 struct dmar_drhd_unit *drhd;
3115 struct intel_iommu *iommu = NULL;
3116
3117 for_each_active_iommu(iommu, drhd)
3118 if (iommu->qi)
3119 dmar_reenable_qi(iommu);
3120
3121 for_each_active_iommu(iommu, drhd) {
3122 iommu_flush_write_buffer(iommu);
3123
3124 iommu_set_root_entry(iommu);
3125
3126 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3127 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3128 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3129 DMA_TLB_GLOBAL_FLUSH);
f59c7b69 3130 iommu_enable_translation(iommu);
b94996c9 3131 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3132 }
3133
3134 return 0;
3135}
3136
3137static void iommu_flush_all(void)
3138{
3139 struct dmar_drhd_unit *drhd;
3140 struct intel_iommu *iommu;
3141
3142 for_each_active_iommu(iommu, drhd) {
3143 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3144 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3145 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3146 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3147 }
3148}
3149
134fac3f 3150static int iommu_suspend(void)
f59c7b69
FY		 3151{
3152 struct dmar_drhd_unit *drhd;
3153 struct intel_iommu *iommu = NULL;
3154 unsigned long flag;
3155
3156 for_each_active_iommu(iommu, drhd) {
3157 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3158 GFP_ATOMIC);
3159 if (!iommu->iommu_state)
3160 goto nomem;
3161 }
3162
3163 iommu_flush_all();
3164
3165 for_each_active_iommu(iommu, drhd) {
3166 iommu_disable_translation(iommu);
3167
3168 spin_lock_irqsave(&iommu->register_lock, flag);
3169
3170 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3171 readl(iommu->reg + DMAR_FECTL_REG);
3172 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3173 readl(iommu->reg + DMAR_FEDATA_REG);
3174 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3175 readl(iommu->reg + DMAR_FEADDR_REG);
3176 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3177 readl(iommu->reg + DMAR_FEUADDR_REG);
3178
3179 spin_unlock_irqrestore(&iommu->register_lock, flag);
3180 }
3181 return 0;
3182
3183nomem:
3184 for_each_active_iommu(iommu, drhd)
3185 kfree(iommu->iommu_state);
3186
3187 return -ENOMEM;
3188}
3189
134fac3f 3190static void iommu_resume(void)
f59c7b69
FY		 3191{
3192 struct dmar_drhd_unit *drhd;
3193 struct intel_iommu *iommu = NULL;
3194 unsigned long flag;
3195
3196 if (init_iommu_hw()) {
3197 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3198 return;
f59c7b69
FY		 3199 }
3200
3201 for_each_active_iommu(iommu, drhd) {
3202
3203 spin_lock_irqsave(&iommu->register_lock, flag);
3204
3205 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3206 iommu->reg + DMAR_FECTL_REG);
3207 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3208 iommu->reg + DMAR_FEDATA_REG);
3209 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3210 iommu->reg + DMAR_FEADDR_REG);
3211 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3212 iommu->reg + DMAR_FEUADDR_REG);
3213
3214 spin_unlock_irqrestore(&iommu->register_lock, flag);
3215 }
3216
3217 for_each_active_iommu(iommu, drhd)
3218 kfree(iommu->iommu_state);
f59c7b69
FY		 3219}
3220
134fac3f 3221static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3222 .resume = iommu_resume,
3223 .suspend = iommu_suspend,
3224};
3225
134fac3f 3226static void __init init_iommu_pm_ops(void)
f59c7b69 3227{
134fac3f 3228 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3229}
3230
3231#else
134fac3f 3232static inline int init_iommu_pm_ops(void) { }
f59c7b69
FY		 3233#endif /* CONFIG_PM */
3234
99dcaded
FY		 3235/*
3236 * Here we only respond to action of unbound device from driver.
3237 *
3238 * Added device is not attached to its DMAR domain here yet. That will happen
3239 * when mapping the device to iova.
3240 */
3241static int device_notifier(struct notifier_block *nb,
3242 unsigned long action, void *data)
3243{
3244 struct device *dev = data;
3245 struct pci_dev *pdev = to_pci_dev(dev);
3246 struct dmar_domain *domain;
3247
44cd613c
DW		 3248 if (iommu_no_mapping(dev))
3249 return 0;
3250
99dcaded
FY		 3251 domain = find_domain(pdev);
3252 if (!domain)
3253 return 0;
3254
a97590e5 3255 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3256 domain_remove_one_dev_info(domain, pdev);
3257
a97590e5
AW		 3258 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3259 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3260 list_empty(&domain->devices))
3261 domain_exit(domain);
3262 }
3263
99dcaded
FY		 3264 return 0;
3265}
3266
3267static struct notifier_block device_nb = {
3268 .notifier_call = device_notifier,
3269};
3270
ba395927
KA		 3271int __init intel_iommu_init(void)
3272{
3273 int ret = 0;
a59b50e9 3274 int force_on = 0;
ba395927 3275
a59b50e9
JC		 3276 /* VT-d is required for a TXT/tboot launch, so enforce that */
3277 force_on = tboot_force_iommu();
3278
3279 if (dmar_table_init()) {
3280 if (force_on)
3281 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3282 return -ENODEV;
a59b50e9 3283 }
ba395927 3284
a59b50e9
JC		 3285 if (dmar_dev_scope_init()) {
3286 if (force_on)
3287 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3288 return -ENODEV;
a59b50e9 3289 }
1886e8a9 3290
2ae21010
SS		 3291 /*
3292 * Check the need for DMA-remapping initialization now.
3293 * Above initialization will also be used by Interrupt-remapping.
3294 */
75f1cdf1 3295 if (no_iommu || dmar_disabled)
2ae21010
SS		 3296 return -ENODEV;
3297
51a63e67
JC		 3298 if (iommu_init_mempool()) {
3299 if (force_on)
3300 panic("tboot: Failed to initialize iommu memory\n");
3301 return -ENODEV;
3302 }
3303
3304 if (dmar_init_reserved_ranges()) {
3305 if (force_on)
3306 panic("tboot: Failed to reserve iommu ranges\n");
3307 return -ENODEV;
3308 }
ba395927
KA		 3309
3310 init_no_remapping_devices();
3311
51a63e67 3312 ret = init_dmars(force_on);
ba395927 3313 if (ret) {
a59b50e9
JC		 3314 if (force_on)
3315 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3316 printk(KERN_ERR "IOMMU: dmar init failed\n");
3317 put_iova_domain(&reserved_iova_list);
3318 iommu_exit_mempool();
3319 return ret;
3320 }
3321 printk(KERN_INFO
3322 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3323
5e0d2a6f 3324 init_timer(&unmap_timer);
75f1cdf1
FT		 3325#ifdef CONFIG_SWIOTLB
3326 swiotlb = 0;
3327#endif
19943b0e 3328 dma_ops = &intel_dma_ops;
4ed0d3e6 3329
134fac3f 3330 init_iommu_pm_ops();
a8bcbb0d
JR		 3331
3332 register_iommu(&intel_iommu_ops);
3333
99dcaded
FY		 3334 bus_register_notifier(&pci_bus_type, &device_nb);
3335
ba395927
KA		 3336 return 0;
3337}
e820482c 3338
3199aa6b
HW		 3339static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3340 struct pci_dev *pdev)
3341{
3342 struct pci_dev *tmp, *parent;
3343
3344 if (!iommu || !pdev)
3345 return;
3346
3347 /* dependent device detach */
3348 tmp = pci_find_upstream_pcie_bridge(pdev);
3349 /* Secondary interface's bus number and devfn 0 */
3350 if (tmp) {
3351 parent = pdev->bus->self;
3352 while (parent != tmp) {
3353 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3354 parent->devfn);
3199aa6b
HW		 3355 parent = parent->bus->self;
3356 }
45e829ea 3357 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3358 iommu_detach_dev(iommu,
3359 tmp->subordinate->number, 0);
3360 else /* this is a legacy PCI bridge */
276dbf99
DW		 3361 iommu_detach_dev(iommu, tmp->bus->number,
3362 tmp->devfn);
3199aa6b
HW		 3363 }
3364}
3365
2c2e2c38 3366static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3367 struct pci_dev *pdev)
3368{
3369 struct device_domain_info *info;
3370 struct intel_iommu *iommu;
3371 unsigned long flags;
3372 int found = 0;
3373 struct list_head *entry, *tmp;
3374
276dbf99
DW		 3375 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3376 pdev->devfn);
c7151a8d
WH		 3377 if (!iommu)
3378 return;
3379
3380 spin_lock_irqsave(&device_domain_lock, flags);
3381 list_for_each_safe(entry, tmp, &domain->devices) {
3382 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3383 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3384 if (info->bus == pdev->bus->number &&
3385 info->devfn == pdev->devfn) {
3386 list_del(&info->link);
3387 list_del(&info->global);
3388 if (info->dev)
3389 info->dev->dev.archdata.iommu = NULL;
3390 spin_unlock_irqrestore(&device_domain_lock, flags);
3391
93a23a72 3392 iommu_disable_dev_iotlb(info);
c7151a8d 3393 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3394 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3395 free_devinfo_mem(info);
3396
3397 spin_lock_irqsave(&device_domain_lock, flags);
3398
3399 if (found)
3400 break;
3401 else
3402 continue;
3403 }
3404
3405 /* if there is no other devices under the same iommu
3406 * owned by this domain, clear this iommu in iommu_bmp
3407 * update iommu count and coherency
3408 */
276dbf99
DW		 3409 if (iommu == device_to_iommu(info->segment, info->bus,
3410 info->devfn))
c7151a8d
WH		 3411 found = 1;
3412 }
3413
3414 if (found == 0) {
3415 unsigned long tmp_flags;
3416 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3417 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3418 domain->iommu_count--;
58c610bd 3419 domain_update_iommu_cap(domain);
c7151a8d 3420 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5
AW		 3421
3422 spin_lock_irqsave(&iommu->lock, tmp_flags);
3423 clear_bit(domain->id, iommu->domain_ids);
3424 iommu->domains[domain->id] = NULL;
3425 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
c7151a8d
WH		 3426 }
3427
3428 spin_unlock_irqrestore(&device_domain_lock, flags);
3429}
3430
3431static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3432{
3433 struct device_domain_info *info;
3434 struct intel_iommu *iommu;
3435 unsigned long flags1, flags2;
3436
3437 spin_lock_irqsave(&device_domain_lock, flags1);
3438 while (!list_empty(&domain->devices)) {
3439 info = list_entry(domain->devices.next,
3440 struct device_domain_info, link);
3441 list_del(&info->link);
3442 list_del(&info->global);
3443 if (info->dev)
3444 info->dev->dev.archdata.iommu = NULL;
3445
3446 spin_unlock_irqrestore(&device_domain_lock, flags1);
3447
93a23a72 3448 iommu_disable_dev_iotlb(info);
276dbf99 3449 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3450 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3451 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3452
3453 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3454 * and capabilities
c7151a8d
WH		 3455 */
3456 spin_lock_irqsave(&domain->iommu_lock, flags2);
3457 if (test_and_clear_bit(iommu->seq_id,
3458 &domain->iommu_bmp)) {
3459 domain->iommu_count--;
58c610bd 3460 domain_update_iommu_cap(domain);
c7151a8d
WH		 3461 }
3462 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3463
3464 free_devinfo_mem(info);
3465 spin_lock_irqsave(&device_domain_lock, flags1);
3466 }
3467 spin_unlock_irqrestore(&device_domain_lock, flags1);
3468}
3469
5e98c4b1
WH		 3470/* domain id for virtual machine, it won't be set in context */
3471static unsigned long vm_domid;
3472
3473static struct dmar_domain *iommu_alloc_vm_domain(void)
3474{
3475 struct dmar_domain *domain;
3476
3477 domain = alloc_domain_mem();
3478 if (!domain)
3479 return NULL;
3480
3481 domain->id = vm_domid++;
4c923d47 3482 domain->nid = -1;
5e98c4b1
WH		 3483 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3484 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3485
3486 return domain;
3487}
3488
2c2e2c38 3489static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3490{
3491 int adjust_width;
3492
3493 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3494 spin_lock_init(&domain->iommu_lock);
3495
3496 domain_reserve_special_ranges(domain);
3497
3498 /* calculate AGAW */
3499 domain->gaw = guest_width;
3500 adjust_width = guestwidth_to_adjustwidth(guest_width);
3501 domain->agaw = width_to_agaw(adjust_width);
3502
3503 INIT_LIST_HEAD(&domain->devices);
3504
3505 domain->iommu_count = 0;
3506 domain->iommu_coherency = 0;
c5b15255 3507 domain->iommu_snooping = 0;
fe40f1e0 3508 domain->max_addr = 0;
4c923d47 3509 domain->nid = -1;
5e98c4b1
WH		 3510
3511 /* always allocate the top pgd */
4c923d47 3512 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3513 if (!domain->pgd)
3514 return -ENOMEM;
3515 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3516 return 0;
3517}
3518
3519static void iommu_free_vm_domain(struct dmar_domain *domain)
3520{
3521 unsigned long flags;
3522 struct dmar_drhd_unit *drhd;
3523 struct intel_iommu *iommu;
3524 unsigned long i;
3525 unsigned long ndomains;
3526
3527 for_each_drhd_unit(drhd) {
3528 if (drhd->ignored)
3529 continue;
3530 iommu = drhd->iommu;
3531
3532 ndomains = cap_ndoms(iommu->cap);
a45946ab 3533 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3534 if (iommu->domains[i] == domain) {
3535 spin_lock_irqsave(&iommu->lock, flags);
3536 clear_bit(i, iommu->domain_ids);
3537 iommu->domains[i] = NULL;
3538 spin_unlock_irqrestore(&iommu->lock, flags);
3539 break;
3540 }
5e98c4b1
WH		 3541 }
3542 }
3543}
3544
3545static void vm_domain_exit(struct dmar_domain *domain)
3546{
5e98c4b1
WH		 3547 /* Domain 0 is reserved, so dont process it */
3548 if (!domain)
3549 return;
3550
3551 vm_domain_remove_all_dev_info(domain);
3552 /* destroy iovas */
3553 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3554
3555 /* clear ptes */
595badf5 3556 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3557
3558 /* free page tables */
d794dc9b 3559 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3560
3561 iommu_free_vm_domain(domain);
3562 free_domain_mem(domain);
3563}
3564
5d450806 3565static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3566{
5d450806 3567 struct dmar_domain *dmar_domain;
38717946 3568
5d450806
JR		 3569 dmar_domain = iommu_alloc_vm_domain();
3570 if (!dmar_domain) {
38717946 3571 printk(KERN_ERR
5d450806
JR		 3572 "intel_iommu_domain_init: dmar_domain == NULL\n");
3573 return -ENOMEM;
38717946 3574 }
2c2e2c38 3575 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3576 printk(KERN_ERR
5d450806
JR		 3577 "intel_iommu_domain_init() failed\n");
3578 vm_domain_exit(dmar_domain);
3579 return -ENOMEM;
38717946 3580 }
5d450806 3581 domain->priv = dmar_domain;
faa3d6f5 3582
5d450806 3583 return 0;
38717946 3584}
38717946 3585
5d450806 3586static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3587{
5d450806
JR		 3588 struct dmar_domain *dmar_domain = domain->priv;
3589
3590 domain->priv = NULL;
3591 vm_domain_exit(dmar_domain);
38717946 3592}
38717946 3593
4c5478c9
JR		 3594static int intel_iommu_attach_device(struct iommu_domain *domain,
3595 struct device *dev)
38717946 3596{
4c5478c9
JR		 3597 struct dmar_domain *dmar_domain = domain->priv;
3598 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3599 struct intel_iommu *iommu;
3600 int addr_width;
faa3d6f5
WH		 3601
3602 /* normally pdev is not mapped */
3603 if (unlikely(domain_context_mapped(pdev))) {
3604 struct dmar_domain *old_domain;
3605
3606 old_domain = find_domain(pdev);
3607 if (old_domain) {
2c2e2c38
FY		 3608 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3609 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3610 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3611 else
3612 domain_remove_dev_info(old_domain);
3613 }
3614 }
3615
276dbf99
DW		 3616 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3617 pdev->devfn);
fe40f1e0
WH		 3618 if (!iommu)
3619 return -ENODEV;
3620
3621 /* check if this iommu agaw is sufficient for max mapped address */
3622 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 3623 if (addr_width > cap_mgaw(iommu->cap))
3624 addr_width = cap_mgaw(iommu->cap);
3625
3626 if (dmar_domain->max_addr > (1LL << addr_width)) {
3627 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3628 "sufficient for the mapped address (%llx)\n",
a99c47a2 3629 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 3630 return -EFAULT;
3631 }
a99c47a2
TL		 3632 dmar_domain->gaw = addr_width;
3633
3634 /*
3635 * Knock out extra levels of page tables if necessary
3636 */
3637 while (iommu->agaw < dmar_domain->agaw) {
3638 struct dma_pte *pte;
3639
3640 pte = dmar_domain->pgd;
3641 if (dma_pte_present(pte)) {
25cbff16
SY		 3642 dmar_domain->pgd = (struct dma_pte *)
3643 phys_to_virt(dma_pte_addr(pte));
7a661013 3644 free_pgtable_page(pte);
a99c47a2
TL		 3645 }
3646 dmar_domain->agaw--;
3647 }
fe40f1e0 3648
5fe60f4e 3649 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 3650}
38717946 3651
4c5478c9
JR		 3652static void intel_iommu_detach_device(struct iommu_domain *domain,
3653 struct device *dev)
38717946 3654{
4c5478c9
JR		 3655 struct dmar_domain *dmar_domain = domain->priv;
3656 struct pci_dev *pdev = to_pci_dev(dev);
3657
2c2e2c38 3658 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3659}
c7151a8d 3660
b146a1c9
JR		 3661static int intel_iommu_map(struct iommu_domain *domain,
3662 unsigned long iova, phys_addr_t hpa,
3663 int gfp_order, int iommu_prot)
faa3d6f5 3664{
dde57a21 3665 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 3666 u64 max_addr;
dde57a21 3667 int prot = 0;
b146a1c9 3668 size_t size;
faa3d6f5 3669 int ret;
fe40f1e0 3670
dde57a21
JR		 3671 if (iommu_prot & IOMMU_READ)
3672 prot |= DMA_PTE_READ;
3673 if (iommu_prot & IOMMU_WRITE)
3674 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3675 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3676 prot |= DMA_PTE_SNP;
dde57a21 3677
b146a1c9 3678 size = PAGE_SIZE << gfp_order;
163cc52c 3679 max_addr = iova + size;
dde57a21 3680 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3681 u64 end;
3682
3683 /* check if minimum agaw is sufficient for mapped address */
8954da1f 3684 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 3685 if (end < max_addr) {
8954da1f 3686 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3687 "sufficient for the mapped address (%llx)\n",
8954da1f 3688 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 3689 return -EFAULT;
3690 }
dde57a21 3691 dmar_domain->max_addr = max_addr;
fe40f1e0 3692 }
ad051221
DW		 3693 /* Round up size to next multiple of PAGE_SIZE, if it and
3694 the low bits of hpa would take us onto the next page */
88cb6a74 3695 size = aligned_nrpages(hpa, size);
ad051221
DW		 3696 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3697 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 3698 return ret;
38717946 3699}
38717946 3700
b146a1c9
JR		 3701static int intel_iommu_unmap(struct iommu_domain *domain,
3702 unsigned long iova, int gfp_order)
38717946 3703{
dde57a21 3704 struct dmar_domain *dmar_domain = domain->priv;
b146a1c9 3705 size_t size = PAGE_SIZE << gfp_order;
4b99d352 3706
163cc52c
DW		 3707 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3708 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 3709
163cc52c
DW		 3710 if (dmar_domain->max_addr == iova + size)
3711 dmar_domain->max_addr = iova;
b146a1c9
JR		 3712
3713 return gfp_order;
38717946 3714}
38717946 3715
d14d6577
JR		 3716static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3717 unsigned long iova)
38717946 3718{
d14d6577 3719 struct dmar_domain *dmar_domain = domain->priv;
38717946 3720 struct dma_pte *pte;
faa3d6f5 3721 u64 phys = 0;
38717946 3722
b026fd28 3723 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
38717946 3724 if (pte)
faa3d6f5 3725 phys = dma_pte_addr(pte);
38717946 3726
faa3d6f5 3727 return phys;
38717946 3728}
a8bcbb0d 3729
dbb9fd86
SY		 3730static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3731 unsigned long cap)
3732{
3733 struct dmar_domain *dmar_domain = domain->priv;
3734
3735 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3736 return dmar_domain->iommu_snooping;
323f99cb
TL		 3737 if (cap == IOMMU_CAP_INTR_REMAP)
3738 return intr_remapping_enabled;
dbb9fd86
SY		 3739
3740 return 0;
3741}
3742
a8bcbb0d
JR		 3743static struct iommu_ops intel_iommu_ops = {
3744 .domain_init = intel_iommu_domain_init,
3745 .domain_destroy = intel_iommu_domain_destroy,
3746 .attach_dev = intel_iommu_attach_device,
3747 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 3748 .map = intel_iommu_map,
3749 .unmap = intel_iommu_unmap,
a8bcbb0d 3750 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3751 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3752};
9af88143
DW		 3753
3754static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3755{
3756 /*
3757 * Mobile 4 Series Chipset neglects to set RWBF capability,
3758 * but needs it:
3759 */
3760 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3761 rwbf_quirk = 1;
2d9e667e
DW		 3762
3763 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3764 if (dev->revision == 0x07) {
3765 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3766 dmar_map_gfx = 0;
3767 }
9af88143
DW		 3768}
3769
3770DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
e0fc7e0b 3771
eecfd57f
AJ		 3772#define GGC 0x52
3773#define GGC_MEMORY_SIZE_MASK (0xf << 8)
3774#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3775#define GGC_MEMORY_SIZE_1M (0x1 << 8)
3776#define GGC_MEMORY_SIZE_2M (0x3 << 8)
3777#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3778#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3779#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3780#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3781
9eecabcb
DW		 3782static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3783{
3784 unsigned short ggc;
3785
eecfd57f 3786 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 3787 return;
3788
eecfd57f 3789 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 3790 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3791 dmar_map_gfx = 0;
3792 }
3793}
3794DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3795DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3796DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3797DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3798
e0fc7e0b
DW		 3799/* On Tylersburg chipsets, some BIOSes have been known to enable the
3800 ISOCH DMAR unit for the Azalia sound device, but not give it any
3801 TLB entries, which causes it to deadlock. Check for that. We do
3802 this in a function called from init_dmars(), instead of in a PCI
3803 quirk, because we don't want to print the obnoxious "BIOS broken"
3804 message if VT-d is actually disabled.
3805*/
3806static void __init check_tylersburg_isoch(void)
3807{
3808 struct pci_dev *pdev;
3809 uint32_t vtisochctrl;
3810
3811 /* If there's no Azalia in the system anyway, forget it. */
3812 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3813 if (!pdev)
3814 return;
3815 pci_dev_put(pdev);
3816
3817 /* System Management Registers. Might be hidden, in which case
3818 we can't do the sanity check. But that's OK, because the
3819 known-broken BIOSes _don't_ actually hide it, so far. */
3820 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3821 if (!pdev)
3822 return;
3823
3824 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3825 pci_dev_put(pdev);
3826 return;
3827 }
3828
3829 pci_dev_put(pdev);
3830
3831 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3832 if (vtisochctrl & 1)
3833 return;
3834
3835 /* Drop all bits other than the number of TLB entries */
3836 vtisochctrl &= 0x1c;
3837
3838 /* If we have the recommended number of TLB entries (16), fine. */
3839 if (vtisochctrl == 0x10)
3840 return;
3841
3842 /* Zero TLB entries? You get to ride the short bus to school. */
3843 if (!vtisochctrl) {
3844 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3845 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3846 dmi_get_system_info(DMI_BIOS_VENDOR),
3847 dmi_get_system_info(DMI_BIOS_VERSION),
3848 dmi_get_system_info(DMI_PRODUCT_VERSION));
3849 iommu_identity_mapping |= IDENTMAP_AZALIA;
3850 return;
3851 }
3852
3853 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3854 vtisochctrl);
3855}
Linux 6.x block layer and io_uring tree(s)