projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu: Add sysfs support for IOMMUs
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 18 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
5e0d2a6f 22#include <linux/debugfs.h>
54485c30 23#include <linux/export.h>
ba395927
KA		 24#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
ba395927
KA		 27#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
75f05569 32#include <linux/memory.h>
5e0d2a6f 33#include <linux/timer.h>
38717946 34#include <linux/iova.h>
5d450806 35#include <linux/iommu.h>
38717946 36#include <linux/intel-iommu.h>
134fac3f 37#include <linux/syscore_ops.h>
69575d38 38#include <linux/tboot.h>
adb2fe02 39#include <linux/dmi.h>
5cdede24 40#include <linux/pci-ats.h>
0ee332c1 41#include <linux/memblock.h>
36746436 42#include <linux/dma-contiguous.h>
8a8f422d 43#include <asm/irq_remapping.h>
ba395927 44#include <asm/cacheflush.h>
46a7fa27 45#include <asm/iommu.h>
ba395927 46
078e1ee2
JR		 47#include "irq_remapping.h"
48
5b6985ce
FY		 49#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
ba395927
KA		 52#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 54#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 55
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
4ed0d3e6 62#define MAX_AGAW_WIDTH 64
5c645b35 63#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 64
2ebe3151
DW		 65#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 73
f27be03b 74#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 75#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 76#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 77
df08cdc7
AM		 78/* page table handling */
79#define LEVEL_STRIDE (9)
80#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
6d1c56a9
OBC		 82/*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
df08cdc7
AM		 100static inline int agaw_to_level(int agaw)
101{
102 return agaw + 2;
103}
104
105static inline int agaw_to_width(int agaw)
106{
5c645b35 107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 108}
109
110static inline int width_to_agaw(int width)
111{
5c645b35 112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 113}
114
115static inline unsigned int level_to_offset_bits(int level)
116{
117 return (level - 1) * LEVEL_STRIDE;
118}
119
120static inline int pfn_level_offset(unsigned long pfn, int level)
121{
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123}
124
125static inline unsigned long level_mask(int level)
126{
127 return -1UL << level_to_offset_bits(level);
128}
129
130static inline unsigned long level_size(int level)
131{
132 return 1UL << level_to_offset_bits(level);
133}
134
135static inline unsigned long align_to_level(unsigned long pfn, int level)
136{
137 return (pfn + level_size(level) - 1) & level_mask(level);
138}
fd18de50 139
6dd9a7c7
YS		 140static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141{
5c645b35 142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 143}
144
dd4e8319
DW		 145/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148{
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150}
151
152static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153{
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155}
156static inline unsigned long page_to_dma_pfn(struct page *pg)
157{
158 return mm_to_dma_pfn(page_to_pfn(pg));
159}
160static inline unsigned long virt_to_dma_pfn(void *p)
161{
162 return page_to_dma_pfn(virt_to_page(p));
163}
164
d9630fe9
WH		 165/* global iommu list, set NULL for ignored DMAR units */
166static struct intel_iommu **g_iommus;
167
e0fc7e0b 168static void __init check_tylersburg_isoch(void);
9af88143
DW		 169static int rwbf_quirk;
170
b779260b
JC		 171/*
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175static int force_on = 0;
176
46b08e1a
MM		 177/*
178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183struct root_entry {
184 u64 val;
185 u64 rsvd1;
186};
187#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188static inline bool root_present(struct root_entry *root)
189{
190 return (root->val & 1);
191}
192static inline void set_root_present(struct root_entry *root)
193{
194 root->val |= 1;
195}
196static inline void set_root_value(struct root_entry *root, unsigned long value)
197{
198 root->val |= value & VTD_PAGE_MASK;
199}
200
201static inline struct context_entry *
202get_context_addr_from_root(struct root_entry *root)
203{
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
207 NULL);
208}
209
7a8fc25e
MM		 210/*
211 * low 64 bits:
212 * 0: present
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
216 * high 64 bits:
217 * 0-2: address width
218 * 3-6: aval
219 * 8-23: domain id
220 */
221struct context_entry {
222 u64 lo;
223 u64 hi;
224};
c07e7d21
MM		 225
226static inline bool context_present(struct context_entry *context)
227{
228 return (context->lo & 1);
229}
230static inline void context_set_present(struct context_entry *context)
231{
232 context->lo |= 1;
233}
234
235static inline void context_set_fault_enable(struct context_entry *context)
236{
237 context->lo &= (((u64)-1) << 2) | 1;
238}
239
c07e7d21
MM		 240static inline void context_set_translation_type(struct context_entry *context,
241 unsigned long value)
242{
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
245}
246
247static inline void context_set_address_root(struct context_entry *context,
248 unsigned long value)
249{
250 context->lo |= value & VTD_PAGE_MASK;
251}
252
253static inline void context_set_address_width(struct context_entry *context,
254 unsigned long value)
255{
256 context->hi |= value & 7;
257}
258
259static inline void context_set_domain_id(struct context_entry *context,
260 unsigned long value)
261{
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
263}
264
265static inline void context_clear_entry(struct context_entry *context)
266{
267 context->lo = 0;
268 context->hi = 0;
269}
7a8fc25e 270
622ba12a
MM		 271/*
272 * 0: readable
273 * 1: writable
274 * 2-6: reserved
275 * 7: super page
9cf06697
SY		 276 * 8-10: available
277 * 11: snoop behavior
622ba12a
MM		 278 * 12-63: Host physcial address
279 */
280struct dma_pte {
281 u64 val;
282};
622ba12a 283
19c239ce
MM		 284static inline void dma_clear_pte(struct dma_pte *pte)
285{
286 pte->val = 0;
287}
288
19c239ce
MM		 289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
c85994e4
DW		 291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
1a8bd481 295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 296#endif
19c239ce
MM		 297}
298
19c239ce
MM		 299static inline bool dma_pte_present(struct dma_pte *pte)
300{
301 return (pte->val & 3) != 0;
302}
622ba12a 303
4399c8bf
AK		 304static inline bool dma_pte_superpage(struct dma_pte *pte)
305{
306 return (pte->val & (1 << 7));
307}
308
75e6bf96
DW		 309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
2c2e2c38
FY		 314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
19943b0e
DW		 320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
2c2e2c38 322
3b5410e7 323/* devices under the same p2p bridge are owned in one domain */
cdc7b837 324#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 325
1ce28feb
WH		 326/* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
328 */
329#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
330
2c2e2c38
FY		 331/* si_domain contains mulitple devices */
332#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
333
1b198bb0
MT		 334/* define the limit of IOMMUs supported in each domain */
335#ifdef CONFIG_X86
336# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
337#else
338# define IOMMU_UNITS_SUPPORTED 64
339#endif
340
99126f7c
MM		 341struct dmar_domain {
342 int id; /* domain id */
4c923d47 343 int nid; /* node id */
1b198bb0
MT		 344 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
345 /* bitmap of iommus this domain uses*/
99126f7c
MM		 346
347 struct list_head devices; /* all devices' list */
348 struct iova_domain iovad; /* iova's that belong to this domain */
349
350 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 351 int gaw; /* max guest address width */
352
353 /* adjusted guest address width, 0 is level 2 30-bit */
354 int agaw;
355
3b5410e7 356 int flags; /* flags to find out type of domain */
8e604097
WH		 357
358 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 359 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 360 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 361 int iommu_superpage;/* Level of superpages supported:
362 0 == 4KiB (no superpages), 1 == 2MiB,
363 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 364 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 365 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 366};
367
a647dacb
MM		 368/* PCI domain-device relationship */
369struct device_domain_info {
370 struct list_head link; /* link to domain siblings */
371 struct list_head global; /* link to global list */
276dbf99 372 u8 bus; /* PCI bus number */
a647dacb 373 u8 devfn; /* PCI devfn number */
0bcb3e28 374 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 375 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 376 struct dmar_domain *domain; /* pointer to domain */
377};
378
b94e4117
JL		 379struct dmar_rmrr_unit {
380 struct list_head list; /* list of rmrr units */
381 struct acpi_dmar_header *hdr; /* ACPI header */
382 u64 base_address; /* reserved base address*/
383 u64 end_address; /* reserved end address */
832bd858 384 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 385 int devices_cnt; /* target device count */
386};
387
388struct dmar_atsr_unit {
389 struct list_head list; /* list of ATSR units */
390 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 391 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 392 int devices_cnt; /* target device count */
393 u8 include_all:1; /* include all ports */
394};
395
396static LIST_HEAD(dmar_atsr_units);
397static LIST_HEAD(dmar_rmrr_units);
398
399#define for_each_rmrr_units(rmrr) \
400 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
401
5e0d2a6f 402static void flush_unmaps_timeout(unsigned long data);
403
b707cb02 404static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 405
80b20dd8 406#define HIGH_WATER_MARK 250
407struct deferred_flush_tables {
408 int next;
409 struct iova *iova[HIGH_WATER_MARK];
410 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 411 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 412};
413
414static struct deferred_flush_tables *deferred_flush;
415
5e0d2a6f 416/* bitmap for indexing intel_iommus */
5e0d2a6f 417static int g_num_of_iommus;
418
419static DEFINE_SPINLOCK(async_umap_flush_lock);
420static LIST_HEAD(unmaps_to_do);
421
422static int timer_on;
423static long list_size;
5e0d2a6f 424
92d03cc8 425static void domain_exit(struct dmar_domain *domain);
ba395927 426static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 427static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 428 struct device *dev);
92d03cc8 429static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 430 struct device *dev);
ba395927 431
d3f13810 432#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 433int dmar_disabled = 0;
434#else
435int dmar_disabled = 1;
d3f13810 436#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 437
8bc1f85c
ED		 438int intel_iommu_enabled = 0;
439EXPORT_SYMBOL_GPL(intel_iommu_enabled);
440
2d9e667e 441static int dmar_map_gfx = 1;
7d3b03ce 442static int dmar_forcedac;
5e0d2a6f 443static int intel_iommu_strict;
6dd9a7c7 444static int intel_iommu_superpage = 1;
ba395927 445
c0771df8
DW		 446int intel_iommu_gfx_mapped;
447EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
448
ba395927
KA		 449#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
450static DEFINE_SPINLOCK(device_domain_lock);
451static LIST_HEAD(device_domain_list);
452
a8bcbb0d
JR		 453static struct iommu_ops intel_iommu_ops;
454
ba395927
KA		 455static int __init intel_iommu_setup(char *str)
456{
457 if (!str)
458 return -EINVAL;
459 while (*str) {
0cd5c3c8
KM		 460 if (!strncmp(str, "on", 2)) {
461 dmar_disabled = 0;
462 printk(KERN_INFO "Intel-IOMMU: enabled\n");
463 } else if (!strncmp(str, "off", 3)) {
ba395927 464 dmar_disabled = 1;
0cd5c3c8 465 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 466 } else if (!strncmp(str, "igfx_off", 8)) {
467 dmar_map_gfx = 0;
468 printk(KERN_INFO
469 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 470 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 471 printk(KERN_INFO
7d3b03ce
KA		 472 "Intel-IOMMU: Forcing DAC for PCI devices\n");
473 dmar_forcedac = 1;
5e0d2a6f 474 } else if (!strncmp(str, "strict", 6)) {
475 printk(KERN_INFO
476 "Intel-IOMMU: disable batched IOTLB flush\n");
477 intel_iommu_strict = 1;
6dd9a7c7
YS		 478 } else if (!strncmp(str, "sp_off", 6)) {
479 printk(KERN_INFO
480 "Intel-IOMMU: disable supported super page\n");
481 intel_iommu_superpage = 0;
ba395927
KA		 482 }
483
484 str += strcspn(str, ",");
485 while (*str == ',')
486 str++;
487 }
488 return 0;
489}
490__setup("intel_iommu=", intel_iommu_setup);
491
492static struct kmem_cache *iommu_domain_cache;
493static struct kmem_cache *iommu_devinfo_cache;
494static struct kmem_cache *iommu_iova_cache;
495
4c923d47 496static inline void *alloc_pgtable_page(int node)
eb3fa7cb 497{
4c923d47
SS		 498 struct page *page;
499 void *vaddr = NULL;
eb3fa7cb 500
4c923d47
SS		 501 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
502 if (page)
503 vaddr = page_address(page);
eb3fa7cb 504 return vaddr;
ba395927
KA		 505}
506
507static inline void free_pgtable_page(void *vaddr)
508{
509 free_page((unsigned long)vaddr);
510}
511
512static inline void *alloc_domain_mem(void)
513{
354bb65e 514 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 515}
516
38717946 517static void free_domain_mem(void *vaddr)
ba395927
KA		 518{
519 kmem_cache_free(iommu_domain_cache, vaddr);
520}
521
522static inline void * alloc_devinfo_mem(void)
523{
354bb65e 524 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 525}
526
527static inline void free_devinfo_mem(void *vaddr)
528{
529 kmem_cache_free(iommu_devinfo_cache, vaddr);
530}
531
532struct iova *alloc_iova_mem(void)
533{
354bb65e 534 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 535}
536
537void free_iova_mem(struct iova *iova)
538{
539 kmem_cache_free(iommu_iova_cache, iova);
540}
541
1b573683 542
4ed0d3e6 543static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 544{
545 unsigned long sagaw;
546 int agaw = -1;
547
548 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 549 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 550 agaw >= 0; agaw--) {
551 if (test_bit(agaw, &sagaw))
552 break;
553 }
554
555 return agaw;
556}
557
4ed0d3e6
FY		 558/*
559 * Calculate max SAGAW for each iommu.
560 */
561int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
562{
563 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
564}
565
566/*
567 * calculate agaw for each iommu.
568 * "SAGAW" may be different across iommus, use a default agaw, and
569 * get a supported less agaw for iommus that don't support the default agaw.
570 */
571int iommu_calculate_agaw(struct intel_iommu *iommu)
572{
573 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
574}
575
2c2e2c38 576/* This functionin only returns single iommu in a domain */
8c11e798
WH		 577static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
578{
579 int iommu_id;
580
2c2e2c38 581 /* si_domain and vm domain should not get here. */
1ce28feb 582 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 583 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 584
1b198bb0 585 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 586 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
587 return NULL;
588
589 return g_iommus[iommu_id];
590}
591
8e604097
WH		 592static void domain_update_iommu_coherency(struct dmar_domain *domain)
593{
d0501960
DW		 594 struct dmar_drhd_unit *drhd;
595 struct intel_iommu *iommu;
596 int i, found = 0;
2e12bc29 597
d0501960 598 domain->iommu_coherency = 1;
8e604097 599
1b198bb0 600 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
d0501960 601 found = 1;
8e604097
WH		 602 if (!ecap_coherent(g_iommus[i]->ecap)) {
603 domain->iommu_coherency = 0;
604 break;
605 }
8e604097 606 }
d0501960
DW		 607 if (found)
608 return;
609
610 /* No hardware attached; use lowest common denominator */
611 rcu_read_lock();
612 for_each_active_iommu(iommu, drhd) {
613 if (!ecap_coherent(iommu->ecap)) {
614 domain->iommu_coherency = 0;
615 break;
616 }
617 }
618 rcu_read_unlock();
8e604097
WH		 619}
620
58c610bd
SY		 621static void domain_update_iommu_snooping(struct dmar_domain *domain)
622{
623 int i;
624
625 domain->iommu_snooping = 1;
626
1b198bb0 627 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 628 if (!ecap_sc_support(g_iommus[i]->ecap)) {
629 domain->iommu_snooping = 0;
630 break;
631 }
58c610bd
SY		 632 }
633}
634
6dd9a7c7
YS		 635static void domain_update_iommu_superpage(struct dmar_domain *domain)
636{
8140a95d
AK		 637 struct dmar_drhd_unit *drhd;
638 struct intel_iommu *iommu = NULL;
639 int mask = 0xf;
6dd9a7c7
YS		 640
641 if (!intel_iommu_superpage) {
642 domain->iommu_superpage = 0;
643 return;
644 }
645
8140a95d 646 /* set iommu_superpage to the smallest common denominator */
0e242612 647 rcu_read_lock();
8140a95d
AK		 648 for_each_active_iommu(iommu, drhd) {
649 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 650 if (!mask) {
651 break;
652 }
653 }
0e242612
JL		 654 rcu_read_unlock();
655
6dd9a7c7
YS		 656 domain->iommu_superpage = fls(mask);
657}
658
58c610bd
SY		 659/* Some capabilities may be different across iommus */
660static void domain_update_iommu_cap(struct dmar_domain *domain)
661{
662 domain_update_iommu_coherency(domain);
663 domain_update_iommu_snooping(domain);
6dd9a7c7 664 domain_update_iommu_superpage(domain);
58c610bd
SY		 665}
666
156baca8 667static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 668{
669 struct dmar_drhd_unit *drhd = NULL;
b683b230 670 struct intel_iommu *iommu;
156baca8
DW		 671 struct device *tmp;
672 struct pci_dev *ptmp, *pdev = NULL;
673 u16 segment;
c7151a8d
WH		 674 int i;
675
156baca8
DW		 676 if (dev_is_pci(dev)) {
677 pdev = to_pci_dev(dev);
678 segment = pci_domain_nr(pdev->bus);
679 } else if (ACPI_COMPANION(dev))
680 dev = &ACPI_COMPANION(dev)->dev;
681
0e242612 682 rcu_read_lock();
b683b230 683 for_each_active_iommu(iommu, drhd) {
156baca8 684 if (pdev && segment != drhd->segment)
276dbf99 685 continue;
c7151a8d 686
b683b230 687 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 688 drhd->devices_cnt, i, tmp) {
689 if (tmp == dev) {
690 *bus = drhd->devices[i].bus;
691 *devfn = drhd->devices[i].devfn;
b683b230 692 goto out;
156baca8
DW		 693 }
694
695 if (!pdev || !dev_is_pci(tmp))
696 continue;
697
698 ptmp = to_pci_dev(tmp);
699 if (ptmp->subordinate &&
700 ptmp->subordinate->number <= pdev->bus->number &&
701 ptmp->subordinate->busn_res.end >= pdev->bus->number)
702 goto got_pdev;
924b6231 703 }
c7151a8d 704
156baca8
DW		 705 if (pdev && drhd->include_all) {
706 got_pdev:
707 *bus = pdev->bus->number;
708 *devfn = pdev->devfn;
b683b230 709 goto out;
156baca8 710 }
c7151a8d 711 }
b683b230 712 iommu = NULL;
156baca8 713 out:
0e242612 714 rcu_read_unlock();
c7151a8d 715
b683b230 716 return iommu;
c7151a8d
WH		 717}
718
5331fe6f
WH		 719static void domain_flush_cache(struct dmar_domain *domain,
720 void *addr, int size)
721{
722 if (!domain->iommu_coherency)
723 clflush_cache_range(addr, size);
724}
725
ba395927
KA		 726/* Gets context entry for a given bus and devfn */
727static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
728 u8 bus, u8 devfn)
729{
730 struct root_entry *root;
731 struct context_entry *context;
732 unsigned long phy_addr;
733 unsigned long flags;
734
735 spin_lock_irqsave(&iommu->lock, flags);
736 root = &iommu->root_entry[bus];
737 context = get_context_addr_from_root(root);
738 if (!context) {
4c923d47
SS		 739 context = (struct context_entry *)
740 alloc_pgtable_page(iommu->node);
ba395927
KA		 741 if (!context) {
742 spin_unlock_irqrestore(&iommu->lock, flags);
743 return NULL;
744 }
5b6985ce 745 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 746 phy_addr = virt_to_phys((void *)context);
747 set_root_value(root, phy_addr);
748 set_root_present(root);
749 __iommu_flush_cache(iommu, root, sizeof(*root));
750 }
751 spin_unlock_irqrestore(&iommu->lock, flags);
752 return &context[devfn];
753}
754
755static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
756{
757 struct root_entry *root;
758 struct context_entry *context;
759 int ret;
760 unsigned long flags;
761
762 spin_lock_irqsave(&iommu->lock, flags);
763 root = &iommu->root_entry[bus];
764 context = get_context_addr_from_root(root);
765 if (!context) {
766 ret = 0;
767 goto out;
768 }
c07e7d21 769 ret = context_present(&context[devfn]);
ba395927
KA		 770out:
771 spin_unlock_irqrestore(&iommu->lock, flags);
772 return ret;
773}
774
775static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
776{
777 struct root_entry *root;
778 struct context_entry *context;
779 unsigned long flags;
780
781 spin_lock_irqsave(&iommu->lock, flags);
782 root = &iommu->root_entry[bus];
783 context = get_context_addr_from_root(root);
784 if (context) {
c07e7d21 785 context_clear_entry(&context[devfn]);
ba395927
KA		 786 __iommu_flush_cache(iommu, &context[devfn], \
787 sizeof(*context));
788 }
789 spin_unlock_irqrestore(&iommu->lock, flags);
790}
791
792static void free_context_table(struct intel_iommu *iommu)
793{
794 struct root_entry *root;
795 int i;
796 unsigned long flags;
797 struct context_entry *context;
798
799 spin_lock_irqsave(&iommu->lock, flags);
800 if (!iommu->root_entry) {
801 goto out;
802 }
803 for (i = 0; i < ROOT_ENTRY_NR; i++) {
804 root = &iommu->root_entry[i];
805 context = get_context_addr_from_root(root);
806 if (context)
807 free_pgtable_page(context);
808 }
809 free_pgtable_page(iommu->root_entry);
810 iommu->root_entry = NULL;
811out:
812 spin_unlock_irqrestore(&iommu->lock, flags);
813}
814
b026fd28 815static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 816 unsigned long pfn, int *target_level)
ba395927 817{
b026fd28 818 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 819 struct dma_pte *parent, *pte = NULL;
820 int level = agaw_to_level(domain->agaw);
4399c8bf 821 int offset;
ba395927
KA		 822
823 BUG_ON(!domain->pgd);
f9423606
JS		 824
825 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
826 /* Address beyond IOMMU's addressing capabilities. */
827 return NULL;
828
ba395927
KA		 829 parent = domain->pgd;
830
5cf0a76f 831 while (1) {
ba395927
KA		 832 void *tmp_page;
833
b026fd28 834 offset = pfn_level_offset(pfn, level);
ba395927 835 pte = &parent[offset];
5cf0a76f 836 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 837 break;
5cf0a76f 838 if (level == *target_level)
ba395927
KA		 839 break;
840
19c239ce 841 if (!dma_pte_present(pte)) {
c85994e4
DW		 842 uint64_t pteval;
843
4c923d47 844 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 845
206a73c1 846 if (!tmp_page)
ba395927 847 return NULL;
206a73c1 848
c85994e4 849 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 850 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 851 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
852 /* Someone else set it while we were thinking; use theirs. */
853 free_pgtable_page(tmp_page);
854 } else {
855 dma_pte_addr(pte);
856 domain_flush_cache(domain, pte, sizeof(*pte));
857 }
ba395927 858 }
5cf0a76f
DW		 859 if (level == 1)
860 break;
861
19c239ce 862 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 863 level--;
864 }
865
5cf0a76f
DW		 866 if (!*target_level)
867 *target_level = level;
868
ba395927
KA		 869 return pte;
870}
871
6dd9a7c7 872
ba395927 873/* return address's pte at specific level */
90dcfb5e
DW		 874static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
875 unsigned long pfn,
6dd9a7c7 876 int level, int *large_page)
ba395927
KA		 877{
878 struct dma_pte *parent, *pte = NULL;
879 int total = agaw_to_level(domain->agaw);
880 int offset;
881
882 parent = domain->pgd;
883 while (level <= total) {
90dcfb5e 884 offset = pfn_level_offset(pfn, total);
ba395927
KA		 885 pte = &parent[offset];
886 if (level == total)
887 return pte;
888
6dd9a7c7
YS		 889 if (!dma_pte_present(pte)) {
890 *large_page = total;
ba395927 891 break;
6dd9a7c7
YS		 892 }
893
894 if (pte->val & DMA_PTE_LARGE_PAGE) {
895 *large_page = total;
896 return pte;
897 }
898
19c239ce 899 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 900 total--;
901 }
902 return NULL;
903}
904
ba395927 905/* clear last level pte, a tlb flush should be followed */
5cf0a76f 906static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 907 unsigned long start_pfn,
908 unsigned long last_pfn)
ba395927 909{
04b18e65 910 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 911 unsigned int large_page = 1;
310a5ab9 912 struct dma_pte *first_pte, *pte;
66eae846 913
04b18e65 914 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 915 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 916 BUG_ON(start_pfn > last_pfn);
ba395927 917
04b18e65 918 /* we don't need lock here; nobody else touches the iova range */
59c36286 919 do {
6dd9a7c7
YS		 920 large_page = 1;
921 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 922 if (!pte) {
6dd9a7c7 923 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 924 continue;
925 }
6dd9a7c7 926 do {
310a5ab9 927 dma_clear_pte(pte);
6dd9a7c7 928 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 929 pte++;
75e6bf96
DW		 930 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
931
310a5ab9
DW		 932 domain_flush_cache(domain, first_pte,
933 (void *)pte - (void *)first_pte);
59c36286
DW		 934
935 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 936}
937
3269ee0b
AW		 938static void dma_pte_free_level(struct dmar_domain *domain, int level,
939 struct dma_pte *pte, unsigned long pfn,
940 unsigned long start_pfn, unsigned long last_pfn)
941{
942 pfn = max(start_pfn, pfn);
943 pte = &pte[pfn_level_offset(pfn, level)];
944
945 do {
946 unsigned long level_pfn;
947 struct dma_pte *level_pte;
948
949 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
950 goto next;
951
952 level_pfn = pfn & level_mask(level - 1);
953 level_pte = phys_to_virt(dma_pte_addr(pte));
954
955 if (level > 2)
956 dma_pte_free_level(domain, level - 1, level_pte,
957 level_pfn, start_pfn, last_pfn);
958
959 /* If range covers entire pagetable, free it */
960 if (!(start_pfn > level_pfn ||
08336fd2 961 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 962 dma_clear_pte(pte);
963 domain_flush_cache(domain, pte, sizeof(*pte));
964 free_pgtable_page(level_pte);
965 }
966next:
967 pfn += level_size(level);
968 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
969}
970
ba395927
KA		 971/* free page table pages. last level pte should already be cleared */
972static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 973 unsigned long start_pfn,
974 unsigned long last_pfn)
ba395927 975{
6660c63a 976 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 977
6660c63a
DW		 978 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
979 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 980 BUG_ON(start_pfn > last_pfn);
ba395927 981
f3a0a52f 982 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 983 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
984 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 985
ba395927 986 /* free pgd */
d794dc9b 987 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 988 free_pgtable_page(domain->pgd);
989 domain->pgd = NULL;
990 }
991}
992
ea8ea460
DW		 993/* When a page at a given level is being unlinked from its parent, we don't
994 need to *modify* it at all. All we need to do is make a list of all the
995 pages which can be freed just as soon as we've flushed the IOTLB and we
996 know the hardware page-walk will no longer touch them.
997 The 'pte' argument is the *parent* PTE, pointing to the page that is to
998 be freed. */
999static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1000 int level, struct dma_pte *pte,
1001 struct page *freelist)
1002{
1003 struct page *pg;
1004
1005 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1006 pg->freelist = freelist;
1007 freelist = pg;
1008
1009 if (level == 1)
1010 return freelist;
1011
adeb2590
JL		 1012 pte = page_address(pg);
1013 do {
ea8ea460
DW		 1014 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1015 freelist = dma_pte_list_pagetables(domain, level - 1,
1016 pte, freelist);
adeb2590
JL		 1017 pte++;
1018 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1019
1020 return freelist;
1021}
1022
1023static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1024 struct dma_pte *pte, unsigned long pfn,
1025 unsigned long start_pfn,
1026 unsigned long last_pfn,
1027 struct page *freelist)
1028{
1029 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1030
1031 pfn = max(start_pfn, pfn);
1032 pte = &pte[pfn_level_offset(pfn, level)];
1033
1034 do {
1035 unsigned long level_pfn;
1036
1037 if (!dma_pte_present(pte))
1038 goto next;
1039
1040 level_pfn = pfn & level_mask(level);
1041
1042 /* If range covers entire pagetable, free it */
1043 if (start_pfn <= level_pfn &&
1044 last_pfn >= level_pfn + level_size(level) - 1) {
1045 /* These suborbinate page tables are going away entirely. Don't
1046 bother to clear them; we're just going to *free* them. */
1047 if (level > 1 && !dma_pte_superpage(pte))
1048 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1049
1050 dma_clear_pte(pte);
1051 if (!first_pte)
1052 first_pte = pte;
1053 last_pte = pte;
1054 } else if (level > 1) {
1055 /* Recurse down into a level that isn't *entirely* obsolete */
1056 freelist = dma_pte_clear_level(domain, level - 1,
1057 phys_to_virt(dma_pte_addr(pte)),
1058 level_pfn, start_pfn, last_pfn,
1059 freelist);
1060 }
1061next:
1062 pfn += level_size(level);
1063 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1064
1065 if (first_pte)
1066 domain_flush_cache(domain, first_pte,
1067 (void *)++last_pte - (void *)first_pte);
1068
1069 return freelist;
1070}
1071
1072/* We can't just free the pages because the IOMMU may still be walking
1073 the page tables, and may have cached the intermediate levels. The
1074 pages can only be freed after the IOTLB flush has been done. */
1075struct page *domain_unmap(struct dmar_domain *domain,
1076 unsigned long start_pfn,
1077 unsigned long last_pfn)
1078{
1079 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1080 struct page *freelist = NULL;
1081
1082 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1083 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1084 BUG_ON(start_pfn > last_pfn);
1085
1086 /* we don't need lock here; nobody else touches the iova range */
1087 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1088 domain->pgd, 0, start_pfn, last_pfn, NULL);
1089
1090 /* free pgd */
1091 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1092 struct page *pgd_page = virt_to_page(domain->pgd);
1093 pgd_page->freelist = freelist;
1094 freelist = pgd_page;
1095
1096 domain->pgd = NULL;
1097 }
1098
1099 return freelist;
1100}
1101
1102void dma_free_pagelist(struct page *freelist)
1103{
1104 struct page *pg;
1105
1106 while ((pg = freelist)) {
1107 freelist = pg->freelist;
1108 free_pgtable_page(page_address(pg));
1109 }
1110}
1111
ba395927
KA		 1112/* iommu handling */
1113static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1114{
1115 struct root_entry *root;
1116 unsigned long flags;
1117
4c923d47 1118 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 1119 if (!root)
1120 return -ENOMEM;
1121
5b6985ce 1122 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1123
1124 spin_lock_irqsave(&iommu->lock, flags);
1125 iommu->root_entry = root;
1126 spin_unlock_irqrestore(&iommu->lock, flags);
1127
1128 return 0;
1129}
1130
ba395927
KA		 1131static void iommu_set_root_entry(struct intel_iommu *iommu)
1132{
1133 void *addr;
c416daa9 1134 u32 sts;
ba395927
KA		 1135 unsigned long flag;
1136
1137 addr = iommu->root_entry;
1138
1f5b3c3f 1139 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1140 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1141
c416daa9 1142 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1143
1144 /* Make sure hardware complete it */
1145 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1146 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1147
1f5b3c3f 1148 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1149}
1150
1151static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1152{
1153 u32 val;
1154 unsigned long flag;
1155
9af88143 1156 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1157 return;
ba395927 1158
1f5b3c3f 1159 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1160 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1161
1162 /* Make sure hardware complete it */
1163 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1164 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1165
1f5b3c3f 1166 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1167}
1168
1169/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1170static void __iommu_flush_context(struct intel_iommu *iommu,
1171 u16 did, u16 source_id, u8 function_mask,
1172 u64 type)
ba395927
KA		 1173{
1174 u64 val = 0;
1175 unsigned long flag;
1176
ba395927
KA		 1177 switch (type) {
1178 case DMA_CCMD_GLOBAL_INVL:
1179 val = DMA_CCMD_GLOBAL_INVL;
1180 break;
1181 case DMA_CCMD_DOMAIN_INVL:
1182 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1183 break;
1184 case DMA_CCMD_DEVICE_INVL:
1185 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1186 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1187 break;
1188 default:
1189 BUG();
1190 }
1191 val |= DMA_CCMD_ICC;
1192
1f5b3c3f 1193 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1194 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1195
1196 /* Make sure hardware complete it */
1197 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1198 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1199
1f5b3c3f 1200 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1201}
1202
ba395927 1203/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1204static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1205 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1206{
1207 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1208 u64 val = 0, val_iva = 0;
1209 unsigned long flag;
1210
ba395927
KA		 1211 switch (type) {
1212 case DMA_TLB_GLOBAL_FLUSH:
1213 /* global flush doesn't need set IVA_REG */
1214 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1215 break;
1216 case DMA_TLB_DSI_FLUSH:
1217 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1218 break;
1219 case DMA_TLB_PSI_FLUSH:
1220 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1221 /* IH bit is passed in as part of address */
ba395927
KA		 1222 val_iva = size_order | addr;
1223 break;
1224 default:
1225 BUG();
1226 }
1227 /* Note: set drain read/write */
1228#if 0
1229 /*
1230 * This is probably to be super secure.. Looks like we can
1231 * ignore it without any impact.
1232 */
1233 if (cap_read_drain(iommu->cap))
1234 val |= DMA_TLB_READ_DRAIN;
1235#endif
1236 if (cap_write_drain(iommu->cap))
1237 val |= DMA_TLB_WRITE_DRAIN;
1238
1f5b3c3f 1239 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1240 /* Note: Only uses first TLB reg currently */
1241 if (val_iva)
1242 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1243 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1244
1245 /* Make sure hardware complete it */
1246 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1247 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1248
1f5b3c3f 1249 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1250
1251 /* check IOTLB invalidation granularity */
1252 if (DMA_TLB_IAIG(val) == 0)
1253 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1254 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1255 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1256 (unsigned long long)DMA_TLB_IIRG(type),
1257 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1258}
1259
64ae892b
DW		 1260static struct device_domain_info *
1261iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1262 u8 bus, u8 devfn)
93a23a72
YZ		 1263{
1264 int found = 0;
1265 unsigned long flags;
1266 struct device_domain_info *info;
0bcb3e28 1267 struct pci_dev *pdev;
93a23a72
YZ		 1268
1269 if (!ecap_dev_iotlb_support(iommu->ecap))
1270 return NULL;
1271
1272 if (!iommu->qi)
1273 return NULL;
1274
1275 spin_lock_irqsave(&device_domain_lock, flags);
1276 list_for_each_entry(info, &domain->devices, link)
1277 if (info->bus == bus && info->devfn == devfn) {
1278 found = 1;
1279 break;
1280 }
1281 spin_unlock_irqrestore(&device_domain_lock, flags);
1282
0bcb3e28 1283 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1284 return NULL;
1285
0bcb3e28
DW		 1286 pdev = to_pci_dev(info->dev);
1287
1288 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1289 return NULL;
1290
0bcb3e28 1291 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1292 return NULL;
1293
93a23a72
YZ		 1294 return info;
1295}
1296
1297static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1298{
0bcb3e28 1299 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1300 return;
1301
0bcb3e28 1302 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1303}
1304
1305static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1306{
0bcb3e28
DW		 1307 if (!info->dev || !dev_is_pci(info->dev) ||
1308 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1309 return;
1310
0bcb3e28 1311 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1312}
1313
1314static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1315 u64 addr, unsigned mask)
1316{
1317 u16 sid, qdep;
1318 unsigned long flags;
1319 struct device_domain_info *info;
1320
1321 spin_lock_irqsave(&device_domain_lock, flags);
1322 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1323 struct pci_dev *pdev;
1324 if (!info->dev || !dev_is_pci(info->dev))
1325 continue;
1326
1327 pdev = to_pci_dev(info->dev);
1328 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1329 continue;
1330
1331 sid = info->bus << 8 | info->devfn;
0bcb3e28 1332 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1333 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1334 }
1335 spin_unlock_irqrestore(&device_domain_lock, flags);
1336}
1337
1f0ef2aa 1338static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1339 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1340{
9dd2fe89 1341 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1342 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1343
ba395927
KA		 1344 BUG_ON(pages == 0);
1345
ea8ea460
DW		 1346 if (ih)
1347 ih = 1 << 6;
ba395927 1348 /*
9dd2fe89
YZ		 1349 * Fallback to domain selective flush if no PSI support or the size is
1350 * too big.
ba395927
KA		 1351 * PSI requires page size to be 2 ^ x, and the base address is naturally
1352 * aligned to the size
1353 */
9dd2fe89
YZ		 1354 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1355 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1356 DMA_TLB_DSI_FLUSH);
9dd2fe89 1357 else
ea8ea460 1358 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1359 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1360
1361 /*
82653633
NA		 1362 * In caching mode, changes of pages from non-present to present require
1363 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1364 */
82653633 1365 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1366 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1367}
1368
f8bab735 1369static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1370{
1371 u32 pmen;
1372 unsigned long flags;
1373
1f5b3c3f 1374 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1375 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1376 pmen &= ~DMA_PMEN_EPM;
1377 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1378
1379 /* wait for the protected region status bit to clear */
1380 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1381 readl, !(pmen & DMA_PMEN_PRS), pmen);
1382
1f5b3c3f 1383 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1384}
1385
ba395927
KA		 1386static int iommu_enable_translation(struct intel_iommu *iommu)
1387{
1388 u32 sts;
1389 unsigned long flags;
1390
1f5b3c3f 1391 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1392 iommu->gcmd |= DMA_GCMD_TE;
1393 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1394
1395 /* Make sure hardware complete it */
1396 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1397 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1398
1f5b3c3f 1399 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1400 return 0;
1401}
1402
1403static int iommu_disable_translation(struct intel_iommu *iommu)
1404{
1405 u32 sts;
1406 unsigned long flag;
1407
1f5b3c3f 1408 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1409 iommu->gcmd &= ~DMA_GCMD_TE;
1410 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1411
1412 /* Make sure hardware complete it */
1413 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1414 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1415
1f5b3c3f 1416 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1417 return 0;
1418}
1419
3460a6d9 1420
ba395927
KA		 1421static int iommu_init_domains(struct intel_iommu *iommu)
1422{
1423 unsigned long ndomains;
1424 unsigned long nlongs;
1425
1426 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1427 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1428 iommu->seq_id, ndomains);
ba395927
KA		 1429 nlongs = BITS_TO_LONGS(ndomains);
1430
94a91b50
DD		 1431 spin_lock_init(&iommu->lock);
1432
ba395927
KA		 1433 /* TBD: there might be 64K domains,
1434 * consider other allocation for future chip
1435 */
1436 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1437 if (!iommu->domain_ids) {
852bdb04
JL		 1438 pr_err("IOMMU%d: allocating domain id array failed\n",
1439 iommu->seq_id);
ba395927
KA		 1440 return -ENOMEM;
1441 }
1442 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1443 GFP_KERNEL);
1444 if (!iommu->domains) {
852bdb04
JL		 1445 pr_err("IOMMU%d: allocating domain array failed\n",
1446 iommu->seq_id);
1447 kfree(iommu->domain_ids);
1448 iommu->domain_ids = NULL;
ba395927
KA		 1449 return -ENOMEM;
1450 }
1451
1452 /*
1453 * if Caching mode is set, then invalid translations are tagged
1454 * with domainid 0. Hence we need to pre-allocate it.
1455 */
1456 if (cap_caching_mode(iommu->cap))
1457 set_bit(0, iommu->domain_ids);
1458 return 0;
1459}
ba395927 1460
a868e6b7 1461static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1462{
1463 struct dmar_domain *domain;
5ced12af 1464 int i, count;
c7151a8d 1465 unsigned long flags;
ba395927 1466
94a91b50 1467 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1468 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1469 /*
1470 * Domain id 0 is reserved for invalid translation
1471 * if hardware supports caching mode.
1472 */
1473 if (cap_caching_mode(iommu->cap) && i == 0)
1474 continue;
1475
94a91b50
DD		 1476 domain = iommu->domains[i];
1477 clear_bit(i, iommu->domain_ids);
1478
1479 spin_lock_irqsave(&domain->iommu_lock, flags);
5ced12af
JL		 1480 count = --domain->iommu_count;
1481 spin_unlock_irqrestore(&domain->iommu_lock, flags);
92d03cc8
JL		 1482 if (count == 0)
1483 domain_exit(domain);
5e98c4b1 1484 }
ba395927
KA		 1485 }
1486
1487 if (iommu->gcmd & DMA_GCMD_TE)
1488 iommu_disable_translation(iommu);
1489
ba395927
KA		 1490 kfree(iommu->domains);
1491 kfree(iommu->domain_ids);
a868e6b7
JL		 1492 iommu->domains = NULL;
1493 iommu->domain_ids = NULL;
ba395927 1494
d9630fe9
WH		 1495 g_iommus[iommu->seq_id] = NULL;
1496
ba395927
KA		 1497 /* free context mapping */
1498 free_context_table(iommu);
ba395927
KA		 1499}
1500
92d03cc8 1501static struct dmar_domain *alloc_domain(bool vm)
ba395927 1502{
92d03cc8
JL		 1503 /* domain id for virtual machine, it won't be set in context */
1504 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1505 struct dmar_domain *domain;
ba395927
KA		 1506
1507 domain = alloc_domain_mem();
1508 if (!domain)
1509 return NULL;
1510
4c923d47 1511 domain->nid = -1;
92d03cc8 1512 domain->iommu_count = 0;
1b198bb0 1513 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38 1514 domain->flags = 0;
92d03cc8
JL		 1515 spin_lock_init(&domain->iommu_lock);
1516 INIT_LIST_HEAD(&domain->devices);
1517 if (vm) {
1518 domain->id = atomic_inc_return(&vm_domid);
1519 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1520 }
2c2e2c38
FY		 1521
1522 return domain;
1523}
1524
1525static int iommu_attach_domain(struct dmar_domain *domain,
1526 struct intel_iommu *iommu)
1527{
1528 int num;
1529 unsigned long ndomains;
1530 unsigned long flags;
1531
ba395927
KA		 1532 ndomains = cap_ndoms(iommu->cap);
1533
1534 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1535
ba395927
KA		 1536 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1537 if (num >= ndomains) {
1538 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1539 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1540 return -ENOMEM;
ba395927
KA		 1541 }
1542
ba395927 1543 domain->id = num;
9ebd682e 1544 domain->iommu_count++;
2c2e2c38 1545 set_bit(num, iommu->domain_ids);
1b198bb0 1546 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1547 iommu->domains[num] = domain;
1548 spin_unlock_irqrestore(&iommu->lock, flags);
1549
2c2e2c38 1550 return 0;
ba395927
KA		 1551}
1552
2c2e2c38
FY		 1553static void iommu_detach_domain(struct dmar_domain *domain,
1554 struct intel_iommu *iommu)
ba395927
KA		 1555{
1556 unsigned long flags;
2c2e2c38 1557 int num, ndomains;
ba395927 1558
8c11e798 1559 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1560 ndomains = cap_ndoms(iommu->cap);
a45946ab 1561 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38 1562 if (iommu->domains[num] == domain) {
92d03cc8
JL		 1563 clear_bit(num, iommu->domain_ids);
1564 iommu->domains[num] = NULL;
2c2e2c38
FY		 1565 break;
1566 }
2c2e2c38 1567 }
8c11e798 1568 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1569}
1570
1571static struct iova_domain reserved_iova_list;
8a443df4 1572static struct lock_class_key reserved_rbtree_key;
ba395927 1573
51a63e67 1574static int dmar_init_reserved_ranges(void)
ba395927
KA		 1575{
1576 struct pci_dev *pdev = NULL;
1577 struct iova *iova;
1578 int i;
ba395927 1579
f661197e 1580 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1581
8a443df4
MG		 1582 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1583 &reserved_rbtree_key);
1584
ba395927
KA		 1585 /* IOAPIC ranges shouldn't be accessed by DMA */
1586 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1587 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1588 if (!iova) {
ba395927 1589 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1590 return -ENODEV;
1591 }
ba395927
KA		 1592
1593 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1594 for_each_pci_dev(pdev) {
1595 struct resource *r;
1596
1597 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1598 r = &pdev->resource[i];
1599 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1600 continue;
1a4a4551
DW		 1601 iova = reserve_iova(&reserved_iova_list,
1602 IOVA_PFN(r->start),
1603 IOVA_PFN(r->end));
51a63e67 1604 if (!iova) {
ba395927 1605 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1606 return -ENODEV;
1607 }
ba395927
KA		 1608 }
1609 }
51a63e67 1610 return 0;
ba395927
KA		 1611}
1612
1613static void domain_reserve_special_ranges(struct dmar_domain *domain)
1614{
1615 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1616}
1617
1618static inline int guestwidth_to_adjustwidth(int gaw)
1619{
1620 int agaw;
1621 int r = (gaw - 12) % 9;
1622
1623 if (r == 0)
1624 agaw = gaw;
1625 else
1626 agaw = gaw + 9 - r;
1627 if (agaw > 64)
1628 agaw = 64;
1629 return agaw;
1630}
1631
1632static int domain_init(struct dmar_domain *domain, int guest_width)
1633{
1634 struct intel_iommu *iommu;
1635 int adjust_width, agaw;
1636 unsigned long sagaw;
1637
f661197e 1638 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927
KA		 1639 domain_reserve_special_ranges(domain);
1640
1641 /* calculate AGAW */
8c11e798 1642 iommu = domain_get_iommu(domain);
ba395927
KA		 1643 if (guest_width > cap_mgaw(iommu->cap))
1644 guest_width = cap_mgaw(iommu->cap);
1645 domain->gaw = guest_width;
1646 adjust_width = guestwidth_to_adjustwidth(guest_width);
1647 agaw = width_to_agaw(adjust_width);
1648 sagaw = cap_sagaw(iommu->cap);
1649 if (!test_bit(agaw, &sagaw)) {
1650 /* hardware doesn't support it, choose a bigger one */
1651 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1652 agaw = find_next_bit(&sagaw, 5, agaw);
1653 if (agaw >= 5)
1654 return -ENODEV;
1655 }
1656 domain->agaw = agaw;
ba395927 1657
8e604097
WH		 1658 if (ecap_coherent(iommu->ecap))
1659 domain->iommu_coherency = 1;
1660 else
1661 domain->iommu_coherency = 0;
1662
58c610bd
SY		 1663 if (ecap_sc_support(iommu->ecap))
1664 domain->iommu_snooping = 1;
1665 else
1666 domain->iommu_snooping = 0;
1667
214e39aa
DW		 1668 if (intel_iommu_superpage)
1669 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1670 else
1671 domain->iommu_superpage = 0;
1672
4c923d47 1673 domain->nid = iommu->node;
c7151a8d 1674
ba395927 1675 /* always allocate the top pgd */
4c923d47 1676 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1677 if (!domain->pgd)
1678 return -ENOMEM;
5b6985ce 1679 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1680 return 0;
1681}
1682
1683static void domain_exit(struct dmar_domain *domain)
1684{
2c2e2c38
FY		 1685 struct dmar_drhd_unit *drhd;
1686 struct intel_iommu *iommu;
ea8ea460 1687 struct page *freelist = NULL;
ba395927
KA		 1688
1689 /* Domain 0 is reserved, so dont process it */
1690 if (!domain)
1691 return;
1692
7b668357
AW		 1693 /* Flush any lazy unmaps that may reference this domain */
1694 if (!intel_iommu_strict)
1695 flush_unmaps_timeout(0);
1696
92d03cc8 1697 /* remove associated devices */
ba395927 1698 domain_remove_dev_info(domain);
92d03cc8 1699
ba395927
KA		 1700 /* destroy iovas */
1701 put_iova_domain(&domain->iovad);
ba395927 1702
ea8ea460 1703 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1704
92d03cc8 1705 /* clear attached or cached domains */
0e242612 1706 rcu_read_lock();
2c2e2c38 1707 for_each_active_iommu(iommu, drhd)
92d03cc8
JL		 1708 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1709 test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38 1710 iommu_detach_domain(domain, iommu);
0e242612 1711 rcu_read_unlock();
2c2e2c38 1712
ea8ea460
DW		 1713 dma_free_pagelist(freelist);
1714
ba395927
KA		 1715 free_domain_mem(domain);
1716}
1717
64ae892b
DW		 1718static int domain_context_mapping_one(struct dmar_domain *domain,
1719 struct intel_iommu *iommu,
1720 u8 bus, u8 devfn, int translation)
ba395927
KA		 1721{
1722 struct context_entry *context;
ba395927 1723 unsigned long flags;
ea6606b0
WH		 1724 struct dma_pte *pgd;
1725 unsigned long num;
1726 unsigned long ndomains;
1727 int id;
1728 int agaw;
93a23a72 1729 struct device_domain_info *info = NULL;
ba395927
KA		 1730
1731 pr_debug("Set context mapping for %02x:%02x.%d\n",
1732 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1733
ba395927 1734 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1735 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1736 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1737
ba395927
KA		 1738 context = device_to_context_entry(iommu, bus, devfn);
1739 if (!context)
1740 return -ENOMEM;
1741 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1742 if (context_present(context)) {
ba395927
KA		 1743 spin_unlock_irqrestore(&iommu->lock, flags);
1744 return 0;
1745 }
1746
ea6606b0
WH		 1747 id = domain->id;
1748 pgd = domain->pgd;
1749
2c2e2c38
FY		 1750 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1751 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1752 int found = 0;
1753
1754 /* find an available domain id for this device in iommu */
1755 ndomains = cap_ndoms(iommu->cap);
a45946ab 1756 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1757 if (iommu->domains[num] == domain) {
1758 id = num;
1759 found = 1;
1760 break;
1761 }
ea6606b0
WH		 1762 }
1763
1764 if (found == 0) {
1765 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1766 if (num >= ndomains) {
1767 spin_unlock_irqrestore(&iommu->lock, flags);
1768 printk(KERN_ERR "IOMMU: no free domain ids\n");
1769 return -EFAULT;
1770 }
1771
1772 set_bit(num, iommu->domain_ids);
1773 iommu->domains[num] = domain;
1774 id = num;
1775 }
1776
1777 /* Skip top levels of page tables for
1778 * iommu which has less agaw than default.
1672af11 1779 * Unnecessary for PT mode.
ea6606b0 1780 */
1672af11
CW		 1781 if (translation != CONTEXT_TT_PASS_THROUGH) {
1782 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1783 pgd = phys_to_virt(dma_pte_addr(pgd));
1784 if (!dma_pte_present(pgd)) {
1785 spin_unlock_irqrestore(&iommu->lock, flags);
1786 return -ENOMEM;
1787 }
ea6606b0
WH		 1788 }
1789 }
1790 }
1791
1792 context_set_domain_id(context, id);
4ed0d3e6 1793
93a23a72 1794 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1795 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1796 translation = info ? CONTEXT_TT_DEV_IOTLB :
1797 CONTEXT_TT_MULTI_LEVEL;
1798 }
4ed0d3e6
FY		 1799 /*
1800 * In pass through mode, AW must be programmed to indicate the largest
1801 * AGAW value supported by hardware. And ASR is ignored by hardware.
1802 */
93a23a72 1803 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1804 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1805 else {
1806 context_set_address_root(context, virt_to_phys(pgd));
1807 context_set_address_width(context, iommu->agaw);
1808 }
4ed0d3e6
FY		 1809
1810 context_set_translation_type(context, translation);
c07e7d21
MM		 1811 context_set_fault_enable(context);
1812 context_set_present(context);
5331fe6f 1813 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1814
4c25a2c1
DW		 1815 /*
1816 * It's a non-present to present mapping. If hardware doesn't cache
1817 * non-present entry we only need to flush the write-buffer. If the
1818 * _does_ cache non-present entries, then it does so in the special
1819 * domain #0, which we have to flush:
1820 */
1821 if (cap_caching_mode(iommu->cap)) {
1822 iommu->flush.flush_context(iommu, 0,
1823 (((u16)bus) << 8) | devfn,
1824 DMA_CCMD_MASK_NOBIT,
1825 DMA_CCMD_DEVICE_INVL);
82653633 1826 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1827 } else {
ba395927 1828 iommu_flush_write_buffer(iommu);
4c25a2c1 1829 }
93a23a72 1830 iommu_enable_dev_iotlb(info);
ba395927 1831 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1832
1833 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1834 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1835 domain->iommu_count++;
4c923d47
SS		 1836 if (domain->iommu_count == 1)
1837 domain->nid = iommu->node;
58c610bd 1838 domain_update_iommu_cap(domain);
c7151a8d
WH		 1839 }
1840 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1841 return 0;
1842}
1843
579305f7
AW		 1844struct domain_context_mapping_data {
1845 struct dmar_domain *domain;
1846 struct intel_iommu *iommu;
1847 int translation;
1848};
1849
1850static int domain_context_mapping_cb(struct pci_dev *pdev,
1851 u16 alias, void *opaque)
1852{
1853 struct domain_context_mapping_data *data = opaque;
1854
1855 return domain_context_mapping_one(data->domain, data->iommu,
1856 PCI_BUS_NUM(alias), alias & 0xff,
1857 data->translation);
1858}
1859
ba395927 1860static int
e1f167f3
DW		 1861domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1862 int translation)
ba395927 1863{
64ae892b 1864 struct intel_iommu *iommu;
156baca8 1865 u8 bus, devfn;
579305f7 1866 struct domain_context_mapping_data data;
64ae892b 1867
e1f167f3 1868 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1869 if (!iommu)
1870 return -ENODEV;
ba395927 1871
579305f7
AW		 1872 if (!dev_is_pci(dev))
1873 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1874 translation);
579305f7
AW		 1875
1876 data.domain = domain;
1877 data.iommu = iommu;
1878 data.translation = translation;
1879
1880 return pci_for_each_dma_alias(to_pci_dev(dev),
1881 &domain_context_mapping_cb, &data);
1882}
1883
1884static int domain_context_mapped_cb(struct pci_dev *pdev,
1885 u16 alias, void *opaque)
1886{
1887 struct intel_iommu *iommu = opaque;
1888
1889 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 1890}
1891
e1f167f3 1892static int domain_context_mapped(struct device *dev)
ba395927 1893{
5331fe6f 1894 struct intel_iommu *iommu;
156baca8 1895 u8 bus, devfn;
5331fe6f 1896
e1f167f3 1897 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 1898 if (!iommu)
1899 return -ENODEV;
ba395927 1900
579305f7
AW		 1901 if (!dev_is_pci(dev))
1902 return device_context_mapped(iommu, bus, devfn);
e1f167f3 1903
579305f7
AW		 1904 return !pci_for_each_dma_alias(to_pci_dev(dev),
1905 domain_context_mapped_cb, iommu);
ba395927
KA		 1906}
1907
f532959b
FY		 1908/* Returns a number of VTD pages, but aligned to MM page size */
1909static inline unsigned long aligned_nrpages(unsigned long host_addr,
1910 size_t size)
1911{
1912 host_addr &= ~PAGE_MASK;
1913 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1914}
1915
6dd9a7c7
YS		 1916/* Return largest possible superpage level for a given mapping */
1917static inline int hardware_largepage_caps(struct dmar_domain *domain,
1918 unsigned long iov_pfn,
1919 unsigned long phy_pfn,
1920 unsigned long pages)
1921{
1922 int support, level = 1;
1923 unsigned long pfnmerge;
1924
1925 support = domain->iommu_superpage;
1926
1927 /* To use a large page, the virtual *and* physical addresses
1928 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1929 of them will mean we have to use smaller pages. So just
1930 merge them and check both at once. */
1931 pfnmerge = iov_pfn | phy_pfn;
1932
1933 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1934 pages >>= VTD_STRIDE_SHIFT;
1935 if (!pages)
1936 break;
1937 pfnmerge >>= VTD_STRIDE_SHIFT;
1938 level++;
1939 support--;
1940 }
1941 return level;
1942}
1943
9051aa02
DW		 1944static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1945 struct scatterlist *sg, unsigned long phys_pfn,
1946 unsigned long nr_pages, int prot)
e1605495
DW		 1947{
1948 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1949 phys_addr_t uninitialized_var(pteval);
e1605495 1950 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1951 unsigned long sg_res;
6dd9a7c7
YS		 1952 unsigned int largepage_lvl = 0;
1953 unsigned long lvl_pages = 0;
e1605495
DW		 1954
1955 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1956
1957 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1958 return -EINVAL;
1959
1960 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1961
9051aa02
DW		 1962 if (sg)
1963 sg_res = 0;
1964 else {
1965 sg_res = nr_pages + 1;
1966 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1967 }
1968
6dd9a7c7 1969 while (nr_pages > 0) {
c85994e4
DW		 1970 uint64_t tmp;
1971
e1605495 1972 if (!sg_res) {
f532959b 1973 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1974 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1975 sg->dma_length = sg->length;
1976 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1977 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1978 }
6dd9a7c7 1979
e1605495 1980 if (!pte) {
6dd9a7c7
YS		 1981 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1982
5cf0a76f 1983 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 1984 if (!pte)
1985 return -ENOMEM;
6dd9a7c7 1986 /* It is large page*/
6491d4d0 1987 if (largepage_lvl > 1) {
6dd9a7c7 1988 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1989 /* Ensure that old small page tables are removed to make room
1990 for superpage, if they exist. */
1991 dma_pte_clear_range(domain, iov_pfn,
1992 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1993 dma_pte_free_pagetable(domain, iov_pfn,
1994 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1995 } else {
6dd9a7c7 1996 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1997 }
6dd9a7c7 1998
e1605495
DW		 1999 }
2000 /* We don't need lock here, nobody else
2001 * touches the iova range
2002 */
7766a3fb 2003 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2004 if (tmp) {
1bf20f0d 2005 static int dumps = 5;
c85994e4
DW		 2006 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2007 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2008 if (dumps) {
2009 dumps--;
2010 debug_dma_dump_mappings(NULL);
2011 }
2012 WARN_ON(1);
2013 }
6dd9a7c7
YS		 2014
2015 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2016
2017 BUG_ON(nr_pages < lvl_pages);
2018 BUG_ON(sg_res < lvl_pages);
2019
2020 nr_pages -= lvl_pages;
2021 iov_pfn += lvl_pages;
2022 phys_pfn += lvl_pages;
2023 pteval += lvl_pages * VTD_PAGE_SIZE;
2024 sg_res -= lvl_pages;
2025
2026 /* If the next PTE would be the first in a new page, then we
2027 need to flush the cache on the entries we've just written.
2028 And then we'll need to recalculate 'pte', so clear it and
2029 let it get set again in the if (!pte) block above.
2030
2031 If we're done (!nr_pages) we need to flush the cache too.
2032
2033 Also if we've been setting superpages, we may need to
2034 recalculate 'pte' and switch back to smaller pages for the
2035 end of the mapping, if the trailing size is not enough to
2036 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2037 pte++;
6dd9a7c7
YS		 2038 if (!nr_pages || first_pte_in_page(pte) ||
2039 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2040 domain_flush_cache(domain, first_pte,
2041 (void *)pte - (void *)first_pte);
2042 pte = NULL;
2043 }
6dd9a7c7
YS		 2044
2045 if (!sg_res && nr_pages)
e1605495
DW		 2046 sg = sg_next(sg);
2047 }
2048 return 0;
2049}
2050
9051aa02
DW		 2051static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2052 struct scatterlist *sg, unsigned long nr_pages,
2053 int prot)
ba395927 2054{
9051aa02
DW		 2055 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2056}
6f6a00e4 2057
9051aa02
DW		 2058static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2059 unsigned long phys_pfn, unsigned long nr_pages,
2060 int prot)
2061{
2062 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2063}
2064
c7151a8d 2065static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2066{
c7151a8d
WH		 2067 if (!iommu)
2068 return;
8c11e798
WH		 2069
2070 clear_context_table(iommu, bus, devfn);
2071 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2072 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2073 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2074}
2075
109b9b04
DW		 2076static inline void unlink_domain_info(struct device_domain_info *info)
2077{
2078 assert_spin_locked(&device_domain_lock);
2079 list_del(&info->link);
2080 list_del(&info->global);
2081 if (info->dev)
0bcb3e28 2082 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2083}
2084
ba395927
KA		 2085static void domain_remove_dev_info(struct dmar_domain *domain)
2086{
2087 struct device_domain_info *info;
92d03cc8 2088 unsigned long flags, flags2;
ba395927
KA		 2089
2090 spin_lock_irqsave(&device_domain_lock, flags);
2091 while (!list_empty(&domain->devices)) {
2092 info = list_entry(domain->devices.next,
2093 struct device_domain_info, link);
109b9b04 2094 unlink_domain_info(info);
ba395927
KA		 2095 spin_unlock_irqrestore(&device_domain_lock, flags);
2096
93a23a72 2097 iommu_disable_dev_iotlb(info);
7c7faa11 2098 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2099
92d03cc8 2100 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
7c7faa11 2101 iommu_detach_dependent_devices(info->iommu, info->dev);
92d03cc8
JL		 2102 /* clear this iommu in iommu_bmp, update iommu count
2103 * and capabilities
2104 */
2105 spin_lock_irqsave(&domain->iommu_lock, flags2);
7c7faa11 2106 if (test_and_clear_bit(info->iommu->seq_id,
92d03cc8
JL		 2107 domain->iommu_bmp)) {
2108 domain->iommu_count--;
2109 domain_update_iommu_cap(domain);
2110 }
2111 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2112 }
2113
2114 free_devinfo_mem(info);
ba395927
KA		 2115 spin_lock_irqsave(&device_domain_lock, flags);
2116 }
2117 spin_unlock_irqrestore(&device_domain_lock, flags);
2118}
2119
2120/*
2121 * find_domain
1525a29a 2122 * Note: we use struct device->archdata.iommu stores the info
ba395927 2123 */
1525a29a 2124static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2125{
2126 struct device_domain_info *info;
2127
2128 /* No lock here, assumes no domain exit in normal case */
1525a29a 2129 info = dev->archdata.iommu;
ba395927
KA		 2130 if (info)
2131 return info->domain;
2132 return NULL;
2133}
2134
5a8f40e8 2135static inline struct device_domain_info *
745f2586
JL		 2136dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2137{
2138 struct device_domain_info *info;
2139
2140 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2141 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2142 info->devfn == devfn)
5a8f40e8 2143 return info;
745f2586
JL		 2144
2145 return NULL;
2146}
2147
5a8f40e8 2148static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2149 int bus, int devfn,
b718cd3d
DW		 2150 struct device *dev,
2151 struct dmar_domain *domain)
745f2586 2152{
5a8f40e8 2153 struct dmar_domain *found = NULL;
745f2586
JL		 2154 struct device_domain_info *info;
2155 unsigned long flags;
2156
2157 info = alloc_devinfo_mem();
2158 if (!info)
b718cd3d 2159 return NULL;
745f2586 2160
745f2586
JL		 2161 info->bus = bus;
2162 info->devfn = devfn;
2163 info->dev = dev;
2164 info->domain = domain;
5a8f40e8 2165 info->iommu = iommu;
745f2586
JL		 2166 if (!dev)
2167 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2168
2169 spin_lock_irqsave(&device_domain_lock, flags);
2170 if (dev)
0bcb3e28 2171 found = find_domain(dev);
5a8f40e8
DW		 2172 else {
2173 struct device_domain_info *info2;
41e80dca 2174 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2175 if (info2)
2176 found = info2->domain;
2177 }
745f2586
JL		 2178 if (found) {
2179 spin_unlock_irqrestore(&device_domain_lock, flags);
2180 free_devinfo_mem(info);
b718cd3d
DW		 2181 /* Caller must free the original domain */
2182 return found;
745f2586
JL		 2183 }
2184
b718cd3d
DW		 2185 list_add(&info->link, &domain->devices);
2186 list_add(&info->global, &device_domain_list);
2187 if (dev)
2188 dev->archdata.iommu = info;
2189 spin_unlock_irqrestore(&device_domain_lock, flags);
2190
2191 return domain;
745f2586
JL		 2192}
2193
579305f7
AW		 2194static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2195{
2196 *(u16 *)opaque = alias;
2197 return 0;
2198}
2199
ba395927 2200/* domain is initialized */
146922ec 2201static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2202{
579305f7
AW		 2203 struct dmar_domain *domain, *tmp;
2204 struct intel_iommu *iommu;
5a8f40e8 2205 struct device_domain_info *info;
579305f7 2206 u16 dma_alias;
ba395927 2207 unsigned long flags;
579305f7 2208 u8 bus, devfn;
ba395927 2209
146922ec 2210 domain = find_domain(dev);
ba395927
KA		 2211 if (domain)
2212 return domain;
2213
579305f7
AW		 2214 iommu = device_to_iommu(dev, &bus, &devfn);
2215 if (!iommu)
2216 return NULL;
2217
146922ec
DW		 2218 if (dev_is_pci(dev)) {
2219 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2220
579305f7
AW		 2221 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2222
2223 spin_lock_irqsave(&device_domain_lock, flags);
2224 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2225 PCI_BUS_NUM(dma_alias),
2226 dma_alias & 0xff);
2227 if (info) {
2228 iommu = info->iommu;
2229 domain = info->domain;
5a8f40e8 2230 }
579305f7 2231 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2232
579305f7
AW		 2233 /* DMA alias already has a domain, uses it */
2234 if (info)
2235 goto found_domain;
2236 }
ba395927 2237
146922ec 2238 /* Allocate and initialize new domain for the device */
92d03cc8 2239 domain = alloc_domain(false);
745f2586 2240 if (!domain)
579305f7
AW		 2241 return NULL;
2242
745f2586 2243 if (iommu_attach_domain(domain, iommu)) {
2fe9723d 2244 free_domain_mem(domain);
579305f7
AW		 2245 return NULL;
2246 }
2247
2248 if (domain_init(domain, gaw)) {
2249 domain_exit(domain);
2250 return NULL;
2c2e2c38 2251 }
ba395927 2252
579305f7
AW		 2253 /* register PCI DMA alias device */
2254 if (dev_is_pci(dev)) {
2255 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2256 dma_alias & 0xff, NULL, domain);
2257
2258 if (!tmp || tmp != domain) {
2259 domain_exit(domain);
2260 domain = tmp;
2261 }
2262
b718cd3d 2263 if (!domain)
579305f7 2264 return NULL;
ba395927
KA		 2265 }
2266
2267found_domain:
579305f7
AW		 2268 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2269
2270 if (!tmp || tmp != domain) {
2271 domain_exit(domain);
2272 domain = tmp;
2273 }
b718cd3d
DW		 2274
2275 return domain;
ba395927
KA		 2276}
2277
2c2e2c38 2278static int iommu_identity_mapping;
e0fc7e0b
DW		 2279#define IDENTMAP_ALL 1
2280#define IDENTMAP_GFX 2
2281#define IDENTMAP_AZALIA 4
2c2e2c38 2282
b213203e
DW		 2283static int iommu_domain_identity_map(struct dmar_domain *domain,
2284 unsigned long long start,
2285 unsigned long long end)
ba395927 2286{
c5395d5c
DW		 2287 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2288 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2289
2290 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2291 dma_to_mm_pfn(last_vpfn))) {
ba395927 2292 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2293 return -ENOMEM;
ba395927
KA		 2294 }
2295
c5395d5c
DW		 2296 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2297 start, end, domain->id);
ba395927
KA		 2298 /*
2299 * RMRR range might have overlap with physical memory range,
2300 * clear it first
2301 */
c5395d5c 2302 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2303
c5395d5c
DW		 2304 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2305 last_vpfn - first_vpfn + 1,
61df7443 2306 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2307}
2308
0b9d9753 2309static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2310 unsigned long long start,
2311 unsigned long long end)
2312{
2313 struct dmar_domain *domain;
2314 int ret;
2315
0b9d9753 2316 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2317 if (!domain)
2318 return -ENOMEM;
2319
19943b0e
DW		 2320 /* For _hardware_ passthrough, don't bother. But for software
2321 passthrough, we do it anyway -- it may indicate a memory
2322 range which is reserved in E820, so which didn't get set
2323 up to start with in si_domain */
2324 if (domain == si_domain && hw_pass_through) {
2325 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2326 dev_name(dev), start, end);
19943b0e
DW		 2327 return 0;
2328 }
2329
2330 printk(KERN_INFO
2331 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2332 dev_name(dev), start, end);
2ff729f5 2333
5595b528
DW		 2334 if (end < start) {
2335 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2336 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2337 dmi_get_system_info(DMI_BIOS_VENDOR),
2338 dmi_get_system_info(DMI_BIOS_VERSION),
2339 dmi_get_system_info(DMI_PRODUCT_VERSION));
2340 ret = -EIO;
2341 goto error;
2342 }
2343
2ff729f5
DW		 2344 if (end >> agaw_to_width(domain->agaw)) {
2345 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2346 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2347 agaw_to_width(domain->agaw),
2348 dmi_get_system_info(DMI_BIOS_VENDOR),
2349 dmi_get_system_info(DMI_BIOS_VERSION),
2350 dmi_get_system_info(DMI_PRODUCT_VERSION));
2351 ret = -EIO;
2352 goto error;
2353 }
19943b0e 2354
b213203e 2355 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2356 if (ret)
2357 goto error;
2358
2359 /* context entry init */
0b9d9753 2360 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2361 if (ret)
2362 goto error;
2363
2364 return 0;
2365
2366 error:
ba395927
KA		 2367 domain_exit(domain);
2368 return ret;
ba395927
KA		 2369}
2370
2371static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2372 struct device *dev)
ba395927 2373{
0b9d9753 2374 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2375 return 0;
0b9d9753
DW		 2376 return iommu_prepare_identity_map(dev, rmrr->base_address,
2377 rmrr->end_address);
ba395927
KA		 2378}
2379
d3f13810 2380#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2381static inline void iommu_prepare_isa(void)
2382{
2383 struct pci_dev *pdev;
2384 int ret;
2385
2386 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2387 if (!pdev)
2388 return;
2389
c7ab48d2 2390 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2391 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2392
2393 if (ret)
c7ab48d2
DW		 2394 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2395 "floppy might not work\n");
49a0429e
KA		 2396
2397}
2398#else
2399static inline void iommu_prepare_isa(void)
2400{
2401 return;
2402}
d3f13810 2403#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2404
2c2e2c38 2405static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2406
071e1374 2407static int __init si_domain_init(int hw)
2c2e2c38
FY		 2408{
2409 struct dmar_drhd_unit *drhd;
2410 struct intel_iommu *iommu;
c7ab48d2 2411 int nid, ret = 0;
2c2e2c38 2412
92d03cc8 2413 si_domain = alloc_domain(false);
2c2e2c38
FY		 2414 if (!si_domain)
2415 return -EFAULT;
2416
92d03cc8
JL		 2417 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2418
2c2e2c38
FY		 2419 for_each_active_iommu(iommu, drhd) {
2420 ret = iommu_attach_domain(si_domain, iommu);
2421 if (ret) {
2422 domain_exit(si_domain);
2423 return -EFAULT;
2424 }
2425 }
2426
2427 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2428 domain_exit(si_domain);
2429 return -EFAULT;
2430 }
2431
9544c003
JL		 2432 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2433 si_domain->id);
2c2e2c38 2434
19943b0e
DW		 2435 if (hw)
2436 return 0;
2437
c7ab48d2 2438 for_each_online_node(nid) {
5dfe8660
TH		 2439 unsigned long start_pfn, end_pfn;
2440 int i;
2441
2442 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2443 ret = iommu_domain_identity_map(si_domain,
2444 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2445 if (ret)
2446 return ret;
2447 }
c7ab48d2
DW		 2448 }
2449
2c2e2c38
FY		 2450 return 0;
2451}
2452
9b226624 2453static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2454{
2455 struct device_domain_info *info;
2456
2457 if (likely(!iommu_identity_mapping))
2458 return 0;
2459
9b226624 2460 info = dev->archdata.iommu;
cb452a40
MT		 2461 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2462 return (info->domain == si_domain);
2c2e2c38 2463
2c2e2c38
FY		 2464 return 0;
2465}
2466
2467static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2468 struct device *dev, int translation)
2c2e2c38 2469{
0ac72664 2470 struct dmar_domain *ndomain;
5a8f40e8 2471 struct intel_iommu *iommu;
156baca8 2472 u8 bus, devfn;
5fe60f4e 2473 int ret;
2c2e2c38 2474
5913c9bf 2475 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2476 if (!iommu)
2477 return -ENODEV;
2478
5913c9bf 2479 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2480 if (ndomain != domain)
2481 return -EBUSY;
2c2e2c38 2482
5913c9bf 2483 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2484 if (ret) {
5913c9bf 2485 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2486 return ret;
2487 }
2488
2c2e2c38
FY		 2489 return 0;
2490}
2491
0b9d9753 2492static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2493{
2494 struct dmar_rmrr_unit *rmrr;
832bd858 2495 struct device *tmp;
ea2447f7
TM		 2496 int i;
2497
0e242612 2498 rcu_read_lock();
ea2447f7 2499 for_each_rmrr_units(rmrr) {
b683b230
JL		 2500 /*
2501 * Return TRUE if this RMRR contains the device that
2502 * is passed in.
2503 */
2504 for_each_active_dev_scope(rmrr->devices,
2505 rmrr->devices_cnt, i, tmp)
0b9d9753 2506 if (tmp == dev) {
0e242612 2507 rcu_read_unlock();
ea2447f7 2508 return true;
b683b230 2509 }
ea2447f7 2510 }
0e242612 2511 rcu_read_unlock();
ea2447f7
TM		 2512 return false;
2513}
2514
3bdb2591 2515static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2516{
ea2447f7 2517
3bdb2591
DW		 2518 if (dev_is_pci(dev)) {
2519 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2520
3bdb2591
DW		 2521 /*
2522 * We want to prevent any device associated with an RMRR from
2523 * getting placed into the SI Domain. This is done because
2524 * problems exist when devices are moved in and out of domains
2525 * and their respective RMRR info is lost. We exempt USB devices
2526 * from this process due to their usage of RMRRs that are known
2527 * to not be needed after BIOS hand-off to OS.
2528 */
2529 if (device_has_rmrr(dev) &&
2530 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2531 return 0;
e0fc7e0b 2532
3bdb2591
DW		 2533 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2534 return 1;
e0fc7e0b 2535
3bdb2591
DW		 2536 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2537 return 1;
6941af28 2538
3bdb2591 2539 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2540 return 0;
3bdb2591
DW		 2541
2542 /*
2543 * We want to start off with all devices in the 1:1 domain, and
2544 * take them out later if we find they can't access all of memory.
2545 *
2546 * However, we can't do this for PCI devices behind bridges,
2547 * because all PCI devices behind the same bridge will end up
2548 * with the same source-id on their transactions.
2549 *
2550 * Practically speaking, we can't change things around for these
2551 * devices at run-time, because we can't be sure there'll be no
2552 * DMA transactions in flight for any of their siblings.
2553 *
2554 * So PCI devices (unless they're on the root bus) as well as
2555 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2556 * the 1:1 domain, just in _case_ one of their siblings turns out
2557 * not to be able to map all of memory.
2558 */
2559 if (!pci_is_pcie(pdev)) {
2560 if (!pci_is_root_bus(pdev->bus))
2561 return 0;
2562 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2563 return 0;
2564 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2565 return 0;
3bdb2591
DW		 2566 } else {
2567 if (device_has_rmrr(dev))
2568 return 0;
2569 }
3dfc813d 2570
3bdb2591 2571 /*
3dfc813d 2572 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2573 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2574 * take them out of the 1:1 domain later.
2575 */
8fcc5372
CW		 2576 if (!startup) {
2577 /*
2578 * If the device's dma_mask is less than the system's memory
2579 * size then this is not a candidate for identity mapping.
2580 */
3bdb2591 2581 u64 dma_mask = *dev->dma_mask;
8fcc5372 2582
3bdb2591
DW		 2583 if (dev->coherent_dma_mask &&
2584 dev->coherent_dma_mask < dma_mask)
2585 dma_mask = dev->coherent_dma_mask;
8fcc5372 2586
3bdb2591 2587 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2588 }
6941af28
DW		 2589
2590 return 1;
2591}
2592
cf04eee8
DW		 2593static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2594{
2595 int ret;
2596
2597 if (!iommu_should_identity_map(dev, 1))
2598 return 0;
2599
2600 ret = domain_add_dev_info(si_domain, dev,
2601 hw ? CONTEXT_TT_PASS_THROUGH :
2602 CONTEXT_TT_MULTI_LEVEL);
2603 if (!ret)
2604 pr_info("IOMMU: %s identity mapping for device %s\n",
2605 hw ? "hardware" : "software", dev_name(dev));
2606 else if (ret == -ENODEV)
2607 /* device not associated with an iommu */
2608 ret = 0;
2609
2610 return ret;
2611}
2612
2613
071e1374 2614static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2615{
2c2e2c38 2616 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2617 struct dmar_drhd_unit *drhd;
2618 struct intel_iommu *iommu;
2619 struct device *dev;
2620 int i;
2621 int ret = 0;
2c2e2c38 2622
19943b0e 2623 ret = si_domain_init(hw);
2c2e2c38
FY		 2624 if (ret)
2625 return -EFAULT;
2626
2c2e2c38 2627 for_each_pci_dev(pdev) {
cf04eee8
DW		 2628 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2629 if (ret)
2630 return ret;
2631 }
2632
2633 for_each_active_iommu(iommu, drhd)
2634 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2635 struct acpi_device_physical_node *pn;
2636 struct acpi_device *adev;
2637
2638 if (dev->bus != &acpi_bus_type)
2639 continue;
2640
2641 adev= to_acpi_device(dev);
2642 mutex_lock(&adev->physical_node_lock);
2643 list_for_each_entry(pn, &adev->physical_node_list, node) {
2644 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2645 if (ret)
2646 break;
eae460b6 2647 }
cf04eee8
DW		 2648 mutex_unlock(&adev->physical_node_lock);
2649 if (ret)
2650 return ret;
62edf5dc 2651 }
2c2e2c38
FY		 2652
2653 return 0;
2654}
2655
b779260b 2656static int __init init_dmars(void)
ba395927
KA		 2657{
2658 struct dmar_drhd_unit *drhd;
2659 struct dmar_rmrr_unit *rmrr;
832bd858 2660 struct device *dev;
ba395927 2661 struct intel_iommu *iommu;
9d783ba0 2662 int i, ret;
2c2e2c38 2663
ba395927
KA		 2664 /*
2665 * for each drhd
2666 * allocate root
2667 * initialize and program root entry to not present
2668 * endfor
2669 */
2670 for_each_drhd_unit(drhd) {
5e0d2a6f 2671 /*
2672 * lock not needed as this is only incremented in the single
2673 * threaded kernel __init code path all other access are read
2674 * only
2675 */
1b198bb0
MT		 2676 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2677 g_num_of_iommus++;
2678 continue;
2679 }
2680 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2681 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2682 }
2683
d9630fe9
WH		 2684 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2685 GFP_KERNEL);
2686 if (!g_iommus) {
2687 printk(KERN_ERR "Allocating global iommu array failed\n");
2688 ret = -ENOMEM;
2689 goto error;
2690 }
2691
80b20dd8 2692 deferred_flush = kzalloc(g_num_of_iommus *
2693 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2694 if (!deferred_flush) {
5e0d2a6f 2695 ret = -ENOMEM;
989d51fc 2696 goto free_g_iommus;
5e0d2a6f 2697 }
2698
7c919779 2699 for_each_active_iommu(iommu, drhd) {
d9630fe9 2700 g_iommus[iommu->seq_id] = iommu;
ba395927 2701
e61d98d8
SS		 2702 ret = iommu_init_domains(iommu);
2703 if (ret)
989d51fc 2704 goto free_iommu;
e61d98d8 2705
ba395927
KA		 2706 /*
2707 * TBD:
2708 * we could share the same root & context tables
25985edc 2709 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2710 */
2711 ret = iommu_alloc_root_entry(iommu);
2712 if (ret) {
2713 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
989d51fc 2714 goto free_iommu;
ba395927 2715 }
4ed0d3e6 2716 if (!ecap_pass_through(iommu->ecap))
19943b0e 2717 hw_pass_through = 0;
ba395927
KA		 2718 }
2719
1531a6a6
SS		 2720 /*
2721 * Start from the sane iommu hardware state.
2722 */
7c919779 2723 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2724 /*
2725 * If the queued invalidation is already initialized by us
2726 * (for example, while enabling interrupt-remapping) then
2727 * we got the things already rolling from a sane state.
2728 */
2729 if (iommu->qi)
2730 continue;
2731
2732 /*
2733 * Clear any previous faults.
2734 */
2735 dmar_fault(-1, iommu);
2736 /*
2737 * Disable queued invalidation if supported and already enabled
2738 * before OS handover.
2739 */
2740 dmar_disable_qi(iommu);
2741 }
2742
7c919779 2743 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2744 if (dmar_enable_qi(iommu)) {
2745 /*
2746 * Queued Invalidate not enabled, use Register Based
2747 * Invalidate
2748 */
2749 iommu->flush.flush_context = __iommu_flush_context;
2750 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2751 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2752 "invalidation\n",
680a7524 2753 iommu->seq_id,
b4e0f9eb 2754 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2755 } else {
2756 iommu->flush.flush_context = qi_flush_context;
2757 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2758 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2759 "invalidation\n",
680a7524 2760 iommu->seq_id,
b4e0f9eb 2761 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2762 }
2763 }
2764
19943b0e 2765 if (iommu_pass_through)
e0fc7e0b
DW		 2766 iommu_identity_mapping |= IDENTMAP_ALL;
2767
d3f13810 2768#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2769 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2770#endif
e0fc7e0b
DW		 2771
2772 check_tylersburg_isoch();
2773
ba395927 2774 /*
19943b0e
DW		 2775 * If pass through is not set or not enabled, setup context entries for
2776 * identity mappings for rmrr, gfx, and isa and may fall back to static
2777 * identity mapping if iommu_identity_mapping is set.
ba395927 2778 */
19943b0e
DW		 2779 if (iommu_identity_mapping) {
2780 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2781 if (ret) {
19943b0e 2782 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
989d51fc 2783 goto free_iommu;
ba395927
KA		 2784 }
2785 }
ba395927 2786 /*
19943b0e
DW		 2787 * For each rmrr
2788 * for each dev attached to rmrr
2789 * do
2790 * locate drhd for dev, alloc domain for dev
2791 * allocate free domain
2792 * allocate page table entries for rmrr
2793 * if context not allocated for bus
2794 * allocate and init context
2795 * set present in root table for this bus
2796 * init context with domain, translation etc
2797 * endfor
2798 * endfor
ba395927 2799 */
19943b0e
DW		 2800 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2801 for_each_rmrr_units(rmrr) {
b683b230
JL		 2802 /* some BIOS lists non-exist devices in DMAR table. */
2803 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 2804 i, dev) {
0b9d9753 2805 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e
DW		 2806 if (ret)
2807 printk(KERN_ERR
2808 "IOMMU: mapping reserved region failed\n");
ba395927 2809 }
4ed0d3e6 2810 }
49a0429e 2811
19943b0e
DW		 2812 iommu_prepare_isa();
2813
ba395927
KA		 2814 /*
2815 * for each drhd
2816 * enable fault log
2817 * global invalidate context cache
2818 * global invalidate iotlb
2819 * enable translation
2820 */
7c919779 2821 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2822 if (drhd->ignored) {
2823 /*
2824 * we always have to disable PMRs or DMA may fail on
2825 * this device
2826 */
2827 if (force_on)
7c919779 2828 iommu_disable_protect_mem_regions(iommu);
ba395927 2829 continue;
51a63e67 2830 }
ba395927
KA		 2831
2832 iommu_flush_write_buffer(iommu);
2833
3460a6d9
KA		 2834 ret = dmar_set_interrupt(iommu);
2835 if (ret)
989d51fc 2836 goto free_iommu;
3460a6d9 2837
ba395927
KA		 2838 iommu_set_root_entry(iommu);
2839
4c25a2c1 2840 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2841 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2842
ba395927
KA		 2843 ret = iommu_enable_translation(iommu);
2844 if (ret)
989d51fc 2845 goto free_iommu;
b94996c9
DW		 2846
2847 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2848 }
2849
2850 return 0;
989d51fc
JL		 2851
2852free_iommu:
7c919779 2853 for_each_active_iommu(iommu, drhd)
a868e6b7 2854 free_dmar_iommu(iommu);
9bdc531e 2855 kfree(deferred_flush);
989d51fc 2856free_g_iommus:
d9630fe9 2857 kfree(g_iommus);
989d51fc 2858error:
ba395927
KA		 2859 return ret;
2860}
2861
5a5e02a6 2862/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2863static struct iova *intel_alloc_iova(struct device *dev,
2864 struct dmar_domain *domain,
2865 unsigned long nrpages, uint64_t dma_mask)
ba395927 2866{
ba395927 2867 struct iova *iova = NULL;
ba395927 2868
875764de
DW		 2869 /* Restrict dma_mask to the width that the iommu can handle */
2870 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2871
2872 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2873 /*
2874 * First try to allocate an io virtual address in
284901a9 2875 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2876 * from higher range
ba395927 2877 */
875764de
DW		 2878 iova = alloc_iova(&domain->iovad, nrpages,
2879 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2880 if (iova)
2881 return iova;
2882 }
2883 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2884 if (unlikely(!iova)) {
2885 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
207e3592 2886 nrpages, dev_name(dev));
f76aec76
KA		 2887 return NULL;
2888 }
2889
2890 return iova;
2891}
2892
d4b709f4 2893static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 2894{
2895 struct dmar_domain *domain;
2896 int ret;
2897
d4b709f4 2898 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 2899 if (!domain) {
d4b709f4
DW		 2900 printk(KERN_ERR "Allocating domain for %s failed",
2901 dev_name(dev));
4fe05bbc 2902 return NULL;
ba395927
KA		 2903 }
2904
2905 /* make sure context mapping is ok */
d4b709f4
DW		 2906 if (unlikely(!domain_context_mapped(dev))) {
2907 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 2908 if (ret) {
d4b709f4
DW		 2909 printk(KERN_ERR "Domain context map for %s failed",
2910 dev_name(dev));
4fe05bbc 2911 return NULL;
f76aec76 2912 }
ba395927
KA		 2913 }
2914
f76aec76
KA		 2915 return domain;
2916}
2917
d4b709f4 2918static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 2919{
2920 struct device_domain_info *info;
2921
2922 /* No lock here, assumes no domain exit in normal case */
d4b709f4 2923 info = dev->archdata.iommu;
147202aa
DW		 2924 if (likely(info))
2925 return info->domain;
2926
2927 return __get_valid_domain_for_dev(dev);
2928}
2929
3d89194a 2930static int iommu_dummy(struct device *dev)
2c2e2c38 2931{
3d89194a 2932 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2c2e2c38
FY		 2933}
2934
ecb509ec 2935/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 2936static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 2937{
2938 int found;
2939
3d89194a 2940 if (iommu_dummy(dev))
1e4c64c4
DW		 2941 return 1;
2942
2c2e2c38 2943 if (!iommu_identity_mapping)
1e4c64c4 2944 return 0;
2c2e2c38 2945
9b226624 2946 found = identity_mapping(dev);
2c2e2c38 2947 if (found) {
ecb509ec 2948 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 2949 return 1;
2950 else {
2951 /*
2952 * 32 bit DMA is removed from si_domain and fall back
2953 * to non-identity mapping.
2954 */
bf9c9eda 2955 domain_remove_one_dev_info(si_domain, dev);
2c2e2c38 2956 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
ecb509ec 2957 dev_name(dev));
2c2e2c38
FY		 2958 return 0;
2959 }
2960 } else {
2961 /*
2962 * In case of a detached 64 bit DMA device from vm, the device
2963 * is put into si_domain for identity mapping.
2964 */
ecb509ec 2965 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 2966 int ret;
5913c9bf 2967 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 2968 hw_pass_through ?
2969 CONTEXT_TT_PASS_THROUGH :
2970 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2971 if (!ret) {
2972 printk(KERN_INFO "64bit %s uses identity mapping\n",
ecb509ec 2973 dev_name(dev));
2c2e2c38
FY		 2974 return 1;
2975 }
2976 }
2977 }
2978
1e4c64c4 2979 return 0;
2c2e2c38
FY		 2980}
2981
5040a918 2982static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 2983 size_t size, int dir, u64 dma_mask)
f76aec76 2984{
f76aec76 2985 struct dmar_domain *domain;
5b6985ce 2986 phys_addr_t start_paddr;
f76aec76
KA		 2987 struct iova *iova;
2988 int prot = 0;
6865f0d1 2989 int ret;
8c11e798 2990 struct intel_iommu *iommu;
33041ec0 2991 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2992
2993 BUG_ON(dir == DMA_NONE);
2c2e2c38 2994
5040a918 2995 if (iommu_no_mapping(dev))
6865f0d1 2996 return paddr;
f76aec76 2997
5040a918 2998 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 2999 if (!domain)
3000 return 0;
3001
8c11e798 3002 iommu = domain_get_iommu(domain);
88cb6a74 3003 size = aligned_nrpages(paddr, size);
f76aec76 3004
5040a918 3005 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3006 if (!iova)
3007 goto error;
3008
ba395927
KA		 3009 /*
3010 * Check if DMAR supports zero-length reads on write only
3011 * mappings..
3012 */
3013 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3014 !cap_zlr(iommu->cap))
ba395927
KA		 3015 prot |= DMA_PTE_READ;
3016 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3017 prot |= DMA_PTE_WRITE;
3018 /*
6865f0d1 3019 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3020 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3021 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3022 * is not a big problem
3023 */
0ab36de2 3024 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3025 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3026 if (ret)
3027 goto error;
3028
1f0ef2aa
DW		 3029 /* it's a non-present to present mapping. Only flush if caching mode */
3030 if (cap_caching_mode(iommu->cap))
ea8ea460 3031 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3032 else
8c11e798 3033 iommu_flush_write_buffer(iommu);
f76aec76 3034
03d6a246
DW		 3035 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3036 start_paddr += paddr & ~PAGE_MASK;
3037 return start_paddr;
ba395927 3038
ba395927 3039error:
f76aec76
KA		 3040 if (iova)
3041 __free_iova(&domain->iovad, iova);
4cf2e75d 3042 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3043 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3044 return 0;
3045}
3046
ffbbef5c
FT		 3047static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3048 unsigned long offset, size_t size,
3049 enum dma_data_direction dir,
3050 struct dma_attrs *attrs)
bb9e6d65 3051{
ffbbef5c 3052 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3053 dir, *dev->dma_mask);
bb9e6d65
FT		 3054}
3055
5e0d2a6f 3056static void flush_unmaps(void)
3057{
80b20dd8 3058 int i, j;
5e0d2a6f 3059
5e0d2a6f 3060 timer_on = 0;
3061
3062 /* just flush them all */
3063 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3064 struct intel_iommu *iommu = g_iommus[i];
3065 if (!iommu)
3066 continue;
c42d9f32 3067
9dd2fe89
YZ		 3068 if (!deferred_flush[i].next)
3069 continue;
3070
78d5f0f5
NA		 3071 /* In caching mode, global flushes turn emulation expensive */
3072 if (!cap_caching_mode(iommu->cap))
3073 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3074 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3075 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3076 unsigned long mask;
3077 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3078 struct dmar_domain *domain = deferred_flush[i].domain[j];
3079
3080 /* On real hardware multiple invalidations are expensive */
3081 if (cap_caching_mode(iommu->cap))
3082 iommu_flush_iotlb_psi(iommu, domain->id,
ea8ea460
DW		 3083 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3084 !deferred_flush[i].freelist[j], 0);
78d5f0f5
NA		 3085 else {
3086 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3087 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3088 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3089 }
93a23a72 3090 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3091 if (deferred_flush[i].freelist[j])
3092 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3093 }
9dd2fe89 3094 deferred_flush[i].next = 0;
5e0d2a6f 3095 }
3096
5e0d2a6f 3097 list_size = 0;
5e0d2a6f 3098}
3099
3100static void flush_unmaps_timeout(unsigned long data)
3101{
80b20dd8 3102 unsigned long flags;
3103
3104 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3105 flush_unmaps();
80b20dd8 3106 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3107}
3108
ea8ea460 3109static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3110{
3111 unsigned long flags;
80b20dd8 3112 int next, iommu_id;
8c11e798 3113 struct intel_iommu *iommu;
5e0d2a6f 3114
3115 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3116 if (list_size == HIGH_WATER_MARK)
3117 flush_unmaps();
3118
8c11e798
WH		 3119 iommu = domain_get_iommu(dom);
3120 iommu_id = iommu->seq_id;
c42d9f32 3121
80b20dd8 3122 next = deferred_flush[iommu_id].next;
3123 deferred_flush[iommu_id].domain[next] = dom;
3124 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3125 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3126 deferred_flush[iommu_id].next++;
5e0d2a6f 3127
3128 if (!timer_on) {
3129 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3130 timer_on = 1;
3131 }
3132 list_size++;
3133 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3134}
3135
ffbbef5c
FT		 3136static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3137 size_t size, enum dma_data_direction dir,
3138 struct dma_attrs *attrs)
ba395927 3139{
f76aec76 3140 struct dmar_domain *domain;
d794dc9b 3141 unsigned long start_pfn, last_pfn;
ba395927 3142 struct iova *iova;
8c11e798 3143 struct intel_iommu *iommu;
ea8ea460 3144 struct page *freelist;
ba395927 3145
73676832 3146 if (iommu_no_mapping(dev))
f76aec76 3147 return;
2c2e2c38 3148
1525a29a 3149 domain = find_domain(dev);
ba395927
KA		 3150 BUG_ON(!domain);
3151
8c11e798
WH		 3152 iommu = domain_get_iommu(domain);
3153
ba395927 3154 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3155 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3156 (unsigned long long)dev_addr))
ba395927 3157 return;
ba395927 3158
d794dc9b
DW		 3159 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3160 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3161
d794dc9b 3162 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3163 dev_name(dev), start_pfn, last_pfn);
ba395927 3164
ea8ea460 3165 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3166
5e0d2a6f 3167 if (intel_iommu_strict) {
03d6a246 3168 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3169 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3170 /* free iova */
3171 __free_iova(&domain->iovad, iova);
ea8ea460 3172 dma_free_pagelist(freelist);
5e0d2a6f 3173 } else {
ea8ea460 3174 add_unmap(domain, iova, freelist);
5e0d2a6f 3175 /*
3176 * queue up the release of the unmap to save the 1/6th of the
3177 * cpu used up by the iotlb flush operation...
3178 */
5e0d2a6f 3179 }
ba395927
KA		 3180}
3181
5040a918 3182static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3183 dma_addr_t *dma_handle, gfp_t flags,
3184 struct dma_attrs *attrs)
ba395927 3185{
36746436 3186 struct page *page = NULL;
ba395927
KA		 3187 int order;
3188
5b6985ce 3189 size = PAGE_ALIGN(size);
ba395927 3190 order = get_order(size);
e8bb910d 3191
5040a918 3192 if (!iommu_no_mapping(dev))
e8bb910d 3193 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3194 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3195 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3196 flags |= GFP_DMA;
3197 else
3198 flags |= GFP_DMA32;
3199 }
ba395927 3200
36746436
AM		 3201 if (flags & __GFP_WAIT) {
3202 unsigned int count = size >> PAGE_SHIFT;
3203
3204 page = dma_alloc_from_contiguous(dev, count, order);
3205 if (page && iommu_no_mapping(dev) &&
3206 page_to_phys(page) + size > dev->coherent_dma_mask) {
3207 dma_release_from_contiguous(dev, page, count);
3208 page = NULL;
3209 }
3210 }
3211
3212 if (!page)
3213 page = alloc_pages(flags, order);
3214 if (!page)
ba395927 3215 return NULL;
36746436 3216 memset(page_address(page), 0, size);
ba395927 3217
36746436 3218 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3219 DMA_BIDIRECTIONAL,
5040a918 3220 dev->coherent_dma_mask);
ba395927 3221 if (*dma_handle)
36746436
AM		 3222 return page_address(page);
3223 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3224 __free_pages(page, order);
3225
ba395927
KA		 3226 return NULL;
3227}
3228
5040a918 3229static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3230 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3231{
3232 int order;
36746436 3233 struct page *page = virt_to_page(vaddr);
ba395927 3234
5b6985ce 3235 size = PAGE_ALIGN(size);
ba395927
KA		 3236 order = get_order(size);
3237
5040a918 3238 intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
36746436
AM		 3239 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3240 __free_pages(page, order);
ba395927
KA		 3241}
3242
5040a918 3243static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3244 int nelems, enum dma_data_direction dir,
3245 struct dma_attrs *attrs)
ba395927 3246{
ba395927 3247 struct dmar_domain *domain;
d794dc9b 3248 unsigned long start_pfn, last_pfn;
f76aec76 3249 struct iova *iova;
8c11e798 3250 struct intel_iommu *iommu;
ea8ea460 3251 struct page *freelist;
ba395927 3252
5040a918 3253 if (iommu_no_mapping(dev))
ba395927
KA		 3254 return;
3255
5040a918 3256 domain = find_domain(dev);
8c11e798
WH		 3257 BUG_ON(!domain);
3258
3259 iommu = domain_get_iommu(domain);
ba395927 3260
c03ab37c 3261 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3262 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3263 (unsigned long long)sglist[0].dma_address))
f76aec76 3264 return;
f76aec76 3265
d794dc9b
DW		 3266 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3267 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76 3268
ea8ea460 3269 freelist = domain_unmap(domain, start_pfn, last_pfn);
f76aec76 3270
acea0018
DW		 3271 if (intel_iommu_strict) {
3272 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3273 last_pfn - start_pfn + 1, !freelist, 0);
acea0018
DW		 3274 /* free iova */
3275 __free_iova(&domain->iovad, iova);
ea8ea460 3276 dma_free_pagelist(freelist);
acea0018 3277 } else {
ea8ea460 3278 add_unmap(domain, iova, freelist);
acea0018
DW		 3279 /*
3280 * queue up the release of the unmap to save the 1/6th of the
3281 * cpu used up by the iotlb flush operation...
3282 */
3283 }
ba395927
KA		 3284}
3285
ba395927 3286static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3287 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3288{
3289 int i;
c03ab37c 3290 struct scatterlist *sg;
ba395927 3291
c03ab37c 3292 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3293 BUG_ON(!sg_page(sg));
4cf2e75d 3294 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3295 sg->dma_length = sg->length;
ba395927
KA		 3296 }
3297 return nelems;
3298}
3299
5040a918 3300static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3301 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3302{
ba395927 3303 int i;
ba395927 3304 struct dmar_domain *domain;
f76aec76
KA		 3305 size_t size = 0;
3306 int prot = 0;
f76aec76
KA		 3307 struct iova *iova = NULL;
3308 int ret;
c03ab37c 3309 struct scatterlist *sg;
b536d24d 3310 unsigned long start_vpfn;
8c11e798 3311 struct intel_iommu *iommu;
ba395927
KA		 3312
3313 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3314 if (iommu_no_mapping(dev))
3315 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3316
5040a918 3317 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3318 if (!domain)
3319 return 0;
3320
8c11e798
WH		 3321 iommu = domain_get_iommu(domain);
3322
b536d24d 3323 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3324 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3325
5040a918
DW		 3326 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3327 *dev->dma_mask);
f76aec76 3328 if (!iova) {
c03ab37c 3329 sglist->dma_length = 0;
f76aec76
KA		 3330 return 0;
3331 }
3332
3333 /*
3334 * Check if DMAR supports zero-length reads on write only
3335 * mappings..
3336 */
3337 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3338 !cap_zlr(iommu->cap))
f76aec76
KA		 3339 prot |= DMA_PTE_READ;
3340 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3341 prot |= DMA_PTE_WRITE;
3342
b536d24d 3343 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3344
f532959b 3345 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3346 if (unlikely(ret)) {
3347 /* clear the page */
3348 dma_pte_clear_range(domain, start_vpfn,
3349 start_vpfn + size - 1);
3350 /* free page tables */
3351 dma_pte_free_pagetable(domain, start_vpfn,
3352 start_vpfn + size - 1);
3353 /* free iova */
3354 __free_iova(&domain->iovad, iova);
3355 return 0;
ba395927
KA		 3356 }
3357
1f0ef2aa
DW		 3358 /* it's a non-present to present mapping. Only flush if caching mode */
3359 if (cap_caching_mode(iommu->cap))
ea8ea460 3360 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3361 else
8c11e798 3362 iommu_flush_write_buffer(iommu);
1f0ef2aa 3363
ba395927
KA		 3364 return nelems;
3365}
3366
dfb805e8
FT		 3367static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3368{
3369 return !dma_addr;
3370}
3371
160c1d8e 3372struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3373 .alloc = intel_alloc_coherent,
3374 .free = intel_free_coherent,
ba395927
KA		 3375 .map_sg = intel_map_sg,
3376 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3377 .map_page = intel_map_page,
3378 .unmap_page = intel_unmap_page,
dfb805e8 3379 .mapping_error = intel_mapping_error,
ba395927
KA		 3380};
3381
3382static inline int iommu_domain_cache_init(void)
3383{
3384 int ret = 0;
3385
3386 iommu_domain_cache = kmem_cache_create("iommu_domain",
3387 sizeof(struct dmar_domain),
3388 0,
3389 SLAB_HWCACHE_ALIGN,
3390
3391 NULL);
3392 if (!iommu_domain_cache) {
3393 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3394 ret = -ENOMEM;
3395 }
3396
3397 return ret;
3398}
3399
3400static inline int iommu_devinfo_cache_init(void)
3401{
3402 int ret = 0;
3403
3404 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3405 sizeof(struct device_domain_info),
3406 0,
3407 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3408 NULL);
3409 if (!iommu_devinfo_cache) {
3410 printk(KERN_ERR "Couldn't create devinfo cache\n");
3411 ret = -ENOMEM;
3412 }
3413
3414 return ret;
3415}
3416
3417static inline int iommu_iova_cache_init(void)
3418{
3419 int ret = 0;
3420
3421 iommu_iova_cache = kmem_cache_create("iommu_iova",
3422 sizeof(struct iova),
3423 0,
3424 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3425 NULL);
3426 if (!iommu_iova_cache) {
3427 printk(KERN_ERR "Couldn't create iova cache\n");
3428 ret = -ENOMEM;
3429 }
3430
3431 return ret;
3432}
3433
3434static int __init iommu_init_mempool(void)
3435{
3436 int ret;
3437 ret = iommu_iova_cache_init();
3438 if (ret)
3439 return ret;
3440
3441 ret = iommu_domain_cache_init();
3442 if (ret)
3443 goto domain_error;
3444
3445 ret = iommu_devinfo_cache_init();
3446 if (!ret)
3447 return ret;
3448
3449 kmem_cache_destroy(iommu_domain_cache);
3450domain_error:
3451 kmem_cache_destroy(iommu_iova_cache);
3452
3453 return -ENOMEM;
3454}
3455
3456static void __init iommu_exit_mempool(void)
3457{
3458 kmem_cache_destroy(iommu_devinfo_cache);
3459 kmem_cache_destroy(iommu_domain_cache);
3460 kmem_cache_destroy(iommu_iova_cache);
3461
3462}
3463
556ab45f
DW		 3464static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3465{
3466 struct dmar_drhd_unit *drhd;
3467 u32 vtbar;
3468 int rc;
3469
3470 /* We know that this device on this chipset has its own IOMMU.
3471 * If we find it under a different IOMMU, then the BIOS is lying
3472 * to us. Hope that the IOMMU for this device is actually
3473 * disabled, and it needs no translation...
3474 */
3475 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3476 if (rc) {
3477 /* "can't" happen */
3478 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3479 return;
3480 }
3481 vtbar &= 0xffff0000;
3482
3483 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3484 drhd = dmar_find_matched_drhd_unit(pdev);
3485 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3486 TAINT_FIRMWARE_WORKAROUND,
3487 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3488 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3489}
3490DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3491
ba395927
KA		 3492static void __init init_no_remapping_devices(void)
3493{
3494 struct dmar_drhd_unit *drhd;
832bd858 3495 struct device *dev;
b683b230 3496 int i;
ba395927
KA		 3497
3498 for_each_drhd_unit(drhd) {
3499 if (!drhd->include_all) {
b683b230
JL		 3500 for_each_active_dev_scope(drhd->devices,
3501 drhd->devices_cnt, i, dev)
3502 break;
832bd858 3503 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3504 if (i == drhd->devices_cnt)
3505 drhd->ignored = 1;
3506 }
3507 }
3508
7c919779 3509 for_each_active_drhd_unit(drhd) {
7c919779 3510 if (drhd->include_all)
ba395927
KA		 3511 continue;
3512
b683b230
JL		 3513 for_each_active_dev_scope(drhd->devices,
3514 drhd->devices_cnt, i, dev)
832bd858 3515 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3516 break;
ba395927
KA		 3517 if (i < drhd->devices_cnt)
3518 continue;
3519
c0771df8
DW		 3520 /* This IOMMU has *only* gfx devices. Either bypass it or
3521 set the gfx_mapped flag, as appropriate */
3522 if (dmar_map_gfx) {
3523 intel_iommu_gfx_mapped = 1;
3524 } else {
3525 drhd->ignored = 1;
b683b230
JL		 3526 for_each_active_dev_scope(drhd->devices,
3527 drhd->devices_cnt, i, dev)
832bd858 3528 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3529 }
3530 }
3531}
3532
f59c7b69
FY		 3533#ifdef CONFIG_SUSPEND
3534static int init_iommu_hw(void)
3535{
3536 struct dmar_drhd_unit *drhd;
3537 struct intel_iommu *iommu = NULL;
3538
3539 for_each_active_iommu(iommu, drhd)
3540 if (iommu->qi)
3541 dmar_reenable_qi(iommu);
3542
b779260b
JC		 3543 for_each_iommu(iommu, drhd) {
3544 if (drhd->ignored) {
3545 /*
3546 * we always have to disable PMRs or DMA may fail on
3547 * this device
3548 */
3549 if (force_on)
3550 iommu_disable_protect_mem_regions(iommu);
3551 continue;
3552 }
3553
f59c7b69
FY		 3554 iommu_flush_write_buffer(iommu);
3555
3556 iommu_set_root_entry(iommu);
3557
3558 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3559 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3560 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3561 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3562 if (iommu_enable_translation(iommu))
3563 return 1;
b94996c9 3564 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3565 }
3566
3567 return 0;
3568}
3569
3570static void iommu_flush_all(void)
3571{
3572 struct dmar_drhd_unit *drhd;
3573 struct intel_iommu *iommu;
3574
3575 for_each_active_iommu(iommu, drhd) {
3576 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3577 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3578 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3579 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3580 }
3581}
3582
134fac3f 3583static int iommu_suspend(void)
f59c7b69
FY		 3584{
3585 struct dmar_drhd_unit *drhd;
3586 struct intel_iommu *iommu = NULL;
3587 unsigned long flag;
3588
3589 for_each_active_iommu(iommu, drhd) {
3590 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3591 GFP_ATOMIC);
3592 if (!iommu->iommu_state)
3593 goto nomem;
3594 }
3595
3596 iommu_flush_all();
3597
3598 for_each_active_iommu(iommu, drhd) {
3599 iommu_disable_translation(iommu);
3600
1f5b3c3f 3601 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3602
3603 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3604 readl(iommu->reg + DMAR_FECTL_REG);
3605 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3606 readl(iommu->reg + DMAR_FEDATA_REG);
3607 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3608 readl(iommu->reg + DMAR_FEADDR_REG);
3609 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3610 readl(iommu->reg + DMAR_FEUADDR_REG);
3611
1f5b3c3f 3612 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3613 }
3614 return 0;
3615
3616nomem:
3617 for_each_active_iommu(iommu, drhd)
3618 kfree(iommu->iommu_state);
3619
3620 return -ENOMEM;
3621}
3622
134fac3f 3623static void iommu_resume(void)
f59c7b69
FY		 3624{
3625 struct dmar_drhd_unit *drhd;
3626 struct intel_iommu *iommu = NULL;
3627 unsigned long flag;
3628
3629 if (init_iommu_hw()) {
b779260b
JC		 3630 if (force_on)
3631 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3632 else
3633 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3634 return;
f59c7b69
FY		 3635 }
3636
3637 for_each_active_iommu(iommu, drhd) {
3638
1f5b3c3f 3639 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3640
3641 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3642 iommu->reg + DMAR_FECTL_REG);
3643 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3644 iommu->reg + DMAR_FEDATA_REG);
3645 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3646 iommu->reg + DMAR_FEADDR_REG);
3647 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3648 iommu->reg + DMAR_FEUADDR_REG);
3649
1f5b3c3f 3650 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3651 }
3652
3653 for_each_active_iommu(iommu, drhd)
3654 kfree(iommu->iommu_state);
f59c7b69
FY		 3655}
3656
134fac3f 3657static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3658 .resume = iommu_resume,
3659 .suspend = iommu_suspend,
3660};
3661
134fac3f 3662static void __init init_iommu_pm_ops(void)
f59c7b69 3663{
134fac3f 3664 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3665}
3666
3667#else
99592ba4 3668static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3669#endif /* CONFIG_PM */
3670
318fe7df
SS		 3671
3672int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3673{
3674 struct acpi_dmar_reserved_memory *rmrr;
3675 struct dmar_rmrr_unit *rmrru;
3676
3677 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3678 if (!rmrru)
3679 return -ENOMEM;
3680
3681 rmrru->hdr = header;
3682 rmrr = (struct acpi_dmar_reserved_memory *)header;
3683 rmrru->base_address = rmrr->base_address;
3684 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3685 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3686 ((void *)rmrr) + rmrr->header.length,
3687 &rmrru->devices_cnt);
3688 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3689 kfree(rmrru);
3690 return -ENOMEM;
3691 }
318fe7df 3692
2e455289 3693 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3694
2e455289 3695 return 0;
318fe7df
SS		 3696}
3697
318fe7df
SS		 3698int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3699{
3700 struct acpi_dmar_atsr *atsr;
3701 struct dmar_atsr_unit *atsru;
3702
3703 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3704 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3705 if (!atsru)
3706 return -ENOMEM;
3707
3708 atsru->hdr = hdr;
3709 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3710 if (!atsru->include_all) {
3711 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3712 (void *)atsr + atsr->header.length,
3713 &atsru->devices_cnt);
3714 if (atsru->devices_cnt && atsru->devices == NULL) {
3715 kfree(atsru);
3716 return -ENOMEM;
3717 }
3718 }
318fe7df 3719
0e242612 3720 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3721
3722 return 0;
3723}
3724
9bdc531e
JL		 3725static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3726{
3727 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3728 kfree(atsru);
3729}
3730
3731static void intel_iommu_free_dmars(void)
3732{
3733 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3734 struct dmar_atsr_unit *atsru, *atsr_n;
3735
3736 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3737 list_del(&rmrru->list);
3738 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3739 kfree(rmrru);
318fe7df
SS		 3740 }
3741
9bdc531e
JL		 3742 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3743 list_del(&atsru->list);
3744 intel_iommu_free_atsr(atsru);
3745 }
318fe7df
SS		 3746}
3747
3748int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3749{
b683b230 3750 int i, ret = 1;
318fe7df 3751 struct pci_bus *bus;
832bd858
DW		 3752 struct pci_dev *bridge = NULL;
3753 struct device *tmp;
318fe7df
SS		 3754 struct acpi_dmar_atsr *atsr;
3755 struct dmar_atsr_unit *atsru;
3756
3757 dev = pci_physfn(dev);
318fe7df 3758 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3759 bridge = bus->self;
318fe7df 3760 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3761 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3762 return 0;
b5f82ddf 3763 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3764 break;
318fe7df 3765 }
b5f82ddf
JL		 3766 if (!bridge)
3767 return 0;
318fe7df 3768
0e242612 3769 rcu_read_lock();
b5f82ddf
JL		 3770 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3771 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3772 if (atsr->segment != pci_domain_nr(dev->bus))
3773 continue;
3774
b683b230 3775 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 3776 if (tmp == &bridge->dev)
b683b230 3777 goto out;
b5f82ddf
JL		 3778
3779 if (atsru->include_all)
b683b230 3780 goto out;
b5f82ddf 3781 }
b683b230
JL		 3782 ret = 0;
3783out:
0e242612 3784 rcu_read_unlock();
318fe7df 3785
b683b230 3786 return ret;
318fe7df
SS		 3787}
3788
59ce0515
JL		 3789int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3790{
3791 int ret = 0;
3792 struct dmar_rmrr_unit *rmrru;
3793 struct dmar_atsr_unit *atsru;
3794 struct acpi_dmar_atsr *atsr;
3795 struct acpi_dmar_reserved_memory *rmrr;
3796
3797 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3798 return 0;
3799
3800 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3801 rmrr = container_of(rmrru->hdr,
3802 struct acpi_dmar_reserved_memory, header);
3803 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3804 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3805 ((void *)rmrr) + rmrr->header.length,
3806 rmrr->segment, rmrru->devices,
3807 rmrru->devices_cnt);
27e24950 3808 if(ret < 0)
59ce0515
JL		 3809 return ret;
3810 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 3811 dmar_remove_dev_scope(info, rmrr->segment,
3812 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 3813 }
3814 }
3815
3816 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3817 if (atsru->include_all)
3818 continue;
3819
3820 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3821 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3822 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3823 (void *)atsr + atsr->header.length,
3824 atsr->segment, atsru->devices,
3825 atsru->devices_cnt);
3826 if (ret > 0)
3827 break;
3828 else if(ret < 0)
3829 return ret;
3830 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3831 if (dmar_remove_dev_scope(info, atsr->segment,
3832 atsru->devices, atsru->devices_cnt))
3833 break;
3834 }
3835 }
3836
3837 return 0;
3838}
3839
99dcaded
FY		 3840/*
3841 * Here we only respond to action of unbound device from driver.
3842 *
3843 * Added device is not attached to its DMAR domain here yet. That will happen
3844 * when mapping the device to iova.
3845 */
3846static int device_notifier(struct notifier_block *nb,
3847 unsigned long action, void *data)
3848{
3849 struct device *dev = data;
99dcaded
FY		 3850 struct dmar_domain *domain;
3851
3d89194a 3852 if (iommu_dummy(dev))
44cd613c
DW		 3853 return 0;
3854
7e7dfab7
JL		 3855 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3856 action != BUS_NOTIFY_DEL_DEVICE)
3857 return 0;
3858
1525a29a 3859 domain = find_domain(dev);
99dcaded
FY		 3860 if (!domain)
3861 return 0;
3862
3a5670e8 3863 down_read(&dmar_global_lock);
bf9c9eda 3864 domain_remove_one_dev_info(domain, dev);
7e7dfab7
JL		 3865 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3866 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3867 list_empty(&domain->devices))
3868 domain_exit(domain);
3a5670e8 3869 up_read(&dmar_global_lock);
a97590e5 3870
99dcaded
FY		 3871 return 0;
3872}
3873
3874static struct notifier_block device_nb = {
3875 .notifier_call = device_notifier,
3876};
3877
75f05569
JL		 3878static int intel_iommu_memory_notifier(struct notifier_block *nb,
3879 unsigned long val, void *v)
3880{
3881 struct memory_notify *mhp = v;
3882 unsigned long long start, end;
3883 unsigned long start_vpfn, last_vpfn;
3884
3885 switch (val) {
3886 case MEM_GOING_ONLINE:
3887 start = mhp->start_pfn << PAGE_SHIFT;
3888 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3889 if (iommu_domain_identity_map(si_domain, start, end)) {
3890 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3891 start, end);
3892 return NOTIFY_BAD;
3893 }
3894 break;
3895
3896 case MEM_OFFLINE:
3897 case MEM_CANCEL_ONLINE:
3898 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3899 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3900 while (start_vpfn <= last_vpfn) {
3901 struct iova *iova;
3902 struct dmar_drhd_unit *drhd;
3903 struct intel_iommu *iommu;
ea8ea460 3904 struct page *freelist;
75f05569
JL		 3905
3906 iova = find_iova(&si_domain->iovad, start_vpfn);
3907 if (iova == NULL) {
3908 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3909 start_vpfn);
3910 break;
3911 }
3912
3913 iova = split_and_remove_iova(&si_domain->iovad, iova,
3914 start_vpfn, last_vpfn);
3915 if (iova == NULL) {
3916 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3917 start_vpfn, last_vpfn);
3918 return NOTIFY_BAD;
3919 }
3920
ea8ea460
DW		 3921 freelist = domain_unmap(si_domain, iova->pfn_lo,
3922 iova->pfn_hi);
3923
75f05569
JL		 3924 rcu_read_lock();
3925 for_each_active_iommu(iommu, drhd)
3926 iommu_flush_iotlb_psi(iommu, si_domain->id,
3927 iova->pfn_lo,
ea8ea460
DW		 3928 iova->pfn_hi - iova->pfn_lo + 1,
3929 !freelist, 0);
75f05569 3930 rcu_read_unlock();
ea8ea460 3931 dma_free_pagelist(freelist);
75f05569
JL		 3932
3933 start_vpfn = iova->pfn_hi + 1;
3934 free_iova_mem(iova);
3935 }
3936 break;
3937 }
3938
3939 return NOTIFY_OK;
3940}
3941
3942static struct notifier_block intel_iommu_memory_nb = {
3943 .notifier_call = intel_iommu_memory_notifier,
3944 .priority = 0
3945};
3946
ba395927
KA		 3947int __init intel_iommu_init(void)
3948{
9bdc531e 3949 int ret = -ENODEV;
3a93c841 3950 struct dmar_drhd_unit *drhd;
7c919779 3951 struct intel_iommu *iommu;
ba395927 3952
a59b50e9
JC		 3953 /* VT-d is required for a TXT/tboot launch, so enforce that */
3954 force_on = tboot_force_iommu();
3955
3a5670e8
JL		 3956 if (iommu_init_mempool()) {
3957 if (force_on)
3958 panic("tboot: Failed to initialize iommu memory\n");
3959 return -ENOMEM;
3960 }
3961
3962 down_write(&dmar_global_lock);
a59b50e9
JC		 3963 if (dmar_table_init()) {
3964 if (force_on)
3965 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 3966 goto out_free_dmar;
a59b50e9 3967 }
ba395927 3968
3a93c841
TI		 3969 /*
3970 * Disable translation if already enabled prior to OS handover.
3971 */
7c919779 3972 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 3973 if (iommu->gcmd & DMA_GCMD_TE)
3974 iommu_disable_translation(iommu);
3a93c841 3975
c2c7286a 3976 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3977 if (force_on)
3978 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 3979 goto out_free_dmar;
a59b50e9 3980 }
1886e8a9 3981
75f1cdf1 3982 if (no_iommu || dmar_disabled)
9bdc531e 3983 goto out_free_dmar;
2ae21010 3984
318fe7df
SS		 3985 if (list_empty(&dmar_rmrr_units))
3986 printk(KERN_INFO "DMAR: No RMRR found\n");
3987
3988 if (list_empty(&dmar_atsr_units))
3989 printk(KERN_INFO "DMAR: No ATSR found\n");
3990
51a63e67
JC		 3991 if (dmar_init_reserved_ranges()) {
3992 if (force_on)
3993 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 3994 goto out_free_reserved_range;
51a63e67 3995 }
ba395927
KA		 3996
3997 init_no_remapping_devices();
3998
b779260b 3999 ret = init_dmars();
ba395927 4000 if (ret) {
a59b50e9
JC		 4001 if (force_on)
4002 panic("tboot: Failed to initialize DMARs\n");
ba395927 4003 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 4004 goto out_free_reserved_range;
ba395927 4005 }
3a5670e8 4006 up_write(&dmar_global_lock);
ba395927
KA		 4007 printk(KERN_INFO
4008 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4009
5e0d2a6f 4010 init_timer(&unmap_timer);
75f1cdf1
FT		 4011#ifdef CONFIG_SWIOTLB
4012 swiotlb = 0;
4013#endif
19943b0e 4014 dma_ops = &intel_dma_ops;
4ed0d3e6 4015
134fac3f 4016 init_iommu_pm_ops();
a8bcbb0d 4017
4236d97d 4018 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4019 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4020 if (si_domain && !hw_pass_through)
4021 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4022
8bc1f85c
ED		 4023 intel_iommu_enabled = 1;
4024
ba395927 4025 return 0;
9bdc531e
JL		 4026
4027out_free_reserved_range:
4028 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4029out_free_dmar:
4030 intel_iommu_free_dmars();
3a5670e8
JL		 4031 up_write(&dmar_global_lock);
4032 iommu_exit_mempool();
9bdc531e 4033 return ret;
ba395927 4034}
e820482c 4035
579305f7
AW		 4036static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4037{
4038 struct intel_iommu *iommu = opaque;
4039
4040 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4041 return 0;
4042}
4043
4044/*
4045 * NB - intel-iommu lacks any sort of reference counting for the users of
4046 * dependent devices. If multiple endpoints have intersecting dependent
4047 * devices, unbinding the driver from any one of them will possibly leave
4048 * the others unable to operate.
4049 */
3199aa6b 4050static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4051 struct device *dev)
3199aa6b 4052{
0bcb3e28 4053 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4054 return;
4055
579305f7 4056 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4057}
4058
2c2e2c38 4059static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4060 struct device *dev)
c7151a8d 4061{
bca2b916 4062 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4063 struct intel_iommu *iommu;
4064 unsigned long flags;
4065 int found = 0;
156baca8 4066 u8 bus, devfn;
c7151a8d 4067
bf9c9eda 4068 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4069 if (!iommu)
4070 return;
4071
4072 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4073 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4074 if (info->iommu == iommu && info->bus == bus &&
4075 info->devfn == devfn) {
109b9b04 4076 unlink_domain_info(info);
c7151a8d
WH		 4077 spin_unlock_irqrestore(&device_domain_lock, flags);
4078
93a23a72 4079 iommu_disable_dev_iotlb(info);
c7151a8d 4080 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4081 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4082 free_devinfo_mem(info);
4083
4084 spin_lock_irqsave(&device_domain_lock, flags);
4085
4086 if (found)
4087 break;
4088 else
4089 continue;
4090 }
4091
4092 /* if there is no other devices under the same iommu
4093 * owned by this domain, clear this iommu in iommu_bmp
4094 * update iommu count and coherency
4095 */
8bbc4410 4096 if (info->iommu == iommu)
c7151a8d
WH		 4097 found = 1;
4098 }
4099
3e7abe25
RD		 4100 spin_unlock_irqrestore(&device_domain_lock, flags);
4101
c7151a8d
WH		 4102 if (found == 0) {
4103 unsigned long tmp_flags;
4104 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 4105 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 4106 domain->iommu_count--;
58c610bd 4107 domain_update_iommu_cap(domain);
c7151a8d 4108 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 4109
9b4554b2
AW		 4110 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4111 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4112 spin_lock_irqsave(&iommu->lock, tmp_flags);
4113 clear_bit(domain->id, iommu->domain_ids);
4114 iommu->domains[domain->id] = NULL;
4115 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4116 }
c7151a8d 4117 }
c7151a8d
WH		 4118}
4119
2c2e2c38 4120static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4121{
4122 int adjust_width;
4123
4124 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 4125 domain_reserve_special_ranges(domain);
4126
4127 /* calculate AGAW */
4128 domain->gaw = guest_width;
4129 adjust_width = guestwidth_to_adjustwidth(guest_width);
4130 domain->agaw = width_to_agaw(adjust_width);
4131
5e98c4b1 4132 domain->iommu_coherency = 0;
c5b15255 4133 domain->iommu_snooping = 0;
6dd9a7c7 4134 domain->iommu_superpage = 0;
fe40f1e0 4135 domain->max_addr = 0;
4c923d47 4136 domain->nid = -1;
5e98c4b1
WH		 4137
4138 /* always allocate the top pgd */
4c923d47 4139 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4140 if (!domain->pgd)
4141 return -ENOMEM;
4142 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4143 return 0;
4144}
4145
5d450806 4146static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 4147{
5d450806 4148 struct dmar_domain *dmar_domain;
38717946 4149
92d03cc8 4150 dmar_domain = alloc_domain(true);
5d450806 4151 if (!dmar_domain) {
38717946 4152 printk(KERN_ERR
5d450806
JR		 4153 "intel_iommu_domain_init: dmar_domain == NULL\n");
4154 return -ENOMEM;
38717946 4155 }
2c2e2c38 4156 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 4157 printk(KERN_ERR
5d450806 4158 "intel_iommu_domain_init() failed\n");
92d03cc8 4159 domain_exit(dmar_domain);
5d450806 4160 return -ENOMEM;
38717946 4161 }
8140a95d 4162 domain_update_iommu_cap(dmar_domain);
5d450806 4163 domain->priv = dmar_domain;
faa3d6f5 4164
8a0e715b
JR		 4165 domain->geometry.aperture_start = 0;
4166 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4167 domain->geometry.force_aperture = true;
4168
5d450806 4169 return 0;
38717946 4170}
38717946 4171
5d450806 4172static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4173{
5d450806
JR		 4174 struct dmar_domain *dmar_domain = domain->priv;
4175
4176 domain->priv = NULL;
92d03cc8 4177 domain_exit(dmar_domain);
38717946 4178}
38717946 4179
4c5478c9
JR		 4180static int intel_iommu_attach_device(struct iommu_domain *domain,
4181 struct device *dev)
38717946 4182{
4c5478c9 4183 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 4184 struct intel_iommu *iommu;
4185 int addr_width;
156baca8 4186 u8 bus, devfn;
faa3d6f5 4187
7207d8f9
DW		 4188 /* normally dev is not mapped */
4189 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4190 struct dmar_domain *old_domain;
4191
1525a29a 4192 old_domain = find_domain(dev);
faa3d6f5 4193 if (old_domain) {
2c2e2c38
FY		 4194 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4195 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
bf9c9eda 4196 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4197 else
4198 domain_remove_dev_info(old_domain);
4199 }
4200 }
4201
156baca8 4202 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4203 if (!iommu)
4204 return -ENODEV;
4205
4206 /* check if this iommu agaw is sufficient for max mapped address */
4207 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4208 if (addr_width > cap_mgaw(iommu->cap))
4209 addr_width = cap_mgaw(iommu->cap);
4210
4211 if (dmar_domain->max_addr > (1LL << addr_width)) {
4212 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4213 "sufficient for the mapped address (%llx)\n",
a99c47a2 4214 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4215 return -EFAULT;
4216 }
a99c47a2
TL		 4217 dmar_domain->gaw = addr_width;
4218
4219 /*
4220 * Knock out extra levels of page tables if necessary
4221 */
4222 while (iommu->agaw < dmar_domain->agaw) {
4223 struct dma_pte *pte;
4224
4225 pte = dmar_domain->pgd;
4226 if (dma_pte_present(pte)) {
25cbff16
SY		 4227 dmar_domain->pgd = (struct dma_pte *)
4228 phys_to_virt(dma_pte_addr(pte));
7a661013 4229 free_pgtable_page(pte);
a99c47a2
TL		 4230 }
4231 dmar_domain->agaw--;
4232 }
fe40f1e0 4233
5913c9bf 4234 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4235}
38717946 4236
4c5478c9
JR		 4237static void intel_iommu_detach_device(struct iommu_domain *domain,
4238 struct device *dev)
38717946 4239{
4c5478c9 4240 struct dmar_domain *dmar_domain = domain->priv;
4c5478c9 4241
bf9c9eda 4242 domain_remove_one_dev_info(dmar_domain, dev);
faa3d6f5 4243}
c7151a8d 4244
b146a1c9
JR		 4245static int intel_iommu_map(struct iommu_domain *domain,
4246 unsigned long iova, phys_addr_t hpa,
5009065d 4247 size_t size, int iommu_prot)
faa3d6f5 4248{
dde57a21 4249 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4250 u64 max_addr;
dde57a21 4251 int prot = 0;
faa3d6f5 4252 int ret;
fe40f1e0 4253
dde57a21
JR		 4254 if (iommu_prot & IOMMU_READ)
4255 prot |= DMA_PTE_READ;
4256 if (iommu_prot & IOMMU_WRITE)
4257 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4258 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4259 prot |= DMA_PTE_SNP;
dde57a21 4260
163cc52c 4261 max_addr = iova + size;
dde57a21 4262 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4263 u64 end;
4264
4265 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4266 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4267 if (end < max_addr) {
8954da1f 4268 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4269 "sufficient for the mapped address (%llx)\n",
8954da1f 4270 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4271 return -EFAULT;
4272 }
dde57a21 4273 dmar_domain->max_addr = max_addr;
fe40f1e0 4274 }
ad051221
DW		 4275 /* Round up size to next multiple of PAGE_SIZE, if it and
4276 the low bits of hpa would take us onto the next page */
88cb6a74 4277 size = aligned_nrpages(hpa, size);
ad051221
DW		 4278 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4279 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4280 return ret;
38717946 4281}
38717946 4282
5009065d 4283static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4284 unsigned long iova, size_t size)
38717946 4285{
dde57a21 4286 struct dmar_domain *dmar_domain = domain->priv;
ea8ea460
DW		 4287 struct page *freelist = NULL;
4288 struct intel_iommu *iommu;
4289 unsigned long start_pfn, last_pfn;
4290 unsigned int npages;
4291 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4292
4293 /* Cope with horrid API which requires us to unmap more than the
4294 size argument if it happens to be a large-page mapping. */
4295 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4296 BUG();
4297
4298 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4299 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4300
ea8ea460
DW		 4301 start_pfn = iova >> VTD_PAGE_SHIFT;
4302 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4303
4304 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4305
4306 npages = last_pfn - start_pfn + 1;
4307
4308 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4309 iommu = g_iommus[iommu_id];
4310
4311 /*
4312 * find bit position of dmar_domain
4313 */
4314 ndomains = cap_ndoms(iommu->cap);
4315 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4316 if (iommu->domains[num] == dmar_domain)
4317 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4318 npages, !freelist, 0);
4319 }
4320
4321 }
4322
4323 dma_free_pagelist(freelist);
fe40f1e0 4324
163cc52c
DW		 4325 if (dmar_domain->max_addr == iova + size)
4326 dmar_domain->max_addr = iova;
b146a1c9 4327
5cf0a76f 4328 return size;
38717946 4329}
38717946 4330
d14d6577 4331static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4332 dma_addr_t iova)
38717946 4333{
d14d6577 4334 struct dmar_domain *dmar_domain = domain->priv;
38717946 4335 struct dma_pte *pte;
5cf0a76f 4336 int level = 0;
faa3d6f5 4337 u64 phys = 0;
38717946 4338
5cf0a76f 4339 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4340 if (pte)
faa3d6f5 4341 phys = dma_pte_addr(pte);
38717946 4342
faa3d6f5 4343 return phys;
38717946 4344}
a8bcbb0d 4345
dbb9fd86
SY		 4346static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4347 unsigned long cap)
4348{
4349 struct dmar_domain *dmar_domain = domain->priv;
4350
4351 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4352 return dmar_domain->iommu_snooping;
323f99cb 4353 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4354 return irq_remapping_enabled;
dbb9fd86
SY		 4355
4356 return 0;
4357}
4358
abdfdde2
AW		 4359static int intel_iommu_add_device(struct device *dev)
4360{
abdfdde2 4361 struct iommu_group *group;
156baca8 4362 u8 bus, devfn;
70ae6f0d 4363
156baca8 4364 if (!device_to_iommu(dev, &bus, &devfn))
70ae6f0d
AW		 4365 return -ENODEV;
4366
e17f9ff4 4367 group = iommu_group_get_for_dev(dev);
783f157b 4368
e17f9ff4
AW		 4369 if (IS_ERR(group))
4370 return PTR_ERR(group);
bcb71abe 4371
abdfdde2 4372 iommu_group_put(group);
e17f9ff4 4373 return 0;
abdfdde2 4374}
70ae6f0d 4375
abdfdde2
AW		 4376static void intel_iommu_remove_device(struct device *dev)
4377{
4378 iommu_group_remove_device(dev);
70ae6f0d
AW		 4379}
4380
a8bcbb0d
JR		 4381static struct iommu_ops intel_iommu_ops = {
4382 .domain_init = intel_iommu_domain_init,
4383 .domain_destroy = intel_iommu_domain_destroy,
4384 .attach_dev = intel_iommu_attach_device,
4385 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4386 .map = intel_iommu_map,
4387 .unmap = intel_iommu_unmap,
a8bcbb0d 4388 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4389 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4390 .add_device = intel_iommu_add_device,
4391 .remove_device = intel_iommu_remove_device,
6d1c56a9 4392 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4393};
9af88143 4394
9452618e
DV		 4395static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4396{
4397 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4398 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4399 dmar_map_gfx = 0;
4400}
4401
4402DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4403DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4404DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4405DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4406DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4407DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4408DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4409
d34d6517 4410static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4411{
4412 /*
4413 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4414 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4415 */
4416 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4417 rwbf_quirk = 1;
4418}
4419
4420DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4421DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4422DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4423DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4424DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4425DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4426DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4427
eecfd57f
AJ		 4428#define GGC 0x52
4429#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4430#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4431#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4432#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4433#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4434#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4435#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4436#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4437
d34d6517 4438static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4439{
4440 unsigned short ggc;
4441
eecfd57f 4442 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4443 return;
4444
eecfd57f 4445 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4446 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4447 dmar_map_gfx = 0;
6fbcfb3e
DW		 4448 } else if (dmar_map_gfx) {
4449 /* we have to ensure the gfx device is idle before we flush */
4450 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4451 intel_iommu_strict = 1;
4452 }
9eecabcb
DW		 4453}
4454DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4455DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4456DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4457DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4458
e0fc7e0b
DW		 4459/* On Tylersburg chipsets, some BIOSes have been known to enable the
4460 ISOCH DMAR unit for the Azalia sound device, but not give it any
4461 TLB entries, which causes it to deadlock. Check for that. We do
4462 this in a function called from init_dmars(), instead of in a PCI
4463 quirk, because we don't want to print the obnoxious "BIOS broken"
4464 message if VT-d is actually disabled.
4465*/
4466static void __init check_tylersburg_isoch(void)
4467{
4468 struct pci_dev *pdev;
4469 uint32_t vtisochctrl;
4470
4471 /* If there's no Azalia in the system anyway, forget it. */
4472 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4473 if (!pdev)
4474 return;
4475 pci_dev_put(pdev);
4476
4477 /* System Management Registers. Might be hidden, in which case
4478 we can't do the sanity check. But that's OK, because the
4479 known-broken BIOSes _don't_ actually hide it, so far. */
4480 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4481 if (!pdev)
4482 return;
4483
4484 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4485 pci_dev_put(pdev);
4486 return;
4487 }
4488
4489 pci_dev_put(pdev);
4490
4491 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4492 if (vtisochctrl & 1)
4493 return;
4494
4495 /* Drop all bits other than the number of TLB entries */
4496 vtisochctrl &= 0x1c;
4497
4498 /* If we have the recommended number of TLB entries (16), fine. */
4499 if (vtisochctrl == 0x10)
4500 return;
4501
4502 /* Zero TLB entries? You get to ride the short bus to school. */
4503 if (!vtisochctrl) {
4504 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4505 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4506 dmi_get_system_info(DMI_BIOS_VENDOR),
4507 dmi_get_system_info(DMI_BIOS_VERSION),
4508 dmi_get_system_info(DMI_PRODUCT_VERSION));
4509 iommu_identity_mapping |= IDENTMAP_AZALIA;
4510 return;
4511 }
4512
4513 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4514 vtisochctrl);
4515}
Linux 6.x block layer and io_uring tree(s)