projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'pm-cpufreq'
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
1ce28feb
WH		 367/* domain represents a virtual machine, more than one devices
368 * across iommus may be owned in one domain, e.g. kvm guest.
369 */
ab8dfe25 370#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 371
2c2e2c38 372/* si_domain contains mulitple devices */
ab8dfe25 373#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 374
99126f7c
MM		 375struct dmar_domain {
376 int id; /* domain id */
4c923d47 377 int nid; /* node id */
78d8e704 378 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 379 /* bitmap of iommus this domain uses*/
99126f7c 380
00a77deb 381 struct list_head devices; /* all devices' list */
99126f7c
MM		 382 struct iova_domain iovad; /* iova's that belong to this domain */
383
384 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 385 int gaw; /* max guest address width */
386
387 /* adjusted guest address width, 0 is level 2 30-bit */
388 int agaw;
389
3b5410e7 390 int flags; /* flags to find out type of domain */
8e604097
WH		 391
392 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 393 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 394 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 395 int iommu_superpage;/* Level of superpages supported:
396 0 == 4KiB (no superpages), 1 == 2MiB,
397 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 398 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 399 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 400
401 struct iommu_domain domain; /* generic domain data structure for
402 iommu core */
99126f7c
MM		 403};
404
a647dacb
MM		 405/* PCI domain-device relationship */
406struct device_domain_info {
407 struct list_head link; /* link to domain siblings */
408 struct list_head global; /* link to global list */
276dbf99 409 u8 bus; /* PCI bus number */
a647dacb 410 u8 devfn; /* PCI devfn number */
fb0cc3aa
BH		 411 struct {
412 u8 enabled:1;
413 u8 qdep;
414 } ats; /* ATS state */
0bcb3e28 415 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 416 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 417 struct dmar_domain *domain; /* pointer to domain */
418};
419
b94e4117
JL		 420struct dmar_rmrr_unit {
421 struct list_head list; /* list of rmrr units */
422 struct acpi_dmar_header *hdr; /* ACPI header */
423 u64 base_address; /* reserved base address*/
424 u64 end_address; /* reserved end address */
832bd858 425 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 426 int devices_cnt; /* target device count */
427};
428
429struct dmar_atsr_unit {
430 struct list_head list; /* list of ATSR units */
431 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 432 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 433 int devices_cnt; /* target device count */
434 u8 include_all:1; /* include all ports */
435};
436
437static LIST_HEAD(dmar_atsr_units);
438static LIST_HEAD(dmar_rmrr_units);
439
440#define for_each_rmrr_units(rmrr) \
441 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
442
5e0d2a6f 443static void flush_unmaps_timeout(unsigned long data);
444
b707cb02 445static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 446
80b20dd8 447#define HIGH_WATER_MARK 250
448struct deferred_flush_tables {
449 int next;
450 struct iova *iova[HIGH_WATER_MARK];
451 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 452 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 453};
454
455static struct deferred_flush_tables *deferred_flush;
456
5e0d2a6f 457/* bitmap for indexing intel_iommus */
5e0d2a6f 458static int g_num_of_iommus;
459
460static DEFINE_SPINLOCK(async_umap_flush_lock);
461static LIST_HEAD(unmaps_to_do);
462
463static int timer_on;
464static long list_size;
5e0d2a6f 465
92d03cc8 466static void domain_exit(struct dmar_domain *domain);
ba395927 467static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 468static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 469 struct device *dev);
92d03cc8 470static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 471 struct device *dev);
2a46ddf7
JL		 472static int domain_detach_iommu(struct dmar_domain *domain,
473 struct intel_iommu *iommu);
ba395927 474
d3f13810 475#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 476int dmar_disabled = 0;
477#else
478int dmar_disabled = 1;
d3f13810 479#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 480
8bc1f85c
ED		 481int intel_iommu_enabled = 0;
482EXPORT_SYMBOL_GPL(intel_iommu_enabled);
483
2d9e667e 484static int dmar_map_gfx = 1;
7d3b03ce 485static int dmar_forcedac;
5e0d2a6f 486static int intel_iommu_strict;
6dd9a7c7 487static int intel_iommu_superpage = 1;
c83b2f20
DW		 488static int intel_iommu_ecs = 1;
489
490/* We only actually use ECS when PASID support (on the new bit 40)
491 * is also advertised. Some early implementations — the ones with
492 * PASID support on bit 28 — have issues even when we *only* use
493 * extended root/context tables. */
494#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
495 ecap_pasid(iommu->ecap))
ba395927 496
c0771df8
DW		 497int intel_iommu_gfx_mapped;
498EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
499
ba395927
KA		 500#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
501static DEFINE_SPINLOCK(device_domain_lock);
502static LIST_HEAD(device_domain_list);
503
b22f6434 504static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 505
4158c2ec
JR		 506static bool translation_pre_enabled(struct intel_iommu *iommu)
507{
508 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
509}
510
091d42e4
JR		 511static void clear_translation_pre_enabled(struct intel_iommu *iommu)
512{
513 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
514}
515
4158c2ec
JR		 516static void init_translation_status(struct intel_iommu *iommu)
517{
518 u32 gsts;
519
520 gsts = readl(iommu->reg + DMAR_GSTS_REG);
521 if (gsts & DMA_GSTS_TES)
522 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
523}
524
00a77deb
JR		 525/* Convert generic 'struct iommu_domain to private struct dmar_domain */
526static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
527{
528 return container_of(dom, struct dmar_domain, domain);
529}
530
ba395927
KA		 531static int __init intel_iommu_setup(char *str)
532{
533 if (!str)
534 return -EINVAL;
535 while (*str) {
0cd5c3c8
KM		 536 if (!strncmp(str, "on", 2)) {
537 dmar_disabled = 0;
9f10e5bf 538 pr_info("IOMMU enabled\n");
0cd5c3c8 539 } else if (!strncmp(str, "off", 3)) {
ba395927 540 dmar_disabled = 1;
9f10e5bf 541 pr_info("IOMMU disabled\n");
ba395927
KA		 542 } else if (!strncmp(str, "igfx_off", 8)) {
543 dmar_map_gfx = 0;
9f10e5bf 544 pr_info("Disable GFX device mapping\n");
7d3b03ce 545 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 546 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 547 dmar_forcedac = 1;
5e0d2a6f 548 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 549 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 550 intel_iommu_strict = 1;
6dd9a7c7 551 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 552 pr_info("Disable supported super page\n");
6dd9a7c7 553 intel_iommu_superpage = 0;
c83b2f20
DW		 554 } else if (!strncmp(str, "ecs_off", 7)) {
555 printk(KERN_INFO
556 "Intel-IOMMU: disable extended context table support\n");
557 intel_iommu_ecs = 0;
ba395927
KA		 558 }
559
560 str += strcspn(str, ",");
561 while (*str == ',')
562 str++;
563 }
564 return 0;
565}
566__setup("intel_iommu=", intel_iommu_setup);
567
568static struct kmem_cache *iommu_domain_cache;
569static struct kmem_cache *iommu_devinfo_cache;
ba395927 570
4c923d47 571static inline void *alloc_pgtable_page(int node)
eb3fa7cb 572{
4c923d47
SS		 573 struct page *page;
574 void *vaddr = NULL;
eb3fa7cb 575
4c923d47
SS		 576 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
577 if (page)
578 vaddr = page_address(page);
eb3fa7cb 579 return vaddr;
ba395927
KA		 580}
581
582static inline void free_pgtable_page(void *vaddr)
583{
584 free_page((unsigned long)vaddr);
585}
586
587static inline void *alloc_domain_mem(void)
588{
354bb65e 589 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 590}
591
38717946 592static void free_domain_mem(void *vaddr)
ba395927
KA		 593{
594 kmem_cache_free(iommu_domain_cache, vaddr);
595}
596
597static inline void * alloc_devinfo_mem(void)
598{
354bb65e 599 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 600}
601
602static inline void free_devinfo_mem(void *vaddr)
603{
604 kmem_cache_free(iommu_devinfo_cache, vaddr);
605}
606
ab8dfe25
JL		 607static inline int domain_type_is_vm(struct dmar_domain *domain)
608{
609 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
610}
611
612static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
613{
614 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
615 DOMAIN_FLAG_STATIC_IDENTITY);
616}
1b573683 617
162d1b10
JL		 618static inline int domain_pfn_supported(struct dmar_domain *domain,
619 unsigned long pfn)
620{
621 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
622
623 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
624}
625
4ed0d3e6 626static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 627{
628 unsigned long sagaw;
629 int agaw = -1;
630
631 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 632 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 633 agaw >= 0; agaw--) {
634 if (test_bit(agaw, &sagaw))
635 break;
636 }
637
638 return agaw;
639}
640
4ed0d3e6
FY		 641/*
642 * Calculate max SAGAW for each iommu.
643 */
644int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
645{
646 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
647}
648
649/*
650 * calculate agaw for each iommu.
651 * "SAGAW" may be different across iommus, use a default agaw, and
652 * get a supported less agaw for iommus that don't support the default agaw.
653 */
654int iommu_calculate_agaw(struct intel_iommu *iommu)
655{
656 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
657}
658
2c2e2c38 659/* This functionin only returns single iommu in a domain */
8c11e798
WH		 660static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
661{
662 int iommu_id;
663
2c2e2c38 664 /* si_domain and vm domain should not get here. */
ab8dfe25 665 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 666 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 667 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
668 return NULL;
669
670 return g_iommus[iommu_id];
671}
672
8e604097
WH		 673static void domain_update_iommu_coherency(struct dmar_domain *domain)
674{
d0501960
DW		 675 struct dmar_drhd_unit *drhd;
676 struct intel_iommu *iommu;
2f119c78
QL		 677 bool found = false;
678 int i;
2e12bc29 679
d0501960 680 domain->iommu_coherency = 1;
8e604097 681
1b198bb0 682 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 683 found = true;
8e604097
WH		 684 if (!ecap_coherent(g_iommus[i]->ecap)) {
685 domain->iommu_coherency = 0;
686 break;
687 }
8e604097 688 }
d0501960
DW		 689 if (found)
690 return;
691
692 /* No hardware attached; use lowest common denominator */
693 rcu_read_lock();
694 for_each_active_iommu(iommu, drhd) {
695 if (!ecap_coherent(iommu->ecap)) {
696 domain->iommu_coherency = 0;
697 break;
698 }
699 }
700 rcu_read_unlock();
8e604097
WH		 701}
702
161f6934 703static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 704{
161f6934
JL		 705 struct dmar_drhd_unit *drhd;
706 struct intel_iommu *iommu;
707 int ret = 1;
58c610bd 708
161f6934
JL		 709 rcu_read_lock();
710 for_each_active_iommu(iommu, drhd) {
711 if (iommu != skip) {
712 if (!ecap_sc_support(iommu->ecap)) {
713 ret = 0;
714 break;
715 }
58c610bd 716 }
58c610bd 717 }
161f6934
JL		 718 rcu_read_unlock();
719
720 return ret;
58c610bd
SY		 721}
722
161f6934 723static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 724{
8140a95d 725 struct dmar_drhd_unit *drhd;
161f6934 726 struct intel_iommu *iommu;
8140a95d 727 int mask = 0xf;
6dd9a7c7
YS		 728
729 if (!intel_iommu_superpage) {
161f6934 730 return 0;
6dd9a7c7
YS		 731 }
732
8140a95d 733 /* set iommu_superpage to the smallest common denominator */
0e242612 734 rcu_read_lock();
8140a95d 735 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 736 if (iommu != skip) {
737 mask &= cap_super_page_val(iommu->cap);
738 if (!mask)
739 break;
6dd9a7c7
YS		 740 }
741 }
0e242612
JL		 742 rcu_read_unlock();
743
161f6934 744 return fls(mask);
6dd9a7c7
YS		 745}
746
58c610bd
SY		 747/* Some capabilities may be different across iommus */
748static void domain_update_iommu_cap(struct dmar_domain *domain)
749{
750 domain_update_iommu_coherency(domain);
161f6934
JL		 751 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
752 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 753}
754
03ecc32c
DW		 755static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
756 u8 bus, u8 devfn, int alloc)
757{
758 struct root_entry *root = &iommu->root_entry[bus];
759 struct context_entry *context;
760 u64 *entry;
761
c83b2f20 762 if (ecs_enabled(iommu)) {
03ecc32c
DW		 763 if (devfn >= 0x80) {
764 devfn -= 0x80;
765 entry = &root->hi;
766 }
767 devfn *= 2;
768 }
769 entry = &root->lo;
770 if (*entry & 1)
771 context = phys_to_virt(*entry & VTD_PAGE_MASK);
772 else {
773 unsigned long phy_addr;
774 if (!alloc)
775 return NULL;
776
777 context = alloc_pgtable_page(iommu->node);
778 if (!context)
779 return NULL;
780
781 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
782 phy_addr = virt_to_phys((void *)context);
783 *entry = phy_addr | 1;
784 __iommu_flush_cache(iommu, entry, sizeof(*entry));
785 }
786 return &context[devfn];
787}
788
4ed6a540
DW		 789static int iommu_dummy(struct device *dev)
790{
791 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
792}
793
156baca8 794static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 795{
796 struct dmar_drhd_unit *drhd = NULL;
b683b230 797 struct intel_iommu *iommu;
156baca8
DW		 798 struct device *tmp;
799 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 800 u16 segment = 0;
c7151a8d
WH		 801 int i;
802
4ed6a540
DW		 803 if (iommu_dummy(dev))
804 return NULL;
805
156baca8
DW		 806 if (dev_is_pci(dev)) {
807 pdev = to_pci_dev(dev);
808 segment = pci_domain_nr(pdev->bus);
ca5b74d2 809 } else if (has_acpi_companion(dev))
156baca8
DW		 810 dev = &ACPI_COMPANION(dev)->dev;
811
0e242612 812 rcu_read_lock();
b683b230 813 for_each_active_iommu(iommu, drhd) {
156baca8 814 if (pdev && segment != drhd->segment)
276dbf99 815 continue;
c7151a8d 816
b683b230 817 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 818 drhd->devices_cnt, i, tmp) {
819 if (tmp == dev) {
820 *bus = drhd->devices[i].bus;
821 *devfn = drhd->devices[i].devfn;
b683b230 822 goto out;
156baca8
DW		 823 }
824
825 if (!pdev || !dev_is_pci(tmp))
826 continue;
827
828 ptmp = to_pci_dev(tmp);
829 if (ptmp->subordinate &&
830 ptmp->subordinate->number <= pdev->bus->number &&
831 ptmp->subordinate->busn_res.end >= pdev->bus->number)
832 goto got_pdev;
924b6231 833 }
c7151a8d 834
156baca8
DW		 835 if (pdev && drhd->include_all) {
836 got_pdev:
837 *bus = pdev->bus->number;
838 *devfn = pdev->devfn;
b683b230 839 goto out;
156baca8 840 }
c7151a8d 841 }
b683b230 842 iommu = NULL;
156baca8 843 out:
0e242612 844 rcu_read_unlock();
c7151a8d 845
b683b230 846 return iommu;
c7151a8d
WH		 847}
848
5331fe6f
WH		 849static void domain_flush_cache(struct dmar_domain *domain,
850 void *addr, int size)
851{
852 if (!domain->iommu_coherency)
853 clflush_cache_range(addr, size);
854}
855
ba395927
KA		 856static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
857{
ba395927 858 struct context_entry *context;
03ecc32c 859 int ret = 0;
ba395927
KA		 860 unsigned long flags;
861
862 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 863 context = iommu_context_addr(iommu, bus, devfn, 0);
864 if (context)
865 ret = context_present(context);
ba395927
KA		 866 spin_unlock_irqrestore(&iommu->lock, flags);
867 return ret;
868}
869
870static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
871{
ba395927
KA		 872 struct context_entry *context;
873 unsigned long flags;
874
875 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 876 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 877 if (context) {
03ecc32c
DW		 878 context_clear_entry(context);
879 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 880 }
881 spin_unlock_irqrestore(&iommu->lock, flags);
882}
883
884static void free_context_table(struct intel_iommu *iommu)
885{
ba395927
KA		 886 int i;
887 unsigned long flags;
888 struct context_entry *context;
889
890 spin_lock_irqsave(&iommu->lock, flags);
891 if (!iommu->root_entry) {
892 goto out;
893 }
894 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 895 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 896 if (context)
897 free_pgtable_page(context);
03ecc32c 898
c83b2f20 899 if (!ecs_enabled(iommu))
03ecc32c
DW		 900 continue;
901
902 context = iommu_context_addr(iommu, i, 0x80, 0);
903 if (context)
904 free_pgtable_page(context);
905
ba395927
KA		 906 }
907 free_pgtable_page(iommu->root_entry);
908 iommu->root_entry = NULL;
909out:
910 spin_unlock_irqrestore(&iommu->lock, flags);
911}
912
b026fd28 913static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 914 unsigned long pfn, int *target_level)
ba395927 915{
ba395927
KA		 916 struct dma_pte *parent, *pte = NULL;
917 int level = agaw_to_level(domain->agaw);
4399c8bf 918 int offset;
ba395927
KA		 919
920 BUG_ON(!domain->pgd);
f9423606 921
162d1b10 922 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 923 /* Address beyond IOMMU's addressing capabilities. */
924 return NULL;
925
ba395927
KA		 926 parent = domain->pgd;
927
5cf0a76f 928 while (1) {
ba395927
KA		 929 void *tmp_page;
930
b026fd28 931 offset = pfn_level_offset(pfn, level);
ba395927 932 pte = &parent[offset];
5cf0a76f 933 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 934 break;
5cf0a76f 935 if (level == *target_level)
ba395927
KA		 936 break;
937
19c239ce 938 if (!dma_pte_present(pte)) {
c85994e4
DW		 939 uint64_t pteval;
940
4c923d47 941 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 942
206a73c1 943 if (!tmp_page)
ba395927 944 return NULL;
206a73c1 945
c85994e4 946 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 947 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 948 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 949 /* Someone else set it while we were thinking; use theirs. */
950 free_pgtable_page(tmp_page);
effad4b5 951 else
c85994e4 952 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 953 }
5cf0a76f
DW		 954 if (level == 1)
955 break;
956
19c239ce 957 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 958 level--;
959 }
960
5cf0a76f
DW		 961 if (!*target_level)
962 *target_level = level;
963
ba395927
KA		 964 return pte;
965}
966
6dd9a7c7 967
ba395927 968/* return address's pte at specific level */
90dcfb5e
DW		 969static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
970 unsigned long pfn,
6dd9a7c7 971 int level, int *large_page)
ba395927
KA		 972{
973 struct dma_pte *parent, *pte = NULL;
974 int total = agaw_to_level(domain->agaw);
975 int offset;
976
977 parent = domain->pgd;
978 while (level <= total) {
90dcfb5e 979 offset = pfn_level_offset(pfn, total);
ba395927
KA		 980 pte = &parent[offset];
981 if (level == total)
982 return pte;
983
6dd9a7c7
YS		 984 if (!dma_pte_present(pte)) {
985 *large_page = total;
ba395927 986 break;
6dd9a7c7
YS		 987 }
988
e16922af 989 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 990 *large_page = total;
991 return pte;
992 }
993
19c239ce 994 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 995 total--;
996 }
997 return NULL;
998}
999
ba395927 1000/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1001static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1002 unsigned long start_pfn,
1003 unsigned long last_pfn)
ba395927 1004{
6dd9a7c7 1005 unsigned int large_page = 1;
310a5ab9 1006 struct dma_pte *first_pte, *pte;
66eae846 1007
162d1b10
JL		 1008 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1009 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1010 BUG_ON(start_pfn > last_pfn);
ba395927 1011
04b18e65 1012 /* we don't need lock here; nobody else touches the iova range */
59c36286 1013 do {
6dd9a7c7
YS		 1014 large_page = 1;
1015 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1016 if (!pte) {
6dd9a7c7 1017 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1018 continue;
1019 }
6dd9a7c7 1020 do {
310a5ab9 1021 dma_clear_pte(pte);
6dd9a7c7 1022 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1023 pte++;
75e6bf96
DW		 1024 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1025
310a5ab9
DW		 1026 domain_flush_cache(domain, first_pte,
1027 (void *)pte - (void *)first_pte);
59c36286
DW		 1028
1029 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1030}
1031
3269ee0b
AW		 1032static void dma_pte_free_level(struct dmar_domain *domain, int level,
1033 struct dma_pte *pte, unsigned long pfn,
1034 unsigned long start_pfn, unsigned long last_pfn)
1035{
1036 pfn = max(start_pfn, pfn);
1037 pte = &pte[pfn_level_offset(pfn, level)];
1038
1039 do {
1040 unsigned long level_pfn;
1041 struct dma_pte *level_pte;
1042
1043 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1044 goto next;
1045
1046 level_pfn = pfn & level_mask(level - 1);
1047 level_pte = phys_to_virt(dma_pte_addr(pte));
1048
1049 if (level > 2)
1050 dma_pte_free_level(domain, level - 1, level_pte,
1051 level_pfn, start_pfn, last_pfn);
1052
1053 /* If range covers entire pagetable, free it */
1054 if (!(start_pfn > level_pfn ||
08336fd2 1055 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1056 dma_clear_pte(pte);
1057 domain_flush_cache(domain, pte, sizeof(*pte));
1058 free_pgtable_page(level_pte);
1059 }
1060next:
1061 pfn += level_size(level);
1062 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1063}
1064
ba395927
KA		 1065/* free page table pages. last level pte should already be cleared */
1066static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1067 unsigned long start_pfn,
1068 unsigned long last_pfn)
ba395927 1069{
162d1b10
JL		 1070 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1071 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1072 BUG_ON(start_pfn > last_pfn);
ba395927 1073
d41a4adb
JL		 1074 dma_pte_clear_range(domain, start_pfn, last_pfn);
1075
f3a0a52f 1076 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1077 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1078 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1079
ba395927 1080 /* free pgd */
d794dc9b 1081 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1082 free_pgtable_page(domain->pgd);
1083 domain->pgd = NULL;
1084 }
1085}
1086
ea8ea460
DW		 1087/* When a page at a given level is being unlinked from its parent, we don't
1088 need to *modify* it at all. All we need to do is make a list of all the
1089 pages which can be freed just as soon as we've flushed the IOTLB and we
1090 know the hardware page-walk will no longer touch them.
1091 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1092 be freed. */
1093static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1094 int level, struct dma_pte *pte,
1095 struct page *freelist)
1096{
1097 struct page *pg;
1098
1099 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1100 pg->freelist = freelist;
1101 freelist = pg;
1102
1103 if (level == 1)
1104 return freelist;
1105
adeb2590
JL		 1106 pte = page_address(pg);
1107 do {
ea8ea460
DW		 1108 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1109 freelist = dma_pte_list_pagetables(domain, level - 1,
1110 pte, freelist);
adeb2590
JL		 1111 pte++;
1112 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1113
1114 return freelist;
1115}
1116
1117static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1118 struct dma_pte *pte, unsigned long pfn,
1119 unsigned long start_pfn,
1120 unsigned long last_pfn,
1121 struct page *freelist)
1122{
1123 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1124
1125 pfn = max(start_pfn, pfn);
1126 pte = &pte[pfn_level_offset(pfn, level)];
1127
1128 do {
1129 unsigned long level_pfn;
1130
1131 if (!dma_pte_present(pte))
1132 goto next;
1133
1134 level_pfn = pfn & level_mask(level);
1135
1136 /* If range covers entire pagetable, free it */
1137 if (start_pfn <= level_pfn &&
1138 last_pfn >= level_pfn + level_size(level) - 1) {
1139 /* These suborbinate page tables are going away entirely. Don't
1140 bother to clear them; we're just going to *free* them. */
1141 if (level > 1 && !dma_pte_superpage(pte))
1142 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1143
1144 dma_clear_pte(pte);
1145 if (!first_pte)
1146 first_pte = pte;
1147 last_pte = pte;
1148 } else if (level > 1) {
1149 /* Recurse down into a level that isn't *entirely* obsolete */
1150 freelist = dma_pte_clear_level(domain, level - 1,
1151 phys_to_virt(dma_pte_addr(pte)),
1152 level_pfn, start_pfn, last_pfn,
1153 freelist);
1154 }
1155next:
1156 pfn += level_size(level);
1157 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1158
1159 if (first_pte)
1160 domain_flush_cache(domain, first_pte,
1161 (void *)++last_pte - (void *)first_pte);
1162
1163 return freelist;
1164}
1165
1166/* We can't just free the pages because the IOMMU may still be walking
1167 the page tables, and may have cached the intermediate levels. The
1168 pages can only be freed after the IOTLB flush has been done. */
1169struct page *domain_unmap(struct dmar_domain *domain,
1170 unsigned long start_pfn,
1171 unsigned long last_pfn)
1172{
ea8ea460
DW		 1173 struct page *freelist = NULL;
1174
162d1b10
JL		 1175 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1176 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1177 BUG_ON(start_pfn > last_pfn);
1178
1179 /* we don't need lock here; nobody else touches the iova range */
1180 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1181 domain->pgd, 0, start_pfn, last_pfn, NULL);
1182
1183 /* free pgd */
1184 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1185 struct page *pgd_page = virt_to_page(domain->pgd);
1186 pgd_page->freelist = freelist;
1187 freelist = pgd_page;
1188
1189 domain->pgd = NULL;
1190 }
1191
1192 return freelist;
1193}
1194
1195void dma_free_pagelist(struct page *freelist)
1196{
1197 struct page *pg;
1198
1199 while ((pg = freelist)) {
1200 freelist = pg->freelist;
1201 free_pgtable_page(page_address(pg));
1202 }
1203}
1204
ba395927
KA		 1205/* iommu handling */
1206static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1207{
1208 struct root_entry *root;
1209 unsigned long flags;
1210
4c923d47 1211 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1212 if (!root) {
9f10e5bf 1213 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1214 iommu->name);
ba395927 1215 return -ENOMEM;
ffebeb46 1216 }
ba395927 1217
5b6985ce 1218 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1219
1220 spin_lock_irqsave(&iommu->lock, flags);
1221 iommu->root_entry = root;
1222 spin_unlock_irqrestore(&iommu->lock, flags);
1223
1224 return 0;
1225}
1226
ba395927
KA		 1227static void iommu_set_root_entry(struct intel_iommu *iommu)
1228{
03ecc32c 1229 u64 addr;
c416daa9 1230 u32 sts;
ba395927
KA		 1231 unsigned long flag;
1232
03ecc32c 1233 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1234 if (ecs_enabled(iommu))
03ecc32c 1235 addr |= DMA_RTADDR_RTT;
ba395927 1236
1f5b3c3f 1237 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1238 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1239
c416daa9 1240 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1241
1242 /* Make sure hardware complete it */
1243 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1244 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1245
1f5b3c3f 1246 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1247}
1248
1249static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1250{
1251 u32 val;
1252 unsigned long flag;
1253
9af88143 1254 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1255 return;
ba395927 1256
1f5b3c3f 1257 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1258 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1259
1260 /* Make sure hardware complete it */
1261 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1262 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1263
1f5b3c3f 1264 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1265}
1266
1267/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1268static void __iommu_flush_context(struct intel_iommu *iommu,
1269 u16 did, u16 source_id, u8 function_mask,
1270 u64 type)
ba395927
KA		 1271{
1272 u64 val = 0;
1273 unsigned long flag;
1274
ba395927
KA		 1275 switch (type) {
1276 case DMA_CCMD_GLOBAL_INVL:
1277 val = DMA_CCMD_GLOBAL_INVL;
1278 break;
1279 case DMA_CCMD_DOMAIN_INVL:
1280 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1281 break;
1282 case DMA_CCMD_DEVICE_INVL:
1283 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1284 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1285 break;
1286 default:
1287 BUG();
1288 }
1289 val |= DMA_CCMD_ICC;
1290
1f5b3c3f 1291 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1292 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1293
1294 /* Make sure hardware complete it */
1295 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1296 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1297
1f5b3c3f 1298 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1299}
1300
ba395927 1301/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1302static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1303 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1304{
1305 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1306 u64 val = 0, val_iva = 0;
1307 unsigned long flag;
1308
ba395927
KA		 1309 switch (type) {
1310 case DMA_TLB_GLOBAL_FLUSH:
1311 /* global flush doesn't need set IVA_REG */
1312 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1313 break;
1314 case DMA_TLB_DSI_FLUSH:
1315 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1316 break;
1317 case DMA_TLB_PSI_FLUSH:
1318 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1319 /* IH bit is passed in as part of address */
ba395927
KA		 1320 val_iva = size_order | addr;
1321 break;
1322 default:
1323 BUG();
1324 }
1325 /* Note: set drain read/write */
1326#if 0
1327 /*
1328 * This is probably to be super secure.. Looks like we can
1329 * ignore it without any impact.
1330 */
1331 if (cap_read_drain(iommu->cap))
1332 val |= DMA_TLB_READ_DRAIN;
1333#endif
1334 if (cap_write_drain(iommu->cap))
1335 val |= DMA_TLB_WRITE_DRAIN;
1336
1f5b3c3f 1337 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1338 /* Note: Only uses first TLB reg currently */
1339 if (val_iva)
1340 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1341 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1342
1343 /* Make sure hardware complete it */
1344 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1345 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1346
1f5b3c3f 1347 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1348
1349 /* check IOTLB invalidation granularity */
1350 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1351 pr_err("Flush IOTLB failed\n");
ba395927 1352 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1353 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1354 (unsigned long long)DMA_TLB_IIRG(type),
1355 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1356}
1357
64ae892b
DW		 1358static struct device_domain_info *
1359iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1360 u8 bus, u8 devfn)
93a23a72 1361{
2f119c78 1362 bool found = false;
93a23a72
YZ		 1363 unsigned long flags;
1364 struct device_domain_info *info;
0bcb3e28 1365 struct pci_dev *pdev;
93a23a72
YZ		 1366
1367 if (!ecap_dev_iotlb_support(iommu->ecap))
1368 return NULL;
1369
1370 if (!iommu->qi)
1371 return NULL;
1372
1373 spin_lock_irqsave(&device_domain_lock, flags);
1374 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1375 if (info->iommu == iommu && info->bus == bus &&
1376 info->devfn == devfn) {
2f119c78 1377 found = true;
93a23a72
YZ		 1378 break;
1379 }
1380 spin_unlock_irqrestore(&device_domain_lock, flags);
1381
0bcb3e28 1382 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1383 return NULL;
1384
0bcb3e28
DW		 1385 pdev = to_pci_dev(info->dev);
1386
1387 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1388 return NULL;
1389
0bcb3e28 1390 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1391 return NULL;
1392
93a23a72
YZ		 1393 return info;
1394}
1395
1396static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1397{
fb0cc3aa
BH		 1398 struct pci_dev *pdev;
1399
0bcb3e28 1400 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1401 return;
1402
fb0cc3aa
BH		 1403 pdev = to_pci_dev(info->dev);
1404 if (pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1405 return;
1406
1407 info->ats.enabled = 1;
1408 info->ats.qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1409}
1410
1411static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1412{
fb0cc3aa 1413 if (!info->ats.enabled)
93a23a72
YZ		 1414 return;
1415
0bcb3e28 1416 pci_disable_ats(to_pci_dev(info->dev));
fb0cc3aa 1417 info->ats.enabled = 0;
93a23a72
YZ		 1418}
1419
1420static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1421 u64 addr, unsigned mask)
1422{
1423 u16 sid, qdep;
1424 unsigned long flags;
1425 struct device_domain_info *info;
1426
1427 spin_lock_irqsave(&device_domain_lock, flags);
1428 list_for_each_entry(info, &domain->devices, link) {
fb0cc3aa 1429 if (!info->ats.enabled)
93a23a72
YZ		 1430 continue;
1431
1432 sid = info->bus << 8 | info->devfn;
fb0cc3aa 1433 qdep = info->ats.qdep;
93a23a72
YZ		 1434 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1435 }
1436 spin_unlock_irqrestore(&device_domain_lock, flags);
1437}
1438
1f0ef2aa 1439static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1440 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1441{
9dd2fe89 1442 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1443 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1444
ba395927
KA		 1445 BUG_ON(pages == 0);
1446
ea8ea460
DW		 1447 if (ih)
1448 ih = 1 << 6;
ba395927 1449 /*
9dd2fe89
YZ		 1450 * Fallback to domain selective flush if no PSI support or the size is
1451 * too big.
ba395927
KA		 1452 * PSI requires page size to be 2 ^ x, and the base address is naturally
1453 * aligned to the size
1454 */
9dd2fe89
YZ		 1455 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1456 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1457 DMA_TLB_DSI_FLUSH);
9dd2fe89 1458 else
ea8ea460 1459 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1460 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1461
1462 /*
82653633
NA		 1463 * In caching mode, changes of pages from non-present to present require
1464 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1465 */
82653633 1466 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1467 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1468}
1469
f8bab735 1470static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1471{
1472 u32 pmen;
1473 unsigned long flags;
1474
1f5b3c3f 1475 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1476 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1477 pmen &= ~DMA_PMEN_EPM;
1478 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1479
1480 /* wait for the protected region status bit to clear */
1481 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1482 readl, !(pmen & DMA_PMEN_PRS), pmen);
1483
1f5b3c3f 1484 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1485}
1486
2a41ccee 1487static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1488{
1489 u32 sts;
1490 unsigned long flags;
1491
1f5b3c3f 1492 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1493 iommu->gcmd |= DMA_GCMD_TE;
1494 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1495
1496 /* Make sure hardware complete it */
1497 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1498 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1499
1f5b3c3f 1500 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1501}
1502
2a41ccee 1503static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1504{
1505 u32 sts;
1506 unsigned long flag;
1507
1f5b3c3f 1508 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1509 iommu->gcmd &= ~DMA_GCMD_TE;
1510 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1511
1512 /* Make sure hardware complete it */
1513 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1514 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1515
1f5b3c3f 1516 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1517}
1518
3460a6d9 1519
ba395927
KA		 1520static int iommu_init_domains(struct intel_iommu *iommu)
1521{
1522 unsigned long ndomains;
1523 unsigned long nlongs;
1524
1525 ndomains = cap_ndoms(iommu->cap);
9f10e5bf
JR		 1526 pr_debug("%s: Number of Domains supported <%ld>\n",
1527 iommu->name, ndomains);
ba395927
KA		 1528 nlongs = BITS_TO_LONGS(ndomains);
1529
94a91b50
DD		 1530 spin_lock_init(&iommu->lock);
1531
ba395927
KA		 1532 /* TBD: there might be 64K domains,
1533 * consider other allocation for future chip
1534 */
1535 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1536 if (!iommu->domain_ids) {
9f10e5bf
JR		 1537 pr_err("%s: Allocating domain id array failed\n",
1538 iommu->name);
ba395927
KA		 1539 return -ENOMEM;
1540 }
1541 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1542 GFP_KERNEL);
1543 if (!iommu->domains) {
9f10e5bf
JR		 1544 pr_err("%s: Allocating domain array failed\n",
1545 iommu->name);
852bdb04
JL		 1546 kfree(iommu->domain_ids);
1547 iommu->domain_ids = NULL;
ba395927
KA		 1548 return -ENOMEM;
1549 }
1550
1551 /*
1552 * if Caching mode is set, then invalid translations are tagged
1553 * with domainid 0. Hence we need to pre-allocate it.
1554 */
1555 if (cap_caching_mode(iommu->cap))
1556 set_bit(0, iommu->domain_ids);
1557 return 0;
1558}
ba395927 1559
ffebeb46 1560static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1561{
1562 struct dmar_domain *domain;
2a46ddf7 1563 int i;
ba395927 1564
94a91b50 1565 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1566 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1567 /*
1568 * Domain id 0 is reserved for invalid translation
1569 * if hardware supports caching mode.
1570 */
1571 if (cap_caching_mode(iommu->cap) && i == 0)
1572 continue;
1573
94a91b50
DD		 1574 domain = iommu->domains[i];
1575 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1576 if (domain_detach_iommu(domain, iommu) == 0 &&
1577 !domain_type_is_vm(domain))
92d03cc8 1578 domain_exit(domain);
5e98c4b1 1579 }
ba395927
KA		 1580 }
1581
1582 if (iommu->gcmd & DMA_GCMD_TE)
1583 iommu_disable_translation(iommu);
ffebeb46 1584}
ba395927 1585
ffebeb46
JL		 1586static void free_dmar_iommu(struct intel_iommu *iommu)
1587{
1588 if ((iommu->domains) && (iommu->domain_ids)) {
1589 kfree(iommu->domains);
1590 kfree(iommu->domain_ids);
1591 iommu->domains = NULL;
1592 iommu->domain_ids = NULL;
1593 }
ba395927 1594
d9630fe9
WH		 1595 g_iommus[iommu->seq_id] = NULL;
1596
ba395927
KA		 1597 /* free context mapping */
1598 free_context_table(iommu);
ba395927
KA		 1599}
1600
ab8dfe25 1601static struct dmar_domain *alloc_domain(int flags)
ba395927 1602{
92d03cc8
JL		 1603 /* domain id for virtual machine, it won't be set in context */
1604 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1605 struct dmar_domain *domain;
ba395927
KA		 1606
1607 domain = alloc_domain_mem();
1608 if (!domain)
1609 return NULL;
1610
ab8dfe25 1611 memset(domain, 0, sizeof(*domain));
4c923d47 1612 domain->nid = -1;
ab8dfe25 1613 domain->flags = flags;
92d03cc8
JL		 1614 spin_lock_init(&domain->iommu_lock);
1615 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1616 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1617 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1618
1619 return domain;
1620}
1621
fb170fb4
JL		 1622static int __iommu_attach_domain(struct dmar_domain *domain,
1623 struct intel_iommu *iommu)
2c2e2c38
FY		 1624{
1625 int num;
1626 unsigned long ndomains;
2c2e2c38 1627
ba395927 1628 ndomains = cap_ndoms(iommu->cap);
ba395927 1629 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1630 if (num < ndomains) {
1631 set_bit(num, iommu->domain_ids);
1632 iommu->domains[num] = domain;
1633 } else {
1634 num = -ENOSPC;
ba395927
KA		 1635 }
1636
fb170fb4
JL		 1637 return num;
1638}
1639
1640static int iommu_attach_domain(struct dmar_domain *domain,
1641 struct intel_iommu *iommu)
1642{
1643 int num;
1644 unsigned long flags;
1645
1646 spin_lock_irqsave(&iommu->lock, flags);
1647 num = __iommu_attach_domain(domain, iommu);
44bde614 1648 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1649 if (num < 0)
9f10e5bf 1650 pr_err("%s: No free domain ids\n", iommu->name);
ba395927 1651
fb170fb4 1652 return num;
ba395927
KA		 1653}
1654
44bde614
JL		 1655static int iommu_attach_vm_domain(struct dmar_domain *domain,
1656 struct intel_iommu *iommu)
1657{
1658 int num;
1659 unsigned long ndomains;
1660
1661 ndomains = cap_ndoms(iommu->cap);
1662 for_each_set_bit(num, iommu->domain_ids, ndomains)
1663 if (iommu->domains[num] == domain)
1664 return num;
1665
1666 return __iommu_attach_domain(domain, iommu);
1667}
1668
2c2e2c38
FY		 1669static void iommu_detach_domain(struct dmar_domain *domain,
1670 struct intel_iommu *iommu)
ba395927
KA		 1671{
1672 unsigned long flags;
2c2e2c38 1673 int num, ndomains;
ba395927 1674
8c11e798 1675 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1676 if (domain_type_is_vm_or_si(domain)) {
1677 ndomains = cap_ndoms(iommu->cap);
1678 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1679 if (iommu->domains[num] == domain) {
1680 clear_bit(num, iommu->domain_ids);
1681 iommu->domains[num] = NULL;
1682 break;
1683 }
2c2e2c38 1684 }
fb170fb4
JL		 1685 } else {
1686 clear_bit(domain->id, iommu->domain_ids);
1687 iommu->domains[domain->id] = NULL;
2c2e2c38 1688 }
8c11e798 1689 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1690}
1691
fb170fb4
JL		 1692static void domain_attach_iommu(struct dmar_domain *domain,
1693 struct intel_iommu *iommu)
1694{
1695 unsigned long flags;
1696
1697 spin_lock_irqsave(&domain->iommu_lock, flags);
1698 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1699 domain->iommu_count++;
1700 if (domain->iommu_count == 1)
1701 domain->nid = iommu->node;
1702 domain_update_iommu_cap(domain);
1703 }
1704 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1705}
1706
1707static int domain_detach_iommu(struct dmar_domain *domain,
1708 struct intel_iommu *iommu)
1709{
1710 unsigned long flags;
1711 int count = INT_MAX;
1712
1713 spin_lock_irqsave(&domain->iommu_lock, flags);
1714 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1715 count = --domain->iommu_count;
1716 domain_update_iommu_cap(domain);
1717 }
1718 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1719
1720 return count;
1721}
1722
ba395927 1723static struct iova_domain reserved_iova_list;
8a443df4 1724static struct lock_class_key reserved_rbtree_key;
ba395927 1725
51a63e67 1726static int dmar_init_reserved_ranges(void)
ba395927
KA		 1727{
1728 struct pci_dev *pdev = NULL;
1729 struct iova *iova;
1730 int i;
ba395927 1731
0fb5fe87
RM		 1732 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1733 DMA_32BIT_PFN);
ba395927 1734
8a443df4
MG		 1735 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1736 &reserved_rbtree_key);
1737
ba395927
KA		 1738 /* IOAPIC ranges shouldn't be accessed by DMA */
1739 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1740 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1741 if (!iova) {
9f10e5bf 1742 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1743 return -ENODEV;
1744 }
ba395927
KA		 1745
1746 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1747 for_each_pci_dev(pdev) {
1748 struct resource *r;
1749
1750 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1751 r = &pdev->resource[i];
1752 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1753 continue;
1a4a4551
DW		 1754 iova = reserve_iova(&reserved_iova_list,
1755 IOVA_PFN(r->start),
1756 IOVA_PFN(r->end));
51a63e67 1757 if (!iova) {
9f10e5bf 1758 pr_err("Reserve iova failed\n");
51a63e67
JC		 1759 return -ENODEV;
1760 }
ba395927
KA		 1761 }
1762 }
51a63e67 1763 return 0;
ba395927
KA		 1764}
1765
1766static void domain_reserve_special_ranges(struct dmar_domain *domain)
1767{
1768 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1769}
1770
1771static inline int guestwidth_to_adjustwidth(int gaw)
1772{
1773 int agaw;
1774 int r = (gaw - 12) % 9;
1775
1776 if (r == 0)
1777 agaw = gaw;
1778 else
1779 agaw = gaw + 9 - r;
1780 if (agaw > 64)
1781 agaw = 64;
1782 return agaw;
1783}
1784
1785static int domain_init(struct dmar_domain *domain, int guest_width)
1786{
1787 struct intel_iommu *iommu;
1788 int adjust_width, agaw;
1789 unsigned long sagaw;
1790
0fb5fe87
RM		 1791 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1792 DMA_32BIT_PFN);
ba395927
KA		 1793 domain_reserve_special_ranges(domain);
1794
1795 /* calculate AGAW */
8c11e798 1796 iommu = domain_get_iommu(domain);
ba395927
KA		 1797 if (guest_width > cap_mgaw(iommu->cap))
1798 guest_width = cap_mgaw(iommu->cap);
1799 domain->gaw = guest_width;
1800 adjust_width = guestwidth_to_adjustwidth(guest_width);
1801 agaw = width_to_agaw(adjust_width);
1802 sagaw = cap_sagaw(iommu->cap);
1803 if (!test_bit(agaw, &sagaw)) {
1804 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1805 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1806 agaw = find_next_bit(&sagaw, 5, agaw);
1807 if (agaw >= 5)
1808 return -ENODEV;
1809 }
1810 domain->agaw = agaw;
ba395927 1811
8e604097
WH		 1812 if (ecap_coherent(iommu->ecap))
1813 domain->iommu_coherency = 1;
1814 else
1815 domain->iommu_coherency = 0;
1816
58c610bd
SY		 1817 if (ecap_sc_support(iommu->ecap))
1818 domain->iommu_snooping = 1;
1819 else
1820 domain->iommu_snooping = 0;
1821
214e39aa
DW		 1822 if (intel_iommu_superpage)
1823 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1824 else
1825 domain->iommu_superpage = 0;
1826
4c923d47 1827 domain->nid = iommu->node;
c7151a8d 1828
ba395927 1829 /* always allocate the top pgd */
4c923d47 1830 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1831 if (!domain->pgd)
1832 return -ENOMEM;
5b6985ce 1833 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1834 return 0;
1835}
1836
1837static void domain_exit(struct dmar_domain *domain)
1838{
46ebb7af
AW		 1839 struct dmar_drhd_unit *drhd;
1840 struct intel_iommu *iommu;
ea8ea460 1841 struct page *freelist = NULL;
ba395927
KA		 1842
1843 /* Domain 0 is reserved, so dont process it */
1844 if (!domain)
1845 return;
1846
7b668357
AW		 1847 /* Flush any lazy unmaps that may reference this domain */
1848 if (!intel_iommu_strict)
1849 flush_unmaps_timeout(0);
1850
92d03cc8 1851 /* remove associated devices */
ba395927 1852 domain_remove_dev_info(domain);
92d03cc8 1853
ba395927
KA		 1854 /* destroy iovas */
1855 put_iova_domain(&domain->iovad);
ba395927 1856
ea8ea460 1857 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1858
92d03cc8 1859 /* clear attached or cached domains */
0e242612 1860 rcu_read_lock();
46ebb7af
AW		 1861 for_each_active_iommu(iommu, drhd)
1862 if (domain_type_is_vm(domain) ||
1863 test_bit(iommu->seq_id, domain->iommu_bmp))
1864 iommu_detach_domain(domain, iommu);
0e242612 1865 rcu_read_unlock();
2c2e2c38 1866
ea8ea460
DW		 1867 dma_free_pagelist(freelist);
1868
ba395927
KA		 1869 free_domain_mem(domain);
1870}
1871
64ae892b
DW		 1872static int domain_context_mapping_one(struct dmar_domain *domain,
1873 struct intel_iommu *iommu,
1874 u8 bus, u8 devfn, int translation)
ba395927
KA		 1875{
1876 struct context_entry *context;
ba395927 1877 unsigned long flags;
ea6606b0 1878 struct dma_pte *pgd;
ea6606b0
WH		 1879 int id;
1880 int agaw;
93a23a72 1881 struct device_domain_info *info = NULL;
ba395927
KA		 1882
1883 pr_debug("Set context mapping for %02x:%02x.%d\n",
1884 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1885
ba395927 1886 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1887 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1888 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1889
03ecc32c
DW		 1890 spin_lock_irqsave(&iommu->lock, flags);
1891 context = iommu_context_addr(iommu, bus, devfn, 1);
1892 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1893 if (!context)
1894 return -ENOMEM;
1895 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1896 if (context_present(context)) {
ba395927
KA		 1897 spin_unlock_irqrestore(&iommu->lock, flags);
1898 return 0;
1899 }
1900
cf484d0e
JR		 1901 context_clear_entry(context);
1902
ea6606b0
WH		 1903 id = domain->id;
1904 pgd = domain->pgd;
1905
ab8dfe25 1906 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1907 if (domain_type_is_vm(domain)) {
1908 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1909 if (id < 0) {
ea6606b0 1910 spin_unlock_irqrestore(&iommu->lock, flags);
9f10e5bf 1911 pr_err("%s: No free domain ids\n", iommu->name);
ea6606b0
WH		 1912 return -EFAULT;
1913 }
ea6606b0
WH		 1914 }
1915
1916 /* Skip top levels of page tables for
1917 * iommu which has less agaw than default.
1672af11 1918 * Unnecessary for PT mode.
ea6606b0 1919 */
1672af11
CW		 1920 if (translation != CONTEXT_TT_PASS_THROUGH) {
1921 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1922 pgd = phys_to_virt(dma_pte_addr(pgd));
1923 if (!dma_pte_present(pgd)) {
1924 spin_unlock_irqrestore(&iommu->lock, flags);
1925 return -ENOMEM;
1926 }
ea6606b0
WH		 1927 }
1928 }
1929 }
1930
1931 context_set_domain_id(context, id);
4ed0d3e6 1932
93a23a72 1933 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1934 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1935 translation = info ? CONTEXT_TT_DEV_IOTLB :
1936 CONTEXT_TT_MULTI_LEVEL;
1937 }
4ed0d3e6
FY		 1938 /*
1939 * In pass through mode, AW must be programmed to indicate the largest
1940 * AGAW value supported by hardware. And ASR is ignored by hardware.
1941 */
93a23a72 1942 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1943 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1944 else {
1945 context_set_address_root(context, virt_to_phys(pgd));
1946 context_set_address_width(context, iommu->agaw);
1947 }
4ed0d3e6
FY		 1948
1949 context_set_translation_type(context, translation);
c07e7d21
MM		 1950 context_set_fault_enable(context);
1951 context_set_present(context);
5331fe6f 1952 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1953
4c25a2c1
DW		 1954 /*
1955 * It's a non-present to present mapping. If hardware doesn't cache
1956 * non-present entry we only need to flush the write-buffer. If the
1957 * _does_ cache non-present entries, then it does so in the special
1958 * domain #0, which we have to flush:
1959 */
1960 if (cap_caching_mode(iommu->cap)) {
1961 iommu->flush.flush_context(iommu, 0,
1962 (((u16)bus) << 8) | devfn,
1963 DMA_CCMD_MASK_NOBIT,
1964 DMA_CCMD_DEVICE_INVL);
18fd779a 1965 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1966 } else {
ba395927 1967 iommu_flush_write_buffer(iommu);
4c25a2c1 1968 }
93a23a72 1969 iommu_enable_dev_iotlb(info);
ba395927 1970 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1971
fb170fb4
JL		 1972 domain_attach_iommu(domain, iommu);
1973
ba395927
KA		 1974 return 0;
1975}
1976
579305f7
AW		 1977struct domain_context_mapping_data {
1978 struct dmar_domain *domain;
1979 struct intel_iommu *iommu;
1980 int translation;
1981};
1982
1983static int domain_context_mapping_cb(struct pci_dev *pdev,
1984 u16 alias, void *opaque)
1985{
1986 struct domain_context_mapping_data *data = opaque;
1987
1988 return domain_context_mapping_one(data->domain, data->iommu,
1989 PCI_BUS_NUM(alias), alias & 0xff,
1990 data->translation);
1991}
1992
ba395927 1993static int
e1f167f3
DW		 1994domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1995 int translation)
ba395927 1996{
64ae892b 1997 struct intel_iommu *iommu;
156baca8 1998 u8 bus, devfn;
579305f7 1999 struct domain_context_mapping_data data;
64ae892b 2000
e1f167f3 2001 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2002 if (!iommu)
2003 return -ENODEV;
ba395927 2004
579305f7
AW		 2005 if (!dev_is_pci(dev))
2006 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 2007 translation);
579305f7
AW		 2008
2009 data.domain = domain;
2010 data.iommu = iommu;
2011 data.translation = translation;
2012
2013 return pci_for_each_dma_alias(to_pci_dev(dev),
2014 &domain_context_mapping_cb, &data);
2015}
2016
2017static int domain_context_mapped_cb(struct pci_dev *pdev,
2018 u16 alias, void *opaque)
2019{
2020 struct intel_iommu *iommu = opaque;
2021
2022 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2023}
2024
e1f167f3 2025static int domain_context_mapped(struct device *dev)
ba395927 2026{
5331fe6f 2027 struct intel_iommu *iommu;
156baca8 2028 u8 bus, devfn;
5331fe6f 2029
e1f167f3 2030 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2031 if (!iommu)
2032 return -ENODEV;
ba395927 2033
579305f7
AW		 2034 if (!dev_is_pci(dev))
2035 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2036
579305f7
AW		 2037 return !pci_for_each_dma_alias(to_pci_dev(dev),
2038 domain_context_mapped_cb, iommu);
ba395927
KA		 2039}
2040
f532959b
FY		 2041/* Returns a number of VTD pages, but aligned to MM page size */
2042static inline unsigned long aligned_nrpages(unsigned long host_addr,
2043 size_t size)
2044{
2045 host_addr &= ~PAGE_MASK;
2046 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2047}
2048
6dd9a7c7
YS		 2049/* Return largest possible superpage level for a given mapping */
2050static inline int hardware_largepage_caps(struct dmar_domain *domain,
2051 unsigned long iov_pfn,
2052 unsigned long phy_pfn,
2053 unsigned long pages)
2054{
2055 int support, level = 1;
2056 unsigned long pfnmerge;
2057
2058 support = domain->iommu_superpage;
2059
2060 /* To use a large page, the virtual *and* physical addresses
2061 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2062 of them will mean we have to use smaller pages. So just
2063 merge them and check both at once. */
2064 pfnmerge = iov_pfn | phy_pfn;
2065
2066 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2067 pages >>= VTD_STRIDE_SHIFT;
2068 if (!pages)
2069 break;
2070 pfnmerge >>= VTD_STRIDE_SHIFT;
2071 level++;
2072 support--;
2073 }
2074 return level;
2075}
2076
9051aa02
DW		 2077static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2078 struct scatterlist *sg, unsigned long phys_pfn,
2079 unsigned long nr_pages, int prot)
e1605495
DW		 2080{
2081 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2082 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2083 unsigned long sg_res = 0;
6dd9a7c7
YS		 2084 unsigned int largepage_lvl = 0;
2085 unsigned long lvl_pages = 0;
e1605495 2086
162d1b10 2087 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2088
2089 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2090 return -EINVAL;
2091
2092 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2093
cc4f14aa
JL		 2094 if (!sg) {
2095 sg_res = nr_pages;
9051aa02
DW		 2096 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2097 }
2098
6dd9a7c7 2099 while (nr_pages > 0) {
c85994e4
DW		 2100 uint64_t tmp;
2101
e1605495 2102 if (!sg_res) {
f532959b 2103 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2104 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2105 sg->dma_length = sg->length;
2106 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2107 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2108 }
6dd9a7c7 2109
e1605495 2110 if (!pte) {
6dd9a7c7
YS		 2111 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2112
5cf0a76f 2113 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2114 if (!pte)
2115 return -ENOMEM;
6dd9a7c7 2116 /* It is large page*/
6491d4d0 2117 if (largepage_lvl > 1) {
6dd9a7c7 2118 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2119 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2120 /*
2121 * Ensure that old small page tables are
2122 * removed to make room for superpage,
2123 * if they exist.
2124 */
6491d4d0 2125 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2126 iov_pfn + lvl_pages - 1);
6491d4d0 2127 } else {
6dd9a7c7 2128 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2129 }
6dd9a7c7 2130
e1605495
DW		 2131 }
2132 /* We don't need lock here, nobody else
2133 * touches the iova range
2134 */
7766a3fb 2135 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2136 if (tmp) {
1bf20f0d 2137 static int dumps = 5;
9f10e5bf
JR		 2138 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2139 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2140 if (dumps) {
2141 dumps--;
2142 debug_dma_dump_mappings(NULL);
2143 }
2144 WARN_ON(1);
2145 }
6dd9a7c7
YS		 2146
2147 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2148
2149 BUG_ON(nr_pages < lvl_pages);
2150 BUG_ON(sg_res < lvl_pages);
2151
2152 nr_pages -= lvl_pages;
2153 iov_pfn += lvl_pages;
2154 phys_pfn += lvl_pages;
2155 pteval += lvl_pages * VTD_PAGE_SIZE;
2156 sg_res -= lvl_pages;
2157
2158 /* If the next PTE would be the first in a new page, then we
2159 need to flush the cache on the entries we've just written.
2160 And then we'll need to recalculate 'pte', so clear it and
2161 let it get set again in the if (!pte) block above.
2162
2163 If we're done (!nr_pages) we need to flush the cache too.
2164
2165 Also if we've been setting superpages, we may need to
2166 recalculate 'pte' and switch back to smaller pages for the
2167 end of the mapping, if the trailing size is not enough to
2168 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2169 pte++;
6dd9a7c7
YS		 2170 if (!nr_pages || first_pte_in_page(pte) ||
2171 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2172 domain_flush_cache(domain, first_pte,
2173 (void *)pte - (void *)first_pte);
2174 pte = NULL;
2175 }
6dd9a7c7
YS		 2176
2177 if (!sg_res && nr_pages)
e1605495
DW		 2178 sg = sg_next(sg);
2179 }
2180 return 0;
2181}
2182
9051aa02
DW		 2183static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2184 struct scatterlist *sg, unsigned long nr_pages,
2185 int prot)
ba395927 2186{
9051aa02
DW		 2187 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2188}
6f6a00e4 2189
9051aa02
DW		 2190static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2191 unsigned long phys_pfn, unsigned long nr_pages,
2192 int prot)
2193{
2194 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2195}
2196
c7151a8d 2197static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2198{
c7151a8d
WH		 2199 if (!iommu)
2200 return;
8c11e798
WH		 2201
2202 clear_context_table(iommu, bus, devfn);
2203 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2204 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2205 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2206}
2207
109b9b04
DW		 2208static inline void unlink_domain_info(struct device_domain_info *info)
2209{
2210 assert_spin_locked(&device_domain_lock);
2211 list_del(&info->link);
2212 list_del(&info->global);
2213 if (info->dev)
0bcb3e28 2214 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2215}
2216
ba395927
KA		 2217static void domain_remove_dev_info(struct dmar_domain *domain)
2218{
3a74ca01 2219 struct device_domain_info *info, *tmp;
fb170fb4 2220 unsigned long flags;
ba395927
KA		 2221
2222 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2223 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2224 unlink_domain_info(info);
ba395927
KA		 2225 spin_unlock_irqrestore(&device_domain_lock, flags);
2226
93a23a72 2227 iommu_disable_dev_iotlb(info);
7c7faa11 2228 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2229
ab8dfe25 2230 if (domain_type_is_vm(domain)) {
7c7faa11 2231 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2232 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2233 }
2234
2235 free_devinfo_mem(info);
ba395927
KA		 2236 spin_lock_irqsave(&device_domain_lock, flags);
2237 }
2238 spin_unlock_irqrestore(&device_domain_lock, flags);
2239}
2240
2241/*
2242 * find_domain
1525a29a 2243 * Note: we use struct device->archdata.iommu stores the info
ba395927 2244 */
1525a29a 2245static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2246{
2247 struct device_domain_info *info;
2248
2249 /* No lock here, assumes no domain exit in normal case */
1525a29a 2250 info = dev->archdata.iommu;
ba395927
KA		 2251 if (info)
2252 return info->domain;
2253 return NULL;
2254}
2255
5a8f40e8 2256static inline struct device_domain_info *
745f2586
JL		 2257dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2258{
2259 struct device_domain_info *info;
2260
2261 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2262 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2263 info->devfn == devfn)
5a8f40e8 2264 return info;
745f2586
JL		 2265
2266 return NULL;
2267}
2268
5a8f40e8 2269static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2270 int bus, int devfn,
b718cd3d
DW		 2271 struct device *dev,
2272 struct dmar_domain *domain)
745f2586 2273{
5a8f40e8 2274 struct dmar_domain *found = NULL;
745f2586
JL		 2275 struct device_domain_info *info;
2276 unsigned long flags;
2277
2278 info = alloc_devinfo_mem();
2279 if (!info)
b718cd3d 2280 return NULL;
745f2586 2281
745f2586
JL		 2282 info->bus = bus;
2283 info->devfn = devfn;
fb0cc3aa
BH		 2284 info->ats.enabled = 0;
2285 info->ats.qdep = 0;
745f2586
JL		 2286 info->dev = dev;
2287 info->domain = domain;
5a8f40e8 2288 info->iommu = iommu;
745f2586
JL		 2289
2290 spin_lock_irqsave(&device_domain_lock, flags);
2291 if (dev)
0bcb3e28 2292 found = find_domain(dev);
5a8f40e8
DW		 2293 else {
2294 struct device_domain_info *info2;
41e80dca 2295 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2296 if (info2)
2297 found = info2->domain;
2298 }
745f2586
JL		 2299 if (found) {
2300 spin_unlock_irqrestore(&device_domain_lock, flags);
2301 free_devinfo_mem(info);
b718cd3d
DW		 2302 /* Caller must free the original domain */
2303 return found;
745f2586
JL		 2304 }
2305
b718cd3d
DW		 2306 list_add(&info->link, &domain->devices);
2307 list_add(&info->global, &device_domain_list);
2308 if (dev)
2309 dev->archdata.iommu = info;
2310 spin_unlock_irqrestore(&device_domain_lock, flags);
2311
2312 return domain;
745f2586
JL		 2313}
2314
579305f7
AW		 2315static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2316{
2317 *(u16 *)opaque = alias;
2318 return 0;
2319}
2320
ba395927 2321/* domain is initialized */
146922ec 2322static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2323{
579305f7
AW		 2324 struct dmar_domain *domain, *tmp;
2325 struct intel_iommu *iommu;
5a8f40e8 2326 struct device_domain_info *info;
579305f7 2327 u16 dma_alias;
ba395927 2328 unsigned long flags;
aa4d066a 2329 u8 bus, devfn;
ba395927 2330
146922ec 2331 domain = find_domain(dev);
ba395927
KA		 2332 if (domain)
2333 return domain;
2334
579305f7
AW		 2335 iommu = device_to_iommu(dev, &bus, &devfn);
2336 if (!iommu)
2337 return NULL;
2338
146922ec
DW		 2339 if (dev_is_pci(dev)) {
2340 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2341
579305f7
AW		 2342 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2343
2344 spin_lock_irqsave(&device_domain_lock, flags);
2345 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2346 PCI_BUS_NUM(dma_alias),
2347 dma_alias & 0xff);
2348 if (info) {
2349 iommu = info->iommu;
2350 domain = info->domain;
5a8f40e8 2351 }
579305f7 2352 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2353
579305f7
AW		 2354 /* DMA alias already has a domain, uses it */
2355 if (info)
2356 goto found_domain;
2357 }
ba395927 2358
146922ec 2359 /* Allocate and initialize new domain for the device */
ab8dfe25 2360 domain = alloc_domain(0);
745f2586 2361 if (!domain)
579305f7 2362 return NULL;
44bde614
JL		 2363 domain->id = iommu_attach_domain(domain, iommu);
2364 if (domain->id < 0) {
2fe9723d 2365 free_domain_mem(domain);
579305f7 2366 return NULL;
2c2e2c38 2367 }
fb170fb4 2368 domain_attach_iommu(domain, iommu);
579305f7
AW		 2369 if (domain_init(domain, gaw)) {
2370 domain_exit(domain);
2371 return NULL;
2c2e2c38 2372 }
ba395927 2373
579305f7
AW		 2374 /* register PCI DMA alias device */
2375 if (dev_is_pci(dev)) {
2376 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2377 dma_alias & 0xff, NULL, domain);
2378
2379 if (!tmp || tmp != domain) {
2380 domain_exit(domain);
2381 domain = tmp;
2382 }
2383
b718cd3d 2384 if (!domain)
579305f7 2385 return NULL;
ba395927
KA		 2386 }
2387
2388found_domain:
579305f7
AW		 2389 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2390
2391 if (!tmp || tmp != domain) {
2392 domain_exit(domain);
2393 domain = tmp;
2394 }
b718cd3d
DW		 2395
2396 return domain;
ba395927
KA		 2397}
2398
2c2e2c38 2399static int iommu_identity_mapping;
e0fc7e0b
DW		 2400#define IDENTMAP_ALL 1
2401#define IDENTMAP_GFX 2
2402#define IDENTMAP_AZALIA 4
2c2e2c38 2403
b213203e
DW		 2404static int iommu_domain_identity_map(struct dmar_domain *domain,
2405 unsigned long long start,
2406 unsigned long long end)
ba395927 2407{
c5395d5c
DW		 2408 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2409 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2410
2411 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2412 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2413 pr_err("Reserving iova failed\n");
b213203e 2414 return -ENOMEM;
ba395927
KA		 2415 }
2416
c5395d5c
DW		 2417 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2418 start, end, domain->id);
ba395927
KA		 2419 /*
2420 * RMRR range might have overlap with physical memory range,
2421 * clear it first
2422 */
c5395d5c 2423 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2424
c5395d5c
DW		 2425 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2426 last_vpfn - first_vpfn + 1,
61df7443 2427 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2428}
2429
0b9d9753 2430static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2431 unsigned long long start,
2432 unsigned long long end)
2433{
2434 struct dmar_domain *domain;
2435 int ret;
2436
0b9d9753 2437 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2438 if (!domain)
2439 return -ENOMEM;
2440
19943b0e
DW		 2441 /* For _hardware_ passthrough, don't bother. But for software
2442 passthrough, we do it anyway -- it may indicate a memory
2443 range which is reserved in E820, so which didn't get set
2444 up to start with in si_domain */
2445 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2446 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2447 dev_name(dev), start, end);
19943b0e
DW		 2448 return 0;
2449 }
2450
9f10e5bf
JR		 2451 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2452 dev_name(dev), start, end);
2453
5595b528
DW		 2454 if (end < start) {
2455 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2456 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2457 dmi_get_system_info(DMI_BIOS_VENDOR),
2458 dmi_get_system_info(DMI_BIOS_VERSION),
2459 dmi_get_system_info(DMI_PRODUCT_VERSION));
2460 ret = -EIO;
2461 goto error;
2462 }
2463
2ff729f5
DW		 2464 if (end >> agaw_to_width(domain->agaw)) {
2465 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2466 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2467 agaw_to_width(domain->agaw),
2468 dmi_get_system_info(DMI_BIOS_VENDOR),
2469 dmi_get_system_info(DMI_BIOS_VERSION),
2470 dmi_get_system_info(DMI_PRODUCT_VERSION));
2471 ret = -EIO;
2472 goto error;
2473 }
19943b0e 2474
b213203e 2475 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2476 if (ret)
2477 goto error;
2478
2479 /* context entry init */
0b9d9753 2480 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2481 if (ret)
2482 goto error;
2483
2484 return 0;
2485
2486 error:
ba395927
KA		 2487 domain_exit(domain);
2488 return ret;
ba395927
KA		 2489}
2490
2491static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2492 struct device *dev)
ba395927 2493{
0b9d9753 2494 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2495 return 0;
0b9d9753
DW		 2496 return iommu_prepare_identity_map(dev, rmrr->base_address,
2497 rmrr->end_address);
ba395927
KA		 2498}
2499
d3f13810 2500#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2501static inline void iommu_prepare_isa(void)
2502{
2503 struct pci_dev *pdev;
2504 int ret;
2505
2506 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2507 if (!pdev)
2508 return;
2509
9f10e5bf 2510 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2511 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2512
2513 if (ret)
9f10e5bf 2514 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2515
9b27e82d 2516 pci_dev_put(pdev);
49a0429e
KA		 2517}
2518#else
2519static inline void iommu_prepare_isa(void)
2520{
2521 return;
2522}
d3f13810 2523#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2524
2c2e2c38 2525static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2526
071e1374 2527static int __init si_domain_init(int hw)
2c2e2c38
FY		 2528{
2529 struct dmar_drhd_unit *drhd;
2530 struct intel_iommu *iommu;
c7ab48d2 2531 int nid, ret = 0;
44bde614 2532 bool first = true;
2c2e2c38 2533
ab8dfe25 2534 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2535 if (!si_domain)
2536 return -EFAULT;
2537
2c2e2c38
FY		 2538 for_each_active_iommu(iommu, drhd) {
2539 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2540 if (ret < 0) {
2c2e2c38
FY		 2541 domain_exit(si_domain);
2542 return -EFAULT;
44bde614
JL		 2543 } else if (first) {
2544 si_domain->id = ret;
2545 first = false;
2546 } else if (si_domain->id != ret) {
2547 domain_exit(si_domain);
2548 return -EFAULT;
2c2e2c38 2549 }
fb170fb4 2550 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2551 }
2552
2553 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2554 domain_exit(si_domain);
2555 return -EFAULT;
2556 }
2557
9f10e5bf 2558 pr_debug("Identity mapping domain is domain %d\n",
9544c003 2559 si_domain->id);
2c2e2c38 2560
19943b0e
DW		 2561 if (hw)
2562 return 0;
2563
c7ab48d2 2564 for_each_online_node(nid) {
5dfe8660
TH		 2565 unsigned long start_pfn, end_pfn;
2566 int i;
2567
2568 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2569 ret = iommu_domain_identity_map(si_domain,
2570 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2571 if (ret)
2572 return ret;
2573 }
c7ab48d2
DW		 2574 }
2575
2c2e2c38
FY		 2576 return 0;
2577}
2578
9b226624 2579static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2580{
2581 struct device_domain_info *info;
2582
2583 if (likely(!iommu_identity_mapping))
2584 return 0;
2585
9b226624 2586 info = dev->archdata.iommu;
cb452a40
MT		 2587 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2588 return (info->domain == si_domain);
2c2e2c38 2589
2c2e2c38
FY		 2590 return 0;
2591}
2592
2593static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2594 struct device *dev, int translation)
2c2e2c38 2595{
0ac72664 2596 struct dmar_domain *ndomain;
5a8f40e8 2597 struct intel_iommu *iommu;
156baca8 2598 u8 bus, devfn;
5fe60f4e 2599 int ret;
2c2e2c38 2600
5913c9bf 2601 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2602 if (!iommu)
2603 return -ENODEV;
2604
5913c9bf 2605 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2606 if (ndomain != domain)
2607 return -EBUSY;
2c2e2c38 2608
5913c9bf 2609 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2610 if (ret) {
5913c9bf 2611 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2612 return ret;
2613 }
2614
2c2e2c38
FY		 2615 return 0;
2616}
2617
0b9d9753 2618static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2619{
2620 struct dmar_rmrr_unit *rmrr;
832bd858 2621 struct device *tmp;
ea2447f7
TM		 2622 int i;
2623
0e242612 2624 rcu_read_lock();
ea2447f7 2625 for_each_rmrr_units(rmrr) {
b683b230
JL		 2626 /*
2627 * Return TRUE if this RMRR contains the device that
2628 * is passed in.
2629 */
2630 for_each_active_dev_scope(rmrr->devices,
2631 rmrr->devices_cnt, i, tmp)
0b9d9753 2632 if (tmp == dev) {
0e242612 2633 rcu_read_unlock();
ea2447f7 2634 return true;
b683b230 2635 }
ea2447f7 2636 }
0e242612 2637 rcu_read_unlock();
ea2447f7
TM		 2638 return false;
2639}
2640
c875d2c1
AW		 2641/*
2642 * There are a couple cases where we need to restrict the functionality of
2643 * devices associated with RMRRs. The first is when evaluating a device for
2644 * identity mapping because problems exist when devices are moved in and out
2645 * of domains and their respective RMRR information is lost. This means that
2646 * a device with associated RMRRs will never be in a "passthrough" domain.
2647 * The second is use of the device through the IOMMU API. This interface
2648 * expects to have full control of the IOVA space for the device. We cannot
2649 * satisfy both the requirement that RMRR access is maintained and have an
2650 * unencumbered IOVA space. We also have no ability to quiesce the device's
2651 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2652 * We therefore prevent devices associated with an RMRR from participating in
2653 * the IOMMU API, which eliminates them from device assignment.
2654 *
2655 * In both cases we assume that PCI USB devices with RMRRs have them largely
2656 * for historical reasons and that the RMRR space is not actively used post
2657 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2658 *
2659 * The same exception is made for graphics devices, with the requirement that
2660 * any use of the RMRR regions will be torn down before assigning the device
2661 * to a guest.
c875d2c1
AW		 2662 */
2663static bool device_is_rmrr_locked(struct device *dev)
2664{
2665 if (!device_has_rmrr(dev))
2666 return false;
2667
2668 if (dev_is_pci(dev)) {
2669 struct pci_dev *pdev = to_pci_dev(dev);
2670
18436afd 2671 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2672 return false;
2673 }
2674
2675 return true;
2676}
2677
3bdb2591 2678static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2679{
ea2447f7 2680
3bdb2591
DW		 2681 if (dev_is_pci(dev)) {
2682 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2683
c875d2c1 2684 if (device_is_rmrr_locked(dev))
3bdb2591 2685 return 0;
e0fc7e0b 2686
3bdb2591
DW		 2687 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2688 return 1;
e0fc7e0b 2689
3bdb2591
DW		 2690 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2691 return 1;
6941af28 2692
3bdb2591 2693 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2694 return 0;
3bdb2591
DW		 2695
2696 /*
2697 * We want to start off with all devices in the 1:1 domain, and
2698 * take them out later if we find they can't access all of memory.
2699 *
2700 * However, we can't do this for PCI devices behind bridges,
2701 * because all PCI devices behind the same bridge will end up
2702 * with the same source-id on their transactions.
2703 *
2704 * Practically speaking, we can't change things around for these
2705 * devices at run-time, because we can't be sure there'll be no
2706 * DMA transactions in flight for any of their siblings.
2707 *
2708 * So PCI devices (unless they're on the root bus) as well as
2709 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2710 * the 1:1 domain, just in _case_ one of their siblings turns out
2711 * not to be able to map all of memory.
2712 */
2713 if (!pci_is_pcie(pdev)) {
2714 if (!pci_is_root_bus(pdev->bus))
2715 return 0;
2716 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2717 return 0;
2718 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2719 return 0;
3bdb2591
DW		 2720 } else {
2721 if (device_has_rmrr(dev))
2722 return 0;
2723 }
3dfc813d 2724
3bdb2591 2725 /*
3dfc813d 2726 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2727 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2728 * take them out of the 1:1 domain later.
2729 */
8fcc5372
CW		 2730 if (!startup) {
2731 /*
2732 * If the device's dma_mask is less than the system's memory
2733 * size then this is not a candidate for identity mapping.
2734 */
3bdb2591 2735 u64 dma_mask = *dev->dma_mask;
8fcc5372 2736
3bdb2591
DW		 2737 if (dev->coherent_dma_mask &&
2738 dev->coherent_dma_mask < dma_mask)
2739 dma_mask = dev->coherent_dma_mask;
8fcc5372 2740
3bdb2591 2741 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2742 }
6941af28
DW		 2743
2744 return 1;
2745}
2746
cf04eee8
DW		 2747static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2748{
2749 int ret;
2750
2751 if (!iommu_should_identity_map(dev, 1))
2752 return 0;
2753
2754 ret = domain_add_dev_info(si_domain, dev,
2755 hw ? CONTEXT_TT_PASS_THROUGH :
2756 CONTEXT_TT_MULTI_LEVEL);
2757 if (!ret)
9f10e5bf
JR		 2758 pr_info("%s identity mapping for device %s\n",
2759 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2760 else if (ret == -ENODEV)
2761 /* device not associated with an iommu */
2762 ret = 0;
2763
2764 return ret;
2765}
2766
2767
071e1374 2768static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2769{
2c2e2c38 2770 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2771 struct dmar_drhd_unit *drhd;
2772 struct intel_iommu *iommu;
2773 struct device *dev;
2774 int i;
2775 int ret = 0;
2c2e2c38 2776
2c2e2c38 2777 for_each_pci_dev(pdev) {
cf04eee8
DW		 2778 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2779 if (ret)
2780 return ret;
2781 }
2782
2783 for_each_active_iommu(iommu, drhd)
2784 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2785 struct acpi_device_physical_node *pn;
2786 struct acpi_device *adev;
2787
2788 if (dev->bus != &acpi_bus_type)
2789 continue;
86080ccc 2790
cf04eee8
DW		 2791 adev= to_acpi_device(dev);
2792 mutex_lock(&adev->physical_node_lock);
2793 list_for_each_entry(pn, &adev->physical_node_list, node) {
2794 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2795 if (ret)
2796 break;
eae460b6 2797 }
cf04eee8
DW		 2798 mutex_unlock(&adev->physical_node_lock);
2799 if (ret)
2800 return ret;
62edf5dc 2801 }
2c2e2c38
FY		 2802
2803 return 0;
2804}
2805
ffebeb46
JL		 2806static void intel_iommu_init_qi(struct intel_iommu *iommu)
2807{
2808 /*
2809 * Start from the sane iommu hardware state.
2810 * If the queued invalidation is already initialized by us
2811 * (for example, while enabling interrupt-remapping) then
2812 * we got the things already rolling from a sane state.
2813 */
2814 if (!iommu->qi) {
2815 /*
2816 * Clear any previous faults.
2817 */
2818 dmar_fault(-1, iommu);
2819 /*
2820 * Disable queued invalidation if supported and already enabled
2821 * before OS handover.
2822 */
2823 dmar_disable_qi(iommu);
2824 }
2825
2826 if (dmar_enable_qi(iommu)) {
2827 /*
2828 * Queued Invalidate not enabled, use Register Based Invalidate
2829 */
2830 iommu->flush.flush_context = __iommu_flush_context;
2831 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2832 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2833 iommu->name);
2834 } else {
2835 iommu->flush.flush_context = qi_flush_context;
2836 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2837 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2838 }
2839}
2840
091d42e4
JR		 2841static int copy_context_table(struct intel_iommu *iommu,
2842 struct root_entry *old_re,
2843 struct context_entry **tbl,
2844 int bus, bool ext)
2845{
2846 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2847 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2848 phys_addr_t old_ce_phys;
2849
2850 tbl_idx = ext ? bus * 2 : bus;
2851
2852 for (devfn = 0; devfn < 256; devfn++) {
2853 /* First calculate the correct index */
2854 idx = (ext ? devfn * 2 : devfn) % 256;
2855
2856 if (idx == 0) {
2857 /* First save what we may have and clean up */
2858 if (new_ce) {
2859 tbl[tbl_idx] = new_ce;
2860 __iommu_flush_cache(iommu, new_ce,
2861 VTD_PAGE_SIZE);
2862 pos = 1;
2863 }
2864
2865 if (old_ce)
2866 iounmap(old_ce);
2867
2868 ret = 0;
2869 if (devfn < 0x80)
2870 old_ce_phys = root_entry_lctp(old_re);
2871 else
2872 old_ce_phys = root_entry_uctp(old_re);
2873
2874 if (!old_ce_phys) {
2875 if (ext && devfn == 0) {
2876 /* No LCTP, try UCTP */
2877 devfn = 0x7f;
2878 continue;
2879 } else {
2880 goto out;
2881 }
2882 }
2883
2884 ret = -ENOMEM;
2885 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2886 if (!old_ce)
2887 goto out;
2888
2889 new_ce = alloc_pgtable_page(iommu->node);
2890 if (!new_ce)
2891 goto out_unmap;
2892
2893 ret = 0;
2894 }
2895
2896 /* Now copy the context entry */
2897 ce = old_ce[idx];
2898
cf484d0e 2899 if (!__context_present(&ce))
091d42e4
JR		 2900 continue;
2901
dbcd861f
JR		 2902 did = context_domain_id(&ce);
2903 if (did >= 0 && did < cap_ndoms(iommu->cap))
2904 set_bit(did, iommu->domain_ids);
2905
cf484d0e
JR		 2906 /*
2907 * We need a marker for copied context entries. This
2908 * marker needs to work for the old format as well as
2909 * for extended context entries.
2910 *
2911 * Bit 67 of the context entry is used. In the old
2912 * format this bit is available to software, in the
2913 * extended format it is the PGE bit, but PGE is ignored
2914 * by HW if PASIDs are disabled (and thus still
2915 * available).
2916 *
2917 * So disable PASIDs first and then mark the entry
2918 * copied. This means that we don't copy PASID
2919 * translations from the old kernel, but this is fine as
2920 * faults there are not fatal.
2921 */
2922 context_clear_pasid_enable(&ce);
2923 context_set_copied(&ce);
2924
091d42e4
JR		 2925 new_ce[idx] = ce;
2926 }
2927
2928 tbl[tbl_idx + pos] = new_ce;
2929
2930 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2931
2932out_unmap:
2933 iounmap(old_ce);
2934
2935out:
2936 return ret;
2937}
2938
2939static int copy_translation_tables(struct intel_iommu *iommu)
2940{
2941 struct context_entry **ctxt_tbls;
2942 struct root_entry *old_rt;
2943 phys_addr_t old_rt_phys;
2944 int ctxt_table_entries;
2945 unsigned long flags;
2946 u64 rtaddr_reg;
2947 int bus, ret;
c3361f2f 2948 bool new_ext, ext;
091d42e4
JR		 2949
2950 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2951 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2952 new_ext = !!ecap_ecs(iommu->ecap);
2953
2954 /*
2955 * The RTT bit can only be changed when translation is disabled,
2956 * but disabling translation means to open a window for data
2957 * corruption. So bail out and don't copy anything if we would
2958 * have to change the bit.
2959 */
2960 if (new_ext != ext)
2961 return -EINVAL;
091d42e4
JR		 2962
2963 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2964 if (!old_rt_phys)
2965 return -EINVAL;
2966
2967 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2968 if (!old_rt)
2969 return -ENOMEM;
2970
2971 /* This is too big for the stack - allocate it from slab */
2972 ctxt_table_entries = ext ? 512 : 256;
2973 ret = -ENOMEM;
2974 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2975 if (!ctxt_tbls)
2976 goto out_unmap;
2977
2978 for (bus = 0; bus < 256; bus++) {
2979 ret = copy_context_table(iommu, &old_rt[bus],
2980 ctxt_tbls, bus, ext);
2981 if (ret) {
2982 pr_err("%s: Failed to copy context table for bus %d\n",
2983 iommu->name, bus);
2984 continue;
2985 }
2986 }
2987
2988 spin_lock_irqsave(&iommu->lock, flags);
2989
2990 /* Context tables are copied, now write them to the root_entry table */
2991 for (bus = 0; bus < 256; bus++) {
2992 int idx = ext ? bus * 2 : bus;
2993 u64 val;
2994
2995 if (ctxt_tbls[idx]) {
2996 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2997 iommu->root_entry[bus].lo = val;
2998 }
2999
3000 if (!ext || !ctxt_tbls[idx + 1])
3001 continue;
3002
3003 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3004 iommu->root_entry[bus].hi = val;
3005 }
3006
3007 spin_unlock_irqrestore(&iommu->lock, flags);
3008
3009 kfree(ctxt_tbls);
3010
3011 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3012
3013 ret = 0;
3014
3015out_unmap:
3016 iounmap(old_rt);
3017
3018 return ret;
3019}
3020
b779260b 3021static int __init init_dmars(void)
ba395927
KA		 3022{
3023 struct dmar_drhd_unit *drhd;
3024 struct dmar_rmrr_unit *rmrr;
a87f4918 3025 bool copied_tables = false;
832bd858 3026 struct device *dev;
ba395927 3027 struct intel_iommu *iommu;
9d783ba0 3028 int i, ret;
2c2e2c38 3029
ba395927
KA		 3030 /*
3031 * for each drhd
3032 * allocate root
3033 * initialize and program root entry to not present
3034 * endfor
3035 */
3036 for_each_drhd_unit(drhd) {
5e0d2a6f 3037 /*
3038 * lock not needed as this is only incremented in the single
3039 * threaded kernel __init code path all other access are read
3040 * only
3041 */
78d8e704 3042 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3043 g_num_of_iommus++;
3044 continue;
3045 }
9f10e5bf 3046 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3047 }
3048
ffebeb46
JL		 3049 /* Preallocate enough resources for IOMMU hot-addition */
3050 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3051 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3052
d9630fe9
WH		 3053 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3054 GFP_KERNEL);
3055 if (!g_iommus) {
9f10e5bf 3056 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3057 ret = -ENOMEM;
3058 goto error;
3059 }
3060
80b20dd8 3061 deferred_flush = kzalloc(g_num_of_iommus *
3062 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3063 if (!deferred_flush) {
5e0d2a6f 3064 ret = -ENOMEM;
989d51fc 3065 goto free_g_iommus;
5e0d2a6f 3066 }
3067
7c919779 3068 for_each_active_iommu(iommu, drhd) {
d9630fe9 3069 g_iommus[iommu->seq_id] = iommu;
ba395927 3070
b63d80d1
JR		 3071 intel_iommu_init_qi(iommu);
3072
e61d98d8
SS		 3073 ret = iommu_init_domains(iommu);
3074 if (ret)
989d51fc 3075 goto free_iommu;
e61d98d8 3076
4158c2ec
JR		 3077 init_translation_status(iommu);
3078
091d42e4
JR		 3079 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3080 iommu_disable_translation(iommu);
3081 clear_translation_pre_enabled(iommu);
3082 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3083 iommu->name);
3084 }
4158c2ec 3085
ba395927
KA		 3086 /*
3087 * TBD:
3088 * we could share the same root & context tables
25985edc 3089 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3090 */
3091 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3092 if (ret)
989d51fc 3093 goto free_iommu;
5f0a7f76 3094
091d42e4
JR		 3095 if (translation_pre_enabled(iommu)) {
3096 pr_info("Translation already enabled - trying to copy translation structures\n");
3097
3098 ret = copy_translation_tables(iommu);
3099 if (ret) {
3100 /*
3101 * We found the IOMMU with translation
3102 * enabled - but failed to copy over the
3103 * old root-entry table. Try to proceed
3104 * by disabling translation now and
3105 * allocating a clean root-entry table.
3106 * This might cause DMAR faults, but
3107 * probably the dump will still succeed.
3108 */
3109 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3110 iommu->name);
3111 iommu_disable_translation(iommu);
3112 clear_translation_pre_enabled(iommu);
3113 } else {
3114 pr_info("Copied translation tables from previous kernel for %s\n",
3115 iommu->name);
a87f4918 3116 copied_tables = true;
091d42e4
JR		 3117 }
3118 }
3119
5f0a7f76
JR		 3120 iommu_flush_write_buffer(iommu);
3121 iommu_set_root_entry(iommu);
3122 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3123 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3124
4ed0d3e6 3125 if (!ecap_pass_through(iommu->ecap))
19943b0e 3126 hw_pass_through = 0;
ba395927
KA		 3127 }
3128
19943b0e 3129 if (iommu_pass_through)
e0fc7e0b
DW		 3130 iommu_identity_mapping |= IDENTMAP_ALL;
3131
d3f13810 3132#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3133 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3134#endif
e0fc7e0b 3135
86080ccc
JR		 3136 if (iommu_identity_mapping) {
3137 ret = si_domain_init(hw_pass_through);
3138 if (ret)
3139 goto free_iommu;
3140 }
3141
e0fc7e0b
DW		 3142 check_tylersburg_isoch();
3143
a87f4918
JR		 3144 /*
3145 * If we copied translations from a previous kernel in the kdump
3146 * case, we can not assign the devices to domains now, as that
3147 * would eliminate the old mappings. So skip this part and defer
3148 * the assignment to device driver initialization time.
3149 */
3150 if (copied_tables)
3151 goto domains_done;
3152
ba395927 3153 /*
19943b0e
DW		 3154 * If pass through is not set or not enabled, setup context entries for
3155 * identity mappings for rmrr, gfx, and isa and may fall back to static
3156 * identity mapping if iommu_identity_mapping is set.
ba395927 3157 */
19943b0e
DW		 3158 if (iommu_identity_mapping) {
3159 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3160 if (ret) {
9f10e5bf 3161 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3162 goto free_iommu;
ba395927
KA		 3163 }
3164 }
ba395927 3165 /*
19943b0e
DW		 3166 * For each rmrr
3167 * for each dev attached to rmrr
3168 * do
3169 * locate drhd for dev, alloc domain for dev
3170 * allocate free domain
3171 * allocate page table entries for rmrr
3172 * if context not allocated for bus
3173 * allocate and init context
3174 * set present in root table for this bus
3175 * init context with domain, translation etc
3176 * endfor
3177 * endfor
ba395927 3178 */
9f10e5bf 3179 pr_info("Setting RMRR:\n");
19943b0e 3180 for_each_rmrr_units(rmrr) {
b683b230
JL		 3181 /* some BIOS lists non-exist devices in DMAR table. */
3182 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3183 i, dev) {
0b9d9753 3184 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3185 if (ret)
9f10e5bf 3186 pr_err("Mapping reserved region failed\n");
ba395927 3187 }
4ed0d3e6 3188 }
49a0429e 3189
19943b0e
DW		 3190 iommu_prepare_isa();
3191
a87f4918
JR		 3192domains_done:
3193
ba395927
KA		 3194 /*
3195 * for each drhd
3196 * enable fault log
3197 * global invalidate context cache
3198 * global invalidate iotlb
3199 * enable translation
3200 */
7c919779 3201 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3202 if (drhd->ignored) {
3203 /*
3204 * we always have to disable PMRs or DMA may fail on
3205 * this device
3206 */
3207 if (force_on)
7c919779 3208 iommu_disable_protect_mem_regions(iommu);
ba395927 3209 continue;
51a63e67 3210 }
ba395927
KA		 3211
3212 iommu_flush_write_buffer(iommu);
3213
3460a6d9
KA		 3214 ret = dmar_set_interrupt(iommu);
3215 if (ret)
989d51fc 3216 goto free_iommu;
3460a6d9 3217
8939ddf6
JR		 3218 if (!translation_pre_enabled(iommu))
3219 iommu_enable_translation(iommu);
3220
b94996c9 3221 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3222 }
3223
3224 return 0;
989d51fc
JL		 3225
3226free_iommu:
ffebeb46
JL		 3227 for_each_active_iommu(iommu, drhd) {
3228 disable_dmar_iommu(iommu);
a868e6b7 3229 free_dmar_iommu(iommu);
ffebeb46 3230 }
9bdc531e 3231 kfree(deferred_flush);
989d51fc 3232free_g_iommus:
d9630fe9 3233 kfree(g_iommus);
989d51fc 3234error:
ba395927
KA		 3235 return ret;
3236}
3237
5a5e02a6 3238/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3239static struct iova *intel_alloc_iova(struct device *dev,
3240 struct dmar_domain *domain,
3241 unsigned long nrpages, uint64_t dma_mask)
ba395927 3242{
ba395927 3243 struct iova *iova = NULL;
ba395927 3244
875764de
DW		 3245 /* Restrict dma_mask to the width that the iommu can handle */
3246 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3247
3248 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3249 /*
3250 * First try to allocate an io virtual address in
284901a9 3251 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3252 * from higher range
ba395927 3253 */
875764de
DW		 3254 iova = alloc_iova(&domain->iovad, nrpages,
3255 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3256 if (iova)
3257 return iova;
3258 }
3259 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3260 if (unlikely(!iova)) {
9f10e5bf 3261 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3262 nrpages, dev_name(dev));
f76aec76
KA		 3263 return NULL;
3264 }
3265
3266 return iova;
3267}
3268
d4b709f4 3269static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3270{
3271 struct dmar_domain *domain;
3272 int ret;
3273
d4b709f4 3274 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3275 if (!domain) {
9f10e5bf 3276 pr_err("Allocating domain for %s failed\n",
d4b709f4 3277 dev_name(dev));
4fe05bbc 3278 return NULL;
ba395927
KA		 3279 }
3280
3281 /* make sure context mapping is ok */
d4b709f4
DW		 3282 if (unlikely(!domain_context_mapped(dev))) {
3283 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 3284 if (ret) {
9f10e5bf 3285 pr_err("Domain context map for %s failed\n",
d4b709f4 3286 dev_name(dev));
4fe05bbc 3287 return NULL;
f76aec76 3288 }
ba395927
KA		 3289 }
3290
f76aec76
KA		 3291 return domain;
3292}
3293
d4b709f4 3294static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3295{
3296 struct device_domain_info *info;
3297
3298 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3299 info = dev->archdata.iommu;
147202aa
DW		 3300 if (likely(info))
3301 return info->domain;
3302
3303 return __get_valid_domain_for_dev(dev);
3304}
3305
ecb509ec 3306/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3307static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3308{
3309 int found;
3310
3d89194a 3311 if (iommu_dummy(dev))
1e4c64c4
DW		 3312 return 1;
3313
2c2e2c38 3314 if (!iommu_identity_mapping)
1e4c64c4 3315 return 0;
2c2e2c38 3316
9b226624 3317 found = identity_mapping(dev);
2c2e2c38 3318 if (found) {
ecb509ec 3319 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3320 return 1;
3321 else {
3322 /*
3323 * 32 bit DMA is removed from si_domain and fall back
3324 * to non-identity mapping.
3325 */
bf9c9eda 3326 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3327 pr_info("32bit %s uses non-identity mapping\n",
3328 dev_name(dev));
2c2e2c38
FY		 3329 return 0;
3330 }
3331 } else {
3332 /*
3333 * In case of a detached 64 bit DMA device from vm, the device
3334 * is put into si_domain for identity mapping.
3335 */
ecb509ec 3336 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3337 int ret;
5913c9bf 3338 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3339 hw_pass_through ?
3340 CONTEXT_TT_PASS_THROUGH :
3341 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38 3342 if (!ret) {
9f10e5bf
JR		 3343 pr_info("64bit %s uses identity mapping\n",
3344 dev_name(dev));
2c2e2c38
FY		 3345 return 1;
3346 }
3347 }
3348 }
3349
1e4c64c4 3350 return 0;
2c2e2c38
FY		 3351}
3352
5040a918 3353static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3354 size_t size, int dir, u64 dma_mask)
f76aec76 3355{
f76aec76 3356 struct dmar_domain *domain;
5b6985ce 3357 phys_addr_t start_paddr;
f76aec76
KA		 3358 struct iova *iova;
3359 int prot = 0;
6865f0d1 3360 int ret;
8c11e798 3361 struct intel_iommu *iommu;
33041ec0 3362 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3363
3364 BUG_ON(dir == DMA_NONE);
2c2e2c38 3365
5040a918 3366 if (iommu_no_mapping(dev))
6865f0d1 3367 return paddr;
f76aec76 3368
5040a918 3369 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3370 if (!domain)
3371 return 0;
3372
8c11e798 3373 iommu = domain_get_iommu(domain);
88cb6a74 3374 size = aligned_nrpages(paddr, size);
f76aec76 3375
5040a918 3376 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3377 if (!iova)
3378 goto error;
3379
ba395927
KA		 3380 /*
3381 * Check if DMAR supports zero-length reads on write only
3382 * mappings..
3383 */
3384 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3385 !cap_zlr(iommu->cap))
ba395927
KA		 3386 prot |= DMA_PTE_READ;
3387 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3388 prot |= DMA_PTE_WRITE;
3389 /*
6865f0d1 3390 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3391 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3392 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3393 * is not a big problem
3394 */
0ab36de2 3395 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3396 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3397 if (ret)
3398 goto error;
3399
1f0ef2aa
DW		 3400 /* it's a non-present to present mapping. Only flush if caching mode */
3401 if (cap_caching_mode(iommu->cap))
ea8ea460 3402 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3403 else
8c11e798 3404 iommu_flush_write_buffer(iommu);
f76aec76 3405
03d6a246
DW		 3406 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3407 start_paddr += paddr & ~PAGE_MASK;
3408 return start_paddr;
ba395927 3409
ba395927 3410error:
f76aec76
KA		 3411 if (iova)
3412 __free_iova(&domain->iovad, iova);
9f10e5bf 3413 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3414 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3415 return 0;
3416}
3417
ffbbef5c
FT		 3418static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3419 unsigned long offset, size_t size,
3420 enum dma_data_direction dir,
3421 struct dma_attrs *attrs)
bb9e6d65 3422{
ffbbef5c 3423 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3424 dir, *dev->dma_mask);
bb9e6d65
FT		 3425}
3426
5e0d2a6f 3427static void flush_unmaps(void)
3428{
80b20dd8 3429 int i, j;
5e0d2a6f 3430
5e0d2a6f 3431 timer_on = 0;
3432
3433 /* just flush them all */
3434 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3435 struct intel_iommu *iommu = g_iommus[i];
3436 if (!iommu)
3437 continue;
c42d9f32 3438
9dd2fe89
YZ		 3439 if (!deferred_flush[i].next)
3440 continue;
3441
78d5f0f5
NA		 3442 /* In caching mode, global flushes turn emulation expensive */
3443 if (!cap_caching_mode(iommu->cap))
3444 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3445 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3446 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3447 unsigned long mask;
3448 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3449 struct dmar_domain *domain = deferred_flush[i].domain[j];
3450
3451 /* On real hardware multiple invalidations are expensive */
3452 if (cap_caching_mode(iommu->cap))
3453 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3454 iova->pfn_lo, iova_size(iova),
ea8ea460 3455 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3456 else {
a156ef99 3457 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3458 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3459 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3460 }
93a23a72 3461 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3462 if (deferred_flush[i].freelist[j])
3463 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3464 }
9dd2fe89 3465 deferred_flush[i].next = 0;
5e0d2a6f 3466 }
3467
5e0d2a6f 3468 list_size = 0;
5e0d2a6f 3469}
3470
3471static void flush_unmaps_timeout(unsigned long data)
3472{
80b20dd8 3473 unsigned long flags;
3474
3475 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3476 flush_unmaps();
80b20dd8 3477 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3478}
3479
ea8ea460 3480static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3481{
3482 unsigned long flags;
80b20dd8 3483 int next, iommu_id;
8c11e798 3484 struct intel_iommu *iommu;
5e0d2a6f 3485
3486 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3487 if (list_size == HIGH_WATER_MARK)
3488 flush_unmaps();
3489
8c11e798
WH		 3490 iommu = domain_get_iommu(dom);
3491 iommu_id = iommu->seq_id;
c42d9f32 3492
80b20dd8 3493 next = deferred_flush[iommu_id].next;
3494 deferred_flush[iommu_id].domain[next] = dom;
3495 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3496 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3497 deferred_flush[iommu_id].next++;
5e0d2a6f 3498
3499 if (!timer_on) {
3500 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3501 timer_on = 1;
3502 }
3503 list_size++;
3504 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3505}
3506
d41a4adb 3507static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3508{
f76aec76 3509 struct dmar_domain *domain;
d794dc9b 3510 unsigned long start_pfn, last_pfn;
ba395927 3511 struct iova *iova;
8c11e798 3512 struct intel_iommu *iommu;
ea8ea460 3513 struct page *freelist;
ba395927 3514
73676832 3515 if (iommu_no_mapping(dev))
f76aec76 3516 return;
2c2e2c38 3517
1525a29a 3518 domain = find_domain(dev);
ba395927
KA		 3519 BUG_ON(!domain);
3520
8c11e798
WH		 3521 iommu = domain_get_iommu(domain);
3522
ba395927 3523 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3524 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3525 (unsigned long long)dev_addr))
ba395927 3526 return;
ba395927 3527
d794dc9b
DW		 3528 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3529 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3530
d794dc9b 3531 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3532 dev_name(dev), start_pfn, last_pfn);
ba395927 3533
ea8ea460 3534 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3535
5e0d2a6f 3536 if (intel_iommu_strict) {
03d6a246 3537 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3538 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3539 /* free iova */
3540 __free_iova(&domain->iovad, iova);
ea8ea460 3541 dma_free_pagelist(freelist);
5e0d2a6f 3542 } else {
ea8ea460 3543 add_unmap(domain, iova, freelist);
5e0d2a6f 3544 /*
3545 * queue up the release of the unmap to save the 1/6th of the
3546 * cpu used up by the iotlb flush operation...
3547 */
5e0d2a6f 3548 }
ba395927
KA		 3549}
3550
d41a4adb
JL		 3551static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3552 size_t size, enum dma_data_direction dir,
3553 struct dma_attrs *attrs)
3554{
3555 intel_unmap(dev, dev_addr);
3556}
3557
5040a918 3558static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3559 dma_addr_t *dma_handle, gfp_t flags,
3560 struct dma_attrs *attrs)
ba395927 3561{
36746436 3562 struct page *page = NULL;
ba395927
KA		 3563 int order;
3564
5b6985ce 3565 size = PAGE_ALIGN(size);
ba395927 3566 order = get_order(size);
e8bb910d 3567
5040a918 3568 if (!iommu_no_mapping(dev))
e8bb910d 3569 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3570 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3571 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3572 flags |= GFP_DMA;
3573 else
3574 flags |= GFP_DMA32;
3575 }
ba395927 3576
36746436
AM		 3577 if (flags & __GFP_WAIT) {
3578 unsigned int count = size >> PAGE_SHIFT;
3579
3580 page = dma_alloc_from_contiguous(dev, count, order);
3581 if (page && iommu_no_mapping(dev) &&
3582 page_to_phys(page) + size > dev->coherent_dma_mask) {
3583 dma_release_from_contiguous(dev, page, count);
3584 page = NULL;
3585 }
3586 }
3587
3588 if (!page)
3589 page = alloc_pages(flags, order);
3590 if (!page)
ba395927 3591 return NULL;
36746436 3592 memset(page_address(page), 0, size);
ba395927 3593
36746436 3594 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3595 DMA_BIDIRECTIONAL,
5040a918 3596 dev->coherent_dma_mask);
ba395927 3597 if (*dma_handle)
36746436
AM		 3598 return page_address(page);
3599 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3600 __free_pages(page, order);
3601
ba395927
KA		 3602 return NULL;
3603}
3604
5040a918 3605static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3606 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3607{
3608 int order;
36746436 3609 struct page *page = virt_to_page(vaddr);
ba395927 3610
5b6985ce 3611 size = PAGE_ALIGN(size);
ba395927
KA		 3612 order = get_order(size);
3613
d41a4adb 3614 intel_unmap(dev, dma_handle);
36746436
AM		 3615 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3616 __free_pages(page, order);
ba395927
KA		 3617}
3618
5040a918 3619static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3620 int nelems, enum dma_data_direction dir,
3621 struct dma_attrs *attrs)
ba395927 3622{
d41a4adb 3623 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3624}
3625
ba395927 3626static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3627 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3628{
3629 int i;
c03ab37c 3630 struct scatterlist *sg;
ba395927 3631
c03ab37c 3632 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3633 BUG_ON(!sg_page(sg));
4cf2e75d 3634 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3635 sg->dma_length = sg->length;
ba395927
KA		 3636 }
3637 return nelems;
3638}
3639
5040a918 3640static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3641 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3642{
ba395927 3643 int i;
ba395927 3644 struct dmar_domain *domain;
f76aec76
KA		 3645 size_t size = 0;
3646 int prot = 0;
f76aec76
KA		 3647 struct iova *iova = NULL;
3648 int ret;
c03ab37c 3649 struct scatterlist *sg;
b536d24d 3650 unsigned long start_vpfn;
8c11e798 3651 struct intel_iommu *iommu;
ba395927
KA		 3652
3653 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3654 if (iommu_no_mapping(dev))
3655 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3656
5040a918 3657 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3658 if (!domain)
3659 return 0;
3660
8c11e798
WH		 3661 iommu = domain_get_iommu(domain);
3662
b536d24d 3663 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3664 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3665
5040a918
DW		 3666 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3667 *dev->dma_mask);
f76aec76 3668 if (!iova) {
c03ab37c 3669 sglist->dma_length = 0;
f76aec76
KA		 3670 return 0;
3671 }
3672
3673 /*
3674 * Check if DMAR supports zero-length reads on write only
3675 * mappings..
3676 */
3677 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3678 !cap_zlr(iommu->cap))
f76aec76
KA		 3679 prot |= DMA_PTE_READ;
3680 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3681 prot |= DMA_PTE_WRITE;
3682
b536d24d 3683 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3684
f532959b 3685 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3686 if (unlikely(ret)) {
e1605495
DW		 3687 dma_pte_free_pagetable(domain, start_vpfn,
3688 start_vpfn + size - 1);
e1605495
DW		 3689 __free_iova(&domain->iovad, iova);
3690 return 0;
ba395927
KA		 3691 }
3692
1f0ef2aa
DW		 3693 /* it's a non-present to present mapping. Only flush if caching mode */
3694 if (cap_caching_mode(iommu->cap))
ea8ea460 3695 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3696 else
8c11e798 3697 iommu_flush_write_buffer(iommu);
1f0ef2aa 3698
ba395927
KA		 3699 return nelems;
3700}
3701
dfb805e8
FT		 3702static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3703{
3704 return !dma_addr;
3705}
3706
160c1d8e 3707struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3708 .alloc = intel_alloc_coherent,
3709 .free = intel_free_coherent,
ba395927
KA		 3710 .map_sg = intel_map_sg,
3711 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3712 .map_page = intel_map_page,
3713 .unmap_page = intel_unmap_page,
dfb805e8 3714 .mapping_error = intel_mapping_error,
ba395927
KA		 3715};
3716
3717static inline int iommu_domain_cache_init(void)
3718{
3719 int ret = 0;
3720
3721 iommu_domain_cache = kmem_cache_create("iommu_domain",
3722 sizeof(struct dmar_domain),
3723 0,
3724 SLAB_HWCACHE_ALIGN,
3725
3726 NULL);
3727 if (!iommu_domain_cache) {
9f10e5bf 3728 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3729 ret = -ENOMEM;
3730 }
3731
3732 return ret;
3733}
3734
3735static inline int iommu_devinfo_cache_init(void)
3736{
3737 int ret = 0;
3738
3739 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3740 sizeof(struct device_domain_info),
3741 0,
3742 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3743 NULL);
3744 if (!iommu_devinfo_cache) {
9f10e5bf 3745 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3746 ret = -ENOMEM;
3747 }
3748
3749 return ret;
3750}
3751
ba395927
KA		 3752static int __init iommu_init_mempool(void)
3753{
3754 int ret;
3755 ret = iommu_iova_cache_init();
3756 if (ret)
3757 return ret;
3758
3759 ret = iommu_domain_cache_init();
3760 if (ret)
3761 goto domain_error;
3762
3763 ret = iommu_devinfo_cache_init();
3764 if (!ret)
3765 return ret;
3766
3767 kmem_cache_destroy(iommu_domain_cache);
3768domain_error:
85b45456 3769 iommu_iova_cache_destroy();
ba395927
KA		 3770
3771 return -ENOMEM;
3772}
3773
3774static void __init iommu_exit_mempool(void)
3775{
3776 kmem_cache_destroy(iommu_devinfo_cache);
3777 kmem_cache_destroy(iommu_domain_cache);
85b45456 3778 iommu_iova_cache_destroy();
ba395927
KA		 3779}
3780
556ab45f
DW		 3781static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3782{
3783 struct dmar_drhd_unit *drhd;
3784 u32 vtbar;
3785 int rc;
3786
3787 /* We know that this device on this chipset has its own IOMMU.
3788 * If we find it under a different IOMMU, then the BIOS is lying
3789 * to us. Hope that the IOMMU for this device is actually
3790 * disabled, and it needs no translation...
3791 */
3792 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3793 if (rc) {
3794 /* "can't" happen */
3795 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3796 return;
3797 }
3798 vtbar &= 0xffff0000;
3799
3800 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3801 drhd = dmar_find_matched_drhd_unit(pdev);
3802 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3803 TAINT_FIRMWARE_WORKAROUND,
3804 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3805 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3806}
3807DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3808
ba395927
KA		 3809static void __init init_no_remapping_devices(void)
3810{
3811 struct dmar_drhd_unit *drhd;
832bd858 3812 struct device *dev;
b683b230 3813 int i;
ba395927
KA		 3814
3815 for_each_drhd_unit(drhd) {
3816 if (!drhd->include_all) {
b683b230
JL		 3817 for_each_active_dev_scope(drhd->devices,
3818 drhd->devices_cnt, i, dev)
3819 break;
832bd858 3820 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3821 if (i == drhd->devices_cnt)
3822 drhd->ignored = 1;
3823 }
3824 }
3825
7c919779 3826 for_each_active_drhd_unit(drhd) {
7c919779 3827 if (drhd->include_all)
ba395927
KA		 3828 continue;
3829
b683b230
JL		 3830 for_each_active_dev_scope(drhd->devices,
3831 drhd->devices_cnt, i, dev)
832bd858 3832 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3833 break;
ba395927
KA		 3834 if (i < drhd->devices_cnt)
3835 continue;
3836
c0771df8
DW		 3837 /* This IOMMU has *only* gfx devices. Either bypass it or
3838 set the gfx_mapped flag, as appropriate */
3839 if (dmar_map_gfx) {
3840 intel_iommu_gfx_mapped = 1;
3841 } else {
3842 drhd->ignored = 1;
b683b230
JL		 3843 for_each_active_dev_scope(drhd->devices,
3844 drhd->devices_cnt, i, dev)
832bd858 3845 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3846 }
3847 }
3848}
3849
f59c7b69
FY		 3850#ifdef CONFIG_SUSPEND
3851static int init_iommu_hw(void)
3852{
3853 struct dmar_drhd_unit *drhd;
3854 struct intel_iommu *iommu = NULL;
3855
3856 for_each_active_iommu(iommu, drhd)
3857 if (iommu->qi)
3858 dmar_reenable_qi(iommu);
3859
b779260b
JC		 3860 for_each_iommu(iommu, drhd) {
3861 if (drhd->ignored) {
3862 /*
3863 * we always have to disable PMRs or DMA may fail on
3864 * this device
3865 */
3866 if (force_on)
3867 iommu_disable_protect_mem_regions(iommu);
3868 continue;
3869 }
3870
f59c7b69
FY		 3871 iommu_flush_write_buffer(iommu);
3872
3873 iommu_set_root_entry(iommu);
3874
3875 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3876 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3877 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3878 iommu_enable_translation(iommu);
b94996c9 3879 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3880 }
3881
3882 return 0;
3883}
3884
3885static void iommu_flush_all(void)
3886{
3887 struct dmar_drhd_unit *drhd;
3888 struct intel_iommu *iommu;
3889
3890 for_each_active_iommu(iommu, drhd) {
3891 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3892 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3893 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3894 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3895 }
3896}
3897
134fac3f 3898static int iommu_suspend(void)
f59c7b69
FY		 3899{
3900 struct dmar_drhd_unit *drhd;
3901 struct intel_iommu *iommu = NULL;
3902 unsigned long flag;
3903
3904 for_each_active_iommu(iommu, drhd) {
3905 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3906 GFP_ATOMIC);
3907 if (!iommu->iommu_state)
3908 goto nomem;
3909 }
3910
3911 iommu_flush_all();
3912
3913 for_each_active_iommu(iommu, drhd) {
3914 iommu_disable_translation(iommu);
3915
1f5b3c3f 3916 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3917
3918 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3919 readl(iommu->reg + DMAR_FECTL_REG);
3920 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3921 readl(iommu->reg + DMAR_FEDATA_REG);
3922 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3923 readl(iommu->reg + DMAR_FEADDR_REG);
3924 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3925 readl(iommu->reg + DMAR_FEUADDR_REG);
3926
1f5b3c3f 3927 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3928 }
3929 return 0;
3930
3931nomem:
3932 for_each_active_iommu(iommu, drhd)
3933 kfree(iommu->iommu_state);
3934
3935 return -ENOMEM;
3936}
3937
134fac3f 3938static void iommu_resume(void)
f59c7b69
FY		 3939{
3940 struct dmar_drhd_unit *drhd;
3941 struct intel_iommu *iommu = NULL;
3942 unsigned long flag;
3943
3944 if (init_iommu_hw()) {
b779260b
JC		 3945 if (force_on)
3946 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3947 else
3948 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3949 return;
f59c7b69
FY		 3950 }
3951
3952 for_each_active_iommu(iommu, drhd) {
3953
1f5b3c3f 3954 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3955
3956 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3957 iommu->reg + DMAR_FECTL_REG);
3958 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3959 iommu->reg + DMAR_FEDATA_REG);
3960 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3961 iommu->reg + DMAR_FEADDR_REG);
3962 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3963 iommu->reg + DMAR_FEUADDR_REG);
3964
1f5b3c3f 3965 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3966 }
3967
3968 for_each_active_iommu(iommu, drhd)
3969 kfree(iommu->iommu_state);
f59c7b69
FY		 3970}
3971
134fac3f 3972static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3973 .resume = iommu_resume,
3974 .suspend = iommu_suspend,
3975};
3976
134fac3f 3977static void __init init_iommu_pm_ops(void)
f59c7b69 3978{
134fac3f 3979 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3980}
3981
3982#else
99592ba4 3983static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3984#endif /* CONFIG_PM */
3985
318fe7df 3986
c2a0b538 3987int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3988{
3989 struct acpi_dmar_reserved_memory *rmrr;
3990 struct dmar_rmrr_unit *rmrru;
3991
3992 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3993 if (!rmrru)
3994 return -ENOMEM;
3995
3996 rmrru->hdr = header;
3997 rmrr = (struct acpi_dmar_reserved_memory *)header;
3998 rmrru->base_address = rmrr->base_address;
3999 rmrru->end_address = rmrr->end_address;
2e455289
JL		 4000 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4001 ((void *)rmrr) + rmrr->header.length,
4002 &rmrru->devices_cnt);
4003 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4004 kfree(rmrru);
4005 return -ENOMEM;
4006 }
318fe7df 4007
2e455289 4008 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4009
2e455289 4010 return 0;
318fe7df
SS		 4011}
4012
6b197249
JL		 4013static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4014{
4015 struct dmar_atsr_unit *atsru;
4016 struct acpi_dmar_atsr *tmp;
4017
4018 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4019 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4020 if (atsr->segment != tmp->segment)
4021 continue;
4022 if (atsr->header.length != tmp->header.length)
4023 continue;
4024 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4025 return atsru;
4026 }
4027
4028 return NULL;
4029}
4030
4031int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4032{
4033 struct acpi_dmar_atsr *atsr;
4034 struct dmar_atsr_unit *atsru;
4035
6b197249
JL		 4036 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4037 return 0;
4038
318fe7df 4039 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4040 atsru = dmar_find_atsr(atsr);
4041 if (atsru)
4042 return 0;
4043
4044 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4045 if (!atsru)
4046 return -ENOMEM;
4047
6b197249
JL		 4048 /*
4049 * If memory is allocated from slab by ACPI _DSM method, we need to
4050 * copy the memory content because the memory buffer will be freed
4051 * on return.
4052 */
4053 atsru->hdr = (void *)(atsru + 1);
4054 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4055 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4056 if (!atsru->include_all) {
4057 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4058 (void *)atsr + atsr->header.length,
4059 &atsru->devices_cnt);
4060 if (atsru->devices_cnt && atsru->devices == NULL) {
4061 kfree(atsru);
4062 return -ENOMEM;
4063 }
4064 }
318fe7df 4065
0e242612 4066 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4067
4068 return 0;
4069}
4070
9bdc531e
JL		 4071static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4072{
4073 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4074 kfree(atsru);
4075}
4076
6b197249
JL		 4077int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4078{
4079 struct acpi_dmar_atsr *atsr;
4080 struct dmar_atsr_unit *atsru;
4081
4082 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4083 atsru = dmar_find_atsr(atsr);
4084 if (atsru) {
4085 list_del_rcu(&atsru->list);
4086 synchronize_rcu();
4087 intel_iommu_free_atsr(atsru);
4088 }
4089
4090 return 0;
4091}
4092
4093int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4094{
4095 int i;
4096 struct device *dev;
4097 struct acpi_dmar_atsr *atsr;
4098 struct dmar_atsr_unit *atsru;
4099
4100 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4101 atsru = dmar_find_atsr(atsr);
4102 if (!atsru)
4103 return 0;
4104
4105 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4106 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4107 i, dev)
4108 return -EBUSY;
4109
4110 return 0;
4111}
4112
ffebeb46
JL		 4113static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4114{
4115 int sp, ret = 0;
4116 struct intel_iommu *iommu = dmaru->iommu;
4117
4118 if (g_iommus[iommu->seq_id])
4119 return 0;
4120
4121 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4122 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4123 iommu->name);
4124 return -ENXIO;
4125 }
4126 if (!ecap_sc_support(iommu->ecap) &&
4127 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4128 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4129 iommu->name);
4130 return -ENXIO;
4131 }
4132 sp = domain_update_iommu_superpage(iommu) - 1;
4133 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4134 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4135 iommu->name);
4136 return -ENXIO;
4137 }
4138
4139 /*
4140 * Disable translation if already enabled prior to OS handover.
4141 */
4142 if (iommu->gcmd & DMA_GCMD_TE)
4143 iommu_disable_translation(iommu);
4144
4145 g_iommus[iommu->seq_id] = iommu;
4146 ret = iommu_init_domains(iommu);
4147 if (ret == 0)
4148 ret = iommu_alloc_root_entry(iommu);
4149 if (ret)
4150 goto out;
4151
4152 if (dmaru->ignored) {
4153 /*
4154 * we always have to disable PMRs or DMA may fail on this device
4155 */
4156 if (force_on)
4157 iommu_disable_protect_mem_regions(iommu);
4158 return 0;
4159 }
4160
4161 intel_iommu_init_qi(iommu);
4162 iommu_flush_write_buffer(iommu);
4163 ret = dmar_set_interrupt(iommu);
4164 if (ret)
4165 goto disable_iommu;
4166
4167 iommu_set_root_entry(iommu);
4168 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4169 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4170 iommu_enable_translation(iommu);
4171
4172 if (si_domain) {
4173 ret = iommu_attach_domain(si_domain, iommu);
4174 if (ret < 0 || si_domain->id != ret)
4175 goto disable_iommu;
4176 domain_attach_iommu(si_domain, iommu);
4177 }
4178
4179 iommu_disable_protect_mem_regions(iommu);
4180 return 0;
4181
4182disable_iommu:
4183 disable_dmar_iommu(iommu);
4184out:
4185 free_dmar_iommu(iommu);
4186 return ret;
4187}
4188
6b197249
JL		 4189int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4190{
ffebeb46
JL		 4191 int ret = 0;
4192 struct intel_iommu *iommu = dmaru->iommu;
4193
4194 if (!intel_iommu_enabled)
4195 return 0;
4196 if (iommu == NULL)
4197 return -EINVAL;
4198
4199 if (insert) {
4200 ret = intel_iommu_add(dmaru);
4201 } else {
4202 disable_dmar_iommu(iommu);
4203 free_dmar_iommu(iommu);
4204 }
4205
4206 return ret;
6b197249
JL		 4207}
4208
9bdc531e
JL		 4209static void intel_iommu_free_dmars(void)
4210{
4211 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4212 struct dmar_atsr_unit *atsru, *atsr_n;
4213
4214 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4215 list_del(&rmrru->list);
4216 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4217 kfree(rmrru);
318fe7df
SS		 4218 }
4219
9bdc531e
JL		 4220 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4221 list_del(&atsru->list);
4222 intel_iommu_free_atsr(atsru);
4223 }
318fe7df
SS		 4224}
4225
4226int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4227{
b683b230 4228 int i, ret = 1;
318fe7df 4229 struct pci_bus *bus;
832bd858
DW		 4230 struct pci_dev *bridge = NULL;
4231 struct device *tmp;
318fe7df
SS		 4232 struct acpi_dmar_atsr *atsr;
4233 struct dmar_atsr_unit *atsru;
4234
4235 dev = pci_physfn(dev);
318fe7df 4236 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4237 bridge = bus->self;
318fe7df 4238 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4239 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4240 return 0;
b5f82ddf 4241 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4242 break;
318fe7df 4243 }
b5f82ddf
JL		 4244 if (!bridge)
4245 return 0;
318fe7df 4246
0e242612 4247 rcu_read_lock();
b5f82ddf
JL		 4248 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4249 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4250 if (atsr->segment != pci_domain_nr(dev->bus))
4251 continue;
4252
b683b230 4253 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4254 if (tmp == &bridge->dev)
b683b230 4255 goto out;
b5f82ddf
JL		 4256
4257 if (atsru->include_all)
b683b230 4258 goto out;
b5f82ddf 4259 }
b683b230
JL		 4260 ret = 0;
4261out:
0e242612 4262 rcu_read_unlock();
318fe7df 4263
b683b230 4264 return ret;
318fe7df
SS		 4265}
4266
59ce0515
JL		 4267int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4268{
4269 int ret = 0;
4270 struct dmar_rmrr_unit *rmrru;
4271 struct dmar_atsr_unit *atsru;
4272 struct acpi_dmar_atsr *atsr;
4273 struct acpi_dmar_reserved_memory *rmrr;
4274
4275 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4276 return 0;
4277
4278 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4279 rmrr = container_of(rmrru->hdr,
4280 struct acpi_dmar_reserved_memory, header);
4281 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4282 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4283 ((void *)rmrr) + rmrr->header.length,
4284 rmrr->segment, rmrru->devices,
4285 rmrru->devices_cnt);
27e24950 4286 if(ret < 0)
59ce0515
JL		 4287 return ret;
4288 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4289 dmar_remove_dev_scope(info, rmrr->segment,
4290 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4291 }
4292 }
4293
4294 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4295 if (atsru->include_all)
4296 continue;
4297
4298 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4299 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4300 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4301 (void *)atsr + atsr->header.length,
4302 atsr->segment, atsru->devices,
4303 atsru->devices_cnt);
4304 if (ret > 0)
4305 break;
4306 else if(ret < 0)
4307 return ret;
4308 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4309 if (dmar_remove_dev_scope(info, atsr->segment,
4310 atsru->devices, atsru->devices_cnt))
4311 break;
4312 }
4313 }
4314
4315 return 0;
4316}
4317
99dcaded
FY		 4318/*
4319 * Here we only respond to action of unbound device from driver.
4320 *
4321 * Added device is not attached to its DMAR domain here yet. That will happen
4322 * when mapping the device to iova.
4323 */
4324static int device_notifier(struct notifier_block *nb,
4325 unsigned long action, void *data)
4326{
4327 struct device *dev = data;
99dcaded
FY		 4328 struct dmar_domain *domain;
4329
3d89194a 4330 if (iommu_dummy(dev))
44cd613c
DW		 4331 return 0;
4332
1196c2fb 4333 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4334 return 0;
4335
1525a29a 4336 domain = find_domain(dev);
99dcaded
FY		 4337 if (!domain)
4338 return 0;
4339
3a5670e8 4340 down_read(&dmar_global_lock);
bf9c9eda 4341 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4342 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4343 domain_exit(domain);
3a5670e8 4344 up_read(&dmar_global_lock);
a97590e5 4345
99dcaded
FY		 4346 return 0;
4347}
4348
4349static struct notifier_block device_nb = {
4350 .notifier_call = device_notifier,
4351};
4352
75f05569
JL		 4353static int intel_iommu_memory_notifier(struct notifier_block *nb,
4354 unsigned long val, void *v)
4355{
4356 struct memory_notify *mhp = v;
4357 unsigned long long start, end;
4358 unsigned long start_vpfn, last_vpfn;
4359
4360 switch (val) {
4361 case MEM_GOING_ONLINE:
4362 start = mhp->start_pfn << PAGE_SHIFT;
4363 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4364 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4365 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4366 start, end);
4367 return NOTIFY_BAD;
4368 }
4369 break;
4370
4371 case MEM_OFFLINE:
4372 case MEM_CANCEL_ONLINE:
4373 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4374 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4375 while (start_vpfn <= last_vpfn) {
4376 struct iova *iova;
4377 struct dmar_drhd_unit *drhd;
4378 struct intel_iommu *iommu;
ea8ea460 4379 struct page *freelist;
75f05569
JL		 4380
4381 iova = find_iova(&si_domain->iovad, start_vpfn);
4382 if (iova == NULL) {
9f10e5bf 4383 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4384 start_vpfn);
4385 break;
4386 }
4387
4388 iova = split_and_remove_iova(&si_domain->iovad, iova,
4389 start_vpfn, last_vpfn);
4390 if (iova == NULL) {
9f10e5bf 4391 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4392 start_vpfn, last_vpfn);
4393 return NOTIFY_BAD;
4394 }
4395
ea8ea460
DW		 4396 freelist = domain_unmap(si_domain, iova->pfn_lo,
4397 iova->pfn_hi);
4398
75f05569
JL		 4399 rcu_read_lock();
4400 for_each_active_iommu(iommu, drhd)
4401 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4402 iova->pfn_lo, iova_size(iova),
ea8ea460 4403 !freelist, 0);
75f05569 4404 rcu_read_unlock();
ea8ea460 4405 dma_free_pagelist(freelist);
75f05569
JL		 4406
4407 start_vpfn = iova->pfn_hi + 1;
4408 free_iova_mem(iova);
4409 }
4410 break;
4411 }
4412
4413 return NOTIFY_OK;
4414}
4415
4416static struct notifier_block intel_iommu_memory_nb = {
4417 .notifier_call = intel_iommu_memory_notifier,
4418 .priority = 0
4419};
4420
a5459cfe
AW		 4421
4422static ssize_t intel_iommu_show_version(struct device *dev,
4423 struct device_attribute *attr,
4424 char *buf)
4425{
4426 struct intel_iommu *iommu = dev_get_drvdata(dev);
4427 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4428 return sprintf(buf, "%d:%d\n",
4429 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4430}
4431static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4432
4433static ssize_t intel_iommu_show_address(struct device *dev,
4434 struct device_attribute *attr,
4435 char *buf)
4436{
4437 struct intel_iommu *iommu = dev_get_drvdata(dev);
4438 return sprintf(buf, "%llx\n", iommu->reg_phys);
4439}
4440static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4441
4442static ssize_t intel_iommu_show_cap(struct device *dev,
4443 struct device_attribute *attr,
4444 char *buf)
4445{
4446 struct intel_iommu *iommu = dev_get_drvdata(dev);
4447 return sprintf(buf, "%llx\n", iommu->cap);
4448}
4449static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4450
4451static ssize_t intel_iommu_show_ecap(struct device *dev,
4452 struct device_attribute *attr,
4453 char *buf)
4454{
4455 struct intel_iommu *iommu = dev_get_drvdata(dev);
4456 return sprintf(buf, "%llx\n", iommu->ecap);
4457}
4458static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4459
4460static struct attribute *intel_iommu_attrs[] = {
4461 &dev_attr_version.attr,
4462 &dev_attr_address.attr,
4463 &dev_attr_cap.attr,
4464 &dev_attr_ecap.attr,
4465 NULL,
4466};
4467
4468static struct attribute_group intel_iommu_group = {
4469 .name = "intel-iommu",
4470 .attrs = intel_iommu_attrs,
4471};
4472
4473const struct attribute_group *intel_iommu_groups[] = {
4474 &intel_iommu_group,
4475 NULL,
4476};
4477
ba395927
KA		 4478int __init intel_iommu_init(void)
4479{
9bdc531e 4480 int ret = -ENODEV;
3a93c841 4481 struct dmar_drhd_unit *drhd;
7c919779 4482 struct intel_iommu *iommu;
ba395927 4483
a59b50e9
JC		 4484 /* VT-d is required for a TXT/tboot launch, so enforce that */
4485 force_on = tboot_force_iommu();
4486
3a5670e8
JL		 4487 if (iommu_init_mempool()) {
4488 if (force_on)
4489 panic("tboot: Failed to initialize iommu memory\n");
4490 return -ENOMEM;
4491 }
4492
4493 down_write(&dmar_global_lock);
a59b50e9
JC		 4494 if (dmar_table_init()) {
4495 if (force_on)
4496 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4497 goto out_free_dmar;
a59b50e9 4498 }
ba395927 4499
c2c7286a 4500 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4501 if (force_on)
4502 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4503 goto out_free_dmar;
a59b50e9 4504 }
1886e8a9 4505
75f1cdf1 4506 if (no_iommu || dmar_disabled)
9bdc531e 4507 goto out_free_dmar;
2ae21010 4508
318fe7df 4509 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4510 pr_info("No RMRR found\n");
318fe7df
SS		 4511
4512 if (list_empty(&dmar_atsr_units))
9f10e5bf 4513 pr_info("No ATSR found\n");
318fe7df 4514
51a63e67
JC		 4515 if (dmar_init_reserved_ranges()) {
4516 if (force_on)
4517 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4518 goto out_free_reserved_range;
51a63e67 4519 }
ba395927
KA		 4520
4521 init_no_remapping_devices();
4522
b779260b 4523 ret = init_dmars();
ba395927 4524 if (ret) {
a59b50e9
JC		 4525 if (force_on)
4526 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4527 pr_err("Initialization failed\n");
9bdc531e 4528 goto out_free_reserved_range;
ba395927 4529 }
3a5670e8 4530 up_write(&dmar_global_lock);
9f10e5bf 4531 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4532
5e0d2a6f 4533 init_timer(&unmap_timer);
75f1cdf1
FT		 4534#ifdef CONFIG_SWIOTLB
4535 swiotlb = 0;
4536#endif
19943b0e 4537 dma_ops = &intel_dma_ops;
4ed0d3e6 4538
134fac3f 4539 init_iommu_pm_ops();
a8bcbb0d 4540
a5459cfe
AW		 4541 for_each_active_iommu(iommu, drhd)
4542 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4543 intel_iommu_groups,
4544 iommu->name);
4545
4236d97d 4546 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4547 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4548 if (si_domain && !hw_pass_through)
4549 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4550
8bc1f85c
ED		 4551 intel_iommu_enabled = 1;
4552
ba395927 4553 return 0;
9bdc531e
JL		 4554
4555out_free_reserved_range:
4556 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4557out_free_dmar:
4558 intel_iommu_free_dmars();
3a5670e8
JL		 4559 up_write(&dmar_global_lock);
4560 iommu_exit_mempool();
9bdc531e 4561 return ret;
ba395927 4562}
e820482c 4563
579305f7
AW		 4564static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4565{
4566 struct intel_iommu *iommu = opaque;
4567
4568 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4569 return 0;
4570}
4571
4572/*
4573 * NB - intel-iommu lacks any sort of reference counting for the users of
4574 * dependent devices. If multiple endpoints have intersecting dependent
4575 * devices, unbinding the driver from any one of them will possibly leave
4576 * the others unable to operate.
4577 */
3199aa6b 4578static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4579 struct device *dev)
3199aa6b 4580{
0bcb3e28 4581 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4582 return;
4583
579305f7 4584 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4585}
4586
2c2e2c38 4587static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4588 struct device *dev)
c7151a8d 4589{
bca2b916 4590 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4591 struct intel_iommu *iommu;
4592 unsigned long flags;
2f119c78 4593 bool found = false;
156baca8 4594 u8 bus, devfn;
c7151a8d 4595
bf9c9eda 4596 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4597 if (!iommu)
4598 return;
4599
4600 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4601 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4602 if (info->iommu == iommu && info->bus == bus &&
4603 info->devfn == devfn) {
109b9b04 4604 unlink_domain_info(info);
c7151a8d
WH		 4605 spin_unlock_irqrestore(&device_domain_lock, flags);
4606
93a23a72 4607 iommu_disable_dev_iotlb(info);
c7151a8d 4608 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4609 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4610 free_devinfo_mem(info);
4611
4612 spin_lock_irqsave(&device_domain_lock, flags);
4613
4614 if (found)
4615 break;
4616 else
4617 continue;
4618 }
4619
4620 /* if there is no other devices under the same iommu
4621 * owned by this domain, clear this iommu in iommu_bmp
4622 * update iommu count and coherency
4623 */
8bbc4410 4624 if (info->iommu == iommu)
2f119c78 4625 found = true;
c7151a8d
WH		 4626 }
4627
3e7abe25
RD		 4628 spin_unlock_irqrestore(&device_domain_lock, flags);
4629
c7151a8d 4630 if (found == 0) {
fb170fb4
JL		 4631 domain_detach_iommu(domain, iommu);
4632 if (!domain_type_is_vm_or_si(domain))
4633 iommu_detach_domain(domain, iommu);
c7151a8d 4634 }
c7151a8d
WH		 4635}
4636
2c2e2c38 4637static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4638{
4639 int adjust_width;
4640
0fb5fe87
RM		 4641 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4642 DMA_32BIT_PFN);
5e98c4b1
WH		 4643 domain_reserve_special_ranges(domain);
4644
4645 /* calculate AGAW */
4646 domain->gaw = guest_width;
4647 adjust_width = guestwidth_to_adjustwidth(guest_width);
4648 domain->agaw = width_to_agaw(adjust_width);
4649
5e98c4b1 4650 domain->iommu_coherency = 0;
c5b15255 4651 domain->iommu_snooping = 0;
6dd9a7c7 4652 domain->iommu_superpage = 0;
fe40f1e0 4653 domain->max_addr = 0;
5e98c4b1
WH		 4654
4655 /* always allocate the top pgd */
4c923d47 4656 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4657 if (!domain->pgd)
4658 return -ENOMEM;
4659 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4660 return 0;
4661}
4662
00a77deb 4663static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4664{
5d450806 4665 struct dmar_domain *dmar_domain;
00a77deb
JR		 4666 struct iommu_domain *domain;
4667
4668 if (type != IOMMU_DOMAIN_UNMANAGED)
4669 return NULL;
38717946 4670
ab8dfe25 4671 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4672 if (!dmar_domain) {
9f10e5bf 4673 pr_err("Can't allocate dmar_domain\n");
00a77deb 4674 return NULL;
38717946 4675 }
2c2e2c38 4676 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4677 pr_err("Domain initialization failed\n");
92d03cc8 4678 domain_exit(dmar_domain);
00a77deb 4679 return NULL;
38717946 4680 }
8140a95d 4681 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4682
00a77deb 4683 domain = &dmar_domain->domain;
8a0e715b
JR		 4684 domain->geometry.aperture_start = 0;
4685 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4686 domain->geometry.force_aperture = true;
4687
00a77deb 4688 return domain;
38717946 4689}
38717946 4690
00a77deb 4691static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4692{
00a77deb 4693 domain_exit(to_dmar_domain(domain));
38717946 4694}
38717946 4695
4c5478c9
JR		 4696static int intel_iommu_attach_device(struct iommu_domain *domain,
4697 struct device *dev)
38717946 4698{
00a77deb 4699 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4700 struct intel_iommu *iommu;
4701 int addr_width;
156baca8 4702 u8 bus, devfn;
faa3d6f5 4703
c875d2c1
AW		 4704 if (device_is_rmrr_locked(dev)) {
4705 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4706 return -EPERM;
4707 }
4708
7207d8f9
DW		 4709 /* normally dev is not mapped */
4710 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4711 struct dmar_domain *old_domain;
4712
1525a29a 4713 old_domain = find_domain(dev);
faa3d6f5 4714 if (old_domain) {
ab8dfe25 4715 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4716 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4717 else
4718 domain_remove_dev_info(old_domain);
62c22167
JR		 4719
4720 if (!domain_type_is_vm_or_si(old_domain) &&
4721 list_empty(&old_domain->devices))
4722 domain_exit(old_domain);
faa3d6f5
WH		 4723 }
4724 }
4725
156baca8 4726 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4727 if (!iommu)
4728 return -ENODEV;
4729
4730 /* check if this iommu agaw is sufficient for max mapped address */
4731 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4732 if (addr_width > cap_mgaw(iommu->cap))
4733 addr_width = cap_mgaw(iommu->cap);
4734
4735 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4736 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4737 "sufficient for the mapped address (%llx)\n",
a99c47a2 4738 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4739 return -EFAULT;
4740 }
a99c47a2
TL		 4741 dmar_domain->gaw = addr_width;
4742
4743 /*
4744 * Knock out extra levels of page tables if necessary
4745 */
4746 while (iommu->agaw < dmar_domain->agaw) {
4747 struct dma_pte *pte;
4748
4749 pte = dmar_domain->pgd;
4750 if (dma_pte_present(pte)) {
25cbff16
SY		 4751 dmar_domain->pgd = (struct dma_pte *)
4752 phys_to_virt(dma_pte_addr(pte));
7a661013 4753 free_pgtable_page(pte);
a99c47a2
TL		 4754 }
4755 dmar_domain->agaw--;
4756 }
fe40f1e0 4757
5913c9bf 4758 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4759}
38717946 4760
4c5478c9
JR		 4761static void intel_iommu_detach_device(struct iommu_domain *domain,
4762 struct device *dev)
38717946 4763{
00a77deb 4764 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4765}
c7151a8d 4766
b146a1c9
JR		 4767static int intel_iommu_map(struct iommu_domain *domain,
4768 unsigned long iova, phys_addr_t hpa,
5009065d 4769 size_t size, int iommu_prot)
faa3d6f5 4770{
00a77deb 4771 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4772 u64 max_addr;
dde57a21 4773 int prot = 0;
faa3d6f5 4774 int ret;
fe40f1e0 4775
dde57a21
JR		 4776 if (iommu_prot & IOMMU_READ)
4777 prot |= DMA_PTE_READ;
4778 if (iommu_prot & IOMMU_WRITE)
4779 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4780 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4781 prot |= DMA_PTE_SNP;
dde57a21 4782
163cc52c 4783 max_addr = iova + size;
dde57a21 4784 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4785 u64 end;
4786
4787 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4788 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4789 if (end < max_addr) {
9f10e5bf 4790 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4791 "sufficient for the mapped address (%llx)\n",
8954da1f 4792 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4793 return -EFAULT;
4794 }
dde57a21 4795 dmar_domain->max_addr = max_addr;
fe40f1e0 4796 }
ad051221
DW		 4797 /* Round up size to next multiple of PAGE_SIZE, if it and
4798 the low bits of hpa would take us onto the next page */
88cb6a74 4799 size = aligned_nrpages(hpa, size);
ad051221
DW		 4800 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4801 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4802 return ret;
38717946 4803}
38717946 4804
5009065d 4805static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4806 unsigned long iova, size_t size)
38717946 4807{
00a77deb 4808 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4809 struct page *freelist = NULL;
4810 struct intel_iommu *iommu;
4811 unsigned long start_pfn, last_pfn;
4812 unsigned int npages;
4813 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4814
4815 /* Cope with horrid API which requires us to unmap more than the
4816 size argument if it happens to be a large-page mapping. */
4817 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4818 BUG();
4819
4820 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4821 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4822
ea8ea460
DW		 4823 start_pfn = iova >> VTD_PAGE_SHIFT;
4824 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4825
4826 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4827
4828 npages = last_pfn - start_pfn + 1;
4829
4830 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4831 iommu = g_iommus[iommu_id];
4832
4833 /*
4834 * find bit position of dmar_domain
4835 */
4836 ndomains = cap_ndoms(iommu->cap);
4837 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4838 if (iommu->domains[num] == dmar_domain)
4839 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4840 npages, !freelist, 0);
4841 }
4842
4843 }
4844
4845 dma_free_pagelist(freelist);
fe40f1e0 4846
163cc52c
DW		 4847 if (dmar_domain->max_addr == iova + size)
4848 dmar_domain->max_addr = iova;
b146a1c9 4849
5cf0a76f 4850 return size;
38717946 4851}
38717946 4852
d14d6577 4853static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4854 dma_addr_t iova)
38717946 4855{
00a77deb 4856 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4857 struct dma_pte *pte;
5cf0a76f 4858 int level = 0;
faa3d6f5 4859 u64 phys = 0;
38717946 4860
5cf0a76f 4861 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4862 if (pte)
faa3d6f5 4863 phys = dma_pte_addr(pte);
38717946 4864
faa3d6f5 4865 return phys;
38717946 4866}
a8bcbb0d 4867
5d587b8d 4868static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4869{
dbb9fd86 4870 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4871 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4872 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4873 return irq_remapping_enabled == 1;
dbb9fd86 4874
5d587b8d 4875 return false;
dbb9fd86
SY		 4876}
4877
abdfdde2
AW		 4878static int intel_iommu_add_device(struct device *dev)
4879{
a5459cfe 4880 struct intel_iommu *iommu;
abdfdde2 4881 struct iommu_group *group;
156baca8 4882 u8 bus, devfn;
70ae6f0d 4883
a5459cfe
AW		 4884 iommu = device_to_iommu(dev, &bus, &devfn);
4885 if (!iommu)
70ae6f0d
AW		 4886 return -ENODEV;
4887
a5459cfe 4888 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4889
e17f9ff4 4890 group = iommu_group_get_for_dev(dev);
783f157b 4891
e17f9ff4
AW		 4892 if (IS_ERR(group))
4893 return PTR_ERR(group);
bcb71abe 4894
abdfdde2 4895 iommu_group_put(group);
e17f9ff4 4896 return 0;
abdfdde2 4897}
70ae6f0d 4898
abdfdde2
AW		 4899static void intel_iommu_remove_device(struct device *dev)
4900{
a5459cfe
AW		 4901 struct intel_iommu *iommu;
4902 u8 bus, devfn;
4903
4904 iommu = device_to_iommu(dev, &bus, &devfn);
4905 if (!iommu)
4906 return;
4907
abdfdde2 4908 iommu_group_remove_device(dev);
a5459cfe
AW		 4909
4910 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4911}
4912
b22f6434 4913static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4914 .capable = intel_iommu_capable,
00a77deb
JR		 4915 .domain_alloc = intel_iommu_domain_alloc,
4916 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4917 .attach_dev = intel_iommu_attach_device,
4918 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4919 .map = intel_iommu_map,
4920 .unmap = intel_iommu_unmap,
315786eb 4921 .map_sg = default_iommu_map_sg,
a8bcbb0d 4922 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4923 .add_device = intel_iommu_add_device,
4924 .remove_device = intel_iommu_remove_device,
6d1c56a9 4925 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4926};
9af88143 4927
9452618e
DV		 4928static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4929{
4930 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4931 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4932 dmar_map_gfx = 0;
4933}
4934
4935DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4936DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4937DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4938DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4939DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4940DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4941DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4942
d34d6517 4943static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4944{
4945 /*
4946 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4947 * but needs it. Same seems to hold for the desktop versions.
9af88143 4948 */
9f10e5bf 4949 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4950 rwbf_quirk = 1;
4951}
4952
4953DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4954DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4955DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4956DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4957DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4958DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4959DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4960
eecfd57f
AJ		 4961#define GGC 0x52
4962#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4963#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4964#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4965#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4966#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4967#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4968#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4969#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4970
d34d6517 4971static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4972{
4973 unsigned short ggc;
4974
eecfd57f 4975 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4976 return;
4977
eecfd57f 4978 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4979 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4980 dmar_map_gfx = 0;
6fbcfb3e
DW		 4981 } else if (dmar_map_gfx) {
4982 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4983 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4984 intel_iommu_strict = 1;
4985 }
9eecabcb
DW		 4986}
4987DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4988DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4989DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4990DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4991
e0fc7e0b
DW		 4992/* On Tylersburg chipsets, some BIOSes have been known to enable the
4993 ISOCH DMAR unit for the Azalia sound device, but not give it any
4994 TLB entries, which causes it to deadlock. Check for that. We do
4995 this in a function called from init_dmars(), instead of in a PCI
4996 quirk, because we don't want to print the obnoxious "BIOS broken"
4997 message if VT-d is actually disabled.
4998*/
4999static void __init check_tylersburg_isoch(void)
5000{
5001 struct pci_dev *pdev;
5002 uint32_t vtisochctrl;
5003
5004 /* If there's no Azalia in the system anyway, forget it. */
5005 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5006 if (!pdev)
5007 return;
5008 pci_dev_put(pdev);
5009
5010 /* System Management Registers. Might be hidden, in which case
5011 we can't do the sanity check. But that's OK, because the
5012 known-broken BIOSes _don't_ actually hide it, so far. */
5013 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5014 if (!pdev)
5015 return;
5016
5017 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5018 pci_dev_put(pdev);
5019 return;
5020 }
5021
5022 pci_dev_put(pdev);
5023
5024 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5025 if (vtisochctrl & 1)
5026 return;
5027
5028 /* Drop all bits other than the number of TLB entries */
5029 vtisochctrl &= 0x1c;
5030
5031 /* If we have the recommended number of TLB entries (16), fine. */
5032 if (vtisochctrl == 0x10)
5033 return;
5034
5035 /* Zero TLB entries? You get to ride the short bus to school. */
5036 if (!vtisochctrl) {
5037 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5038 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5039 dmi_get_system_info(DMI_BIOS_VENDOR),
5040 dmi_get_system_info(DMI_BIOS_VERSION),
5041 dmi_get_system_info(DMI_PRODUCT_VERSION));
5042 iommu_identity_mapping |= IDENTMAP_AZALIA;
5043 return;
5044 }
9f10e5bf
JR		 5045
5046 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5047 vtisochctrl);
5048}
Linux 6.x block layer and io_uring tree(s)