projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Rename domain_remove_one_dev_info()
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
29a27719
JR		 376#define for_each_domain_iommu(idx, domain) \
377 for (idx = 0; idx < g_num_of_iommus; idx++) \
378 if (domain->iommu_refcnt[idx])
379
99126f7c 380struct dmar_domain {
4c923d47 381 int nid; /* node id */
29a27719
JR		 382
383 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
384 /* Refcount of devices per iommu */
385
99126f7c 386
c0e8a6c8
JR		 387 u16 iommu_did[DMAR_UNITS_SUPPORTED];
388 /* Domain ids per IOMMU. Use u16 since
389 * domain ids are 16 bit wide according
390 * to VT-d spec, section 9.3 */
391
00a77deb 392 struct list_head devices; /* all devices' list */
99126f7c
MM		 393 struct iova_domain iovad; /* iova's that belong to this domain */
394
395 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 396 int gaw; /* max guest address width */
397
398 /* adjusted guest address width, 0 is level 2 30-bit */
399 int agaw;
400
3b5410e7 401 int flags; /* flags to find out type of domain */
8e604097
WH		 402
403 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 404 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 405 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 406 int iommu_superpage;/* Level of superpages supported:
407 0 == 4KiB (no superpages), 1 == 2MiB,
408 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 409 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 410 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 411
412 struct iommu_domain domain; /* generic domain data structure for
413 iommu core */
99126f7c
MM		 414};
415
a647dacb
MM		 416/* PCI domain-device relationship */
417struct device_domain_info {
418 struct list_head link; /* link to domain siblings */
419 struct list_head global; /* link to global list */
276dbf99 420 u8 bus; /* PCI bus number */
a647dacb 421 u8 devfn; /* PCI devfn number */
0bcb3e28 422 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 423 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 424 struct dmar_domain *domain; /* pointer to domain */
425};
426
b94e4117
JL		 427struct dmar_rmrr_unit {
428 struct list_head list; /* list of rmrr units */
429 struct acpi_dmar_header *hdr; /* ACPI header */
430 u64 base_address; /* reserved base address*/
431 u64 end_address; /* reserved end address */
832bd858 432 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 433 int devices_cnt; /* target device count */
434};
435
436struct dmar_atsr_unit {
437 struct list_head list; /* list of ATSR units */
438 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 439 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 440 int devices_cnt; /* target device count */
441 u8 include_all:1; /* include all ports */
442};
443
444static LIST_HEAD(dmar_atsr_units);
445static LIST_HEAD(dmar_rmrr_units);
446
447#define for_each_rmrr_units(rmrr) \
448 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
449
5e0d2a6f 450static void flush_unmaps_timeout(unsigned long data);
451
b707cb02 452static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 453
80b20dd8 454#define HIGH_WATER_MARK 250
455struct deferred_flush_tables {
456 int next;
457 struct iova *iova[HIGH_WATER_MARK];
458 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 459 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 460};
461
462static struct deferred_flush_tables *deferred_flush;
463
5e0d2a6f 464/* bitmap for indexing intel_iommus */
5e0d2a6f 465static int g_num_of_iommus;
466
467static DEFINE_SPINLOCK(async_umap_flush_lock);
468static LIST_HEAD(unmaps_to_do);
469
470static int timer_on;
471static long list_size;
5e0d2a6f 472
92d03cc8 473static void domain_exit(struct dmar_domain *domain);
ba395927 474static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 475static void dmar_remove_one_dev_info(struct dmar_domain *domain,
476 struct device *dev);
92d03cc8 477static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 478 struct device *dev);
2a46ddf7
JL		 479static int domain_detach_iommu(struct dmar_domain *domain,
480 struct intel_iommu *iommu);
ba395927 481
d3f13810 482#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 483int dmar_disabled = 0;
484#else
485int dmar_disabled = 1;
d3f13810 486#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 487
8bc1f85c
ED		 488int intel_iommu_enabled = 0;
489EXPORT_SYMBOL_GPL(intel_iommu_enabled);
490
2d9e667e 491static int dmar_map_gfx = 1;
7d3b03ce 492static int dmar_forcedac;
5e0d2a6f 493static int intel_iommu_strict;
6dd9a7c7 494static int intel_iommu_superpage = 1;
c83b2f20
DW		 495static int intel_iommu_ecs = 1;
496
497/* We only actually use ECS when PASID support (on the new bit 40)
498 * is also advertised. Some early implementations — the ones with
499 * PASID support on bit 28 — have issues even when we *only* use
500 * extended root/context tables. */
501#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
502 ecap_pasid(iommu->ecap))
ba395927 503
c0771df8
DW		 504int intel_iommu_gfx_mapped;
505EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
506
ba395927
KA		 507#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
508static DEFINE_SPINLOCK(device_domain_lock);
509static LIST_HEAD(device_domain_list);
510
b22f6434 511static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 512
4158c2ec
JR		 513static bool translation_pre_enabled(struct intel_iommu *iommu)
514{
515 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
516}
517
091d42e4
JR		 518static void clear_translation_pre_enabled(struct intel_iommu *iommu)
519{
520 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
521}
522
4158c2ec
JR		 523static void init_translation_status(struct intel_iommu *iommu)
524{
525 u32 gsts;
526
527 gsts = readl(iommu->reg + DMAR_GSTS_REG);
528 if (gsts & DMA_GSTS_TES)
529 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
530}
531
00a77deb
JR		 532/* Convert generic 'struct iommu_domain to private struct dmar_domain */
533static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
534{
535 return container_of(dom, struct dmar_domain, domain);
536}
537
ba395927
KA		 538static int __init intel_iommu_setup(char *str)
539{
540 if (!str)
541 return -EINVAL;
542 while (*str) {
0cd5c3c8
KM		 543 if (!strncmp(str, "on", 2)) {
544 dmar_disabled = 0;
9f10e5bf 545 pr_info("IOMMU enabled\n");
0cd5c3c8 546 } else if (!strncmp(str, "off", 3)) {
ba395927 547 dmar_disabled = 1;
9f10e5bf 548 pr_info("IOMMU disabled\n");
ba395927
KA		 549 } else if (!strncmp(str, "igfx_off", 8)) {
550 dmar_map_gfx = 0;
9f10e5bf 551 pr_info("Disable GFX device mapping\n");
7d3b03ce 552 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 553 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 554 dmar_forcedac = 1;
5e0d2a6f 555 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 556 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 557 intel_iommu_strict = 1;
6dd9a7c7 558 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 559 pr_info("Disable supported super page\n");
6dd9a7c7 560 intel_iommu_superpage = 0;
c83b2f20
DW		 561 } else if (!strncmp(str, "ecs_off", 7)) {
562 printk(KERN_INFO
563 "Intel-IOMMU: disable extended context table support\n");
564 intel_iommu_ecs = 0;
ba395927
KA		 565 }
566
567 str += strcspn(str, ",");
568 while (*str == ',')
569 str++;
570 }
571 return 0;
572}
573__setup("intel_iommu=", intel_iommu_setup);
574
575static struct kmem_cache *iommu_domain_cache;
576static struct kmem_cache *iommu_devinfo_cache;
ba395927 577
9452d5bf
JR		 578static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
579{
8bf47816
JR		 580 struct dmar_domain **domains;
581 int idx = did >> 8;
582
583 domains = iommu->domains[idx];
584 if (!domains)
585 return NULL;
586
587 return domains[did & 0xff];
9452d5bf
JR		 588}
589
590static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
591 struct dmar_domain *domain)
592{
8bf47816
JR		 593 struct dmar_domain **domains;
594 int idx = did >> 8;
595
596 if (!iommu->domains[idx]) {
597 size_t size = 256 * sizeof(struct dmar_domain *);
598 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
599 }
600
601 domains = iommu->domains[idx];
602 if (WARN_ON(!domains))
603 return;
604 else
605 domains[did & 0xff] = domain;
9452d5bf
JR		 606}
607
4c923d47 608static inline void *alloc_pgtable_page(int node)
eb3fa7cb 609{
4c923d47
SS		 610 struct page *page;
611 void *vaddr = NULL;
eb3fa7cb 612
4c923d47
SS		 613 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
614 if (page)
615 vaddr = page_address(page);
eb3fa7cb 616 return vaddr;
ba395927
KA		 617}
618
619static inline void free_pgtable_page(void *vaddr)
620{
621 free_page((unsigned long)vaddr);
622}
623
624static inline void *alloc_domain_mem(void)
625{
354bb65e 626 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 627}
628
38717946 629static void free_domain_mem(void *vaddr)
ba395927
KA		 630{
631 kmem_cache_free(iommu_domain_cache, vaddr);
632}
633
634static inline void * alloc_devinfo_mem(void)
635{
354bb65e 636 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 637}
638
639static inline void free_devinfo_mem(void *vaddr)
640{
641 kmem_cache_free(iommu_devinfo_cache, vaddr);
642}
643
ab8dfe25
JL		 644static inline int domain_type_is_vm(struct dmar_domain *domain)
645{
646 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
647}
648
28ccce0d
JR		 649static inline int domain_type_is_si(struct dmar_domain *domain)
650{
651 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
652}
653
ab8dfe25
JL		 654static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
655{
656 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
657 DOMAIN_FLAG_STATIC_IDENTITY);
658}
1b573683 659
162d1b10
JL		 660static inline int domain_pfn_supported(struct dmar_domain *domain,
661 unsigned long pfn)
662{
663 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
664
665 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
666}
667
4ed0d3e6 668static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 669{
670 unsigned long sagaw;
671 int agaw = -1;
672
673 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 674 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 675 agaw >= 0; agaw--) {
676 if (test_bit(agaw, &sagaw))
677 break;
678 }
679
680 return agaw;
681}
682
4ed0d3e6
FY		 683/*
684 * Calculate max SAGAW for each iommu.
685 */
686int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
687{
688 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
689}
690
691/*
692 * calculate agaw for each iommu.
693 * "SAGAW" may be different across iommus, use a default agaw, and
694 * get a supported less agaw for iommus that don't support the default agaw.
695 */
696int iommu_calculate_agaw(struct intel_iommu *iommu)
697{
698 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
699}
700
2c2e2c38 701/* This functionin only returns single iommu in a domain */
8c11e798
WH		 702static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
703{
704 int iommu_id;
705
2c2e2c38 706 /* si_domain and vm domain should not get here. */
ab8dfe25 707 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 708 for_each_domain_iommu(iommu_id, domain)
709 break;
710
8c11e798
WH		 711 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
712 return NULL;
713
714 return g_iommus[iommu_id];
715}
716
8e604097
WH		 717static void domain_update_iommu_coherency(struct dmar_domain *domain)
718{
d0501960
DW		 719 struct dmar_drhd_unit *drhd;
720 struct intel_iommu *iommu;
2f119c78
QL		 721 bool found = false;
722 int i;
2e12bc29 723
d0501960 724 domain->iommu_coherency = 1;
8e604097 725
29a27719 726 for_each_domain_iommu(i, domain) {
2f119c78 727 found = true;
8e604097
WH		 728 if (!ecap_coherent(g_iommus[i]->ecap)) {
729 domain->iommu_coherency = 0;
730 break;
731 }
8e604097 732 }
d0501960
DW		 733 if (found)
734 return;
735
736 /* No hardware attached; use lowest common denominator */
737 rcu_read_lock();
738 for_each_active_iommu(iommu, drhd) {
739 if (!ecap_coherent(iommu->ecap)) {
740 domain->iommu_coherency = 0;
741 break;
742 }
743 }
744 rcu_read_unlock();
8e604097
WH		 745}
746
161f6934 747static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 748{
161f6934
JL		 749 struct dmar_drhd_unit *drhd;
750 struct intel_iommu *iommu;
751 int ret = 1;
58c610bd 752
161f6934
JL		 753 rcu_read_lock();
754 for_each_active_iommu(iommu, drhd) {
755 if (iommu != skip) {
756 if (!ecap_sc_support(iommu->ecap)) {
757 ret = 0;
758 break;
759 }
58c610bd 760 }
58c610bd 761 }
161f6934
JL		 762 rcu_read_unlock();
763
764 return ret;
58c610bd
SY		 765}
766
161f6934 767static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 768{
8140a95d 769 struct dmar_drhd_unit *drhd;
161f6934 770 struct intel_iommu *iommu;
8140a95d 771 int mask = 0xf;
6dd9a7c7
YS		 772
773 if (!intel_iommu_superpage) {
161f6934 774 return 0;
6dd9a7c7
YS		 775 }
776
8140a95d 777 /* set iommu_superpage to the smallest common denominator */
0e242612 778 rcu_read_lock();
8140a95d 779 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 780 if (iommu != skip) {
781 mask &= cap_super_page_val(iommu->cap);
782 if (!mask)
783 break;
6dd9a7c7
YS		 784 }
785 }
0e242612
JL		 786 rcu_read_unlock();
787
161f6934 788 return fls(mask);
6dd9a7c7
YS		 789}
790
58c610bd
SY		 791/* Some capabilities may be different across iommus */
792static void domain_update_iommu_cap(struct dmar_domain *domain)
793{
794 domain_update_iommu_coherency(domain);
161f6934
JL		 795 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
796 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 797}
798
03ecc32c
DW		 799static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
800 u8 bus, u8 devfn, int alloc)
801{
802 struct root_entry *root = &iommu->root_entry[bus];
803 struct context_entry *context;
804 u64 *entry;
805
c83b2f20 806 if (ecs_enabled(iommu)) {
03ecc32c
DW		 807 if (devfn >= 0x80) {
808 devfn -= 0x80;
809 entry = &root->hi;
810 }
811 devfn *= 2;
812 }
813 entry = &root->lo;
814 if (*entry & 1)
815 context = phys_to_virt(*entry & VTD_PAGE_MASK);
816 else {
817 unsigned long phy_addr;
818 if (!alloc)
819 return NULL;
820
821 context = alloc_pgtable_page(iommu->node);
822 if (!context)
823 return NULL;
824
825 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
826 phy_addr = virt_to_phys((void *)context);
827 *entry = phy_addr | 1;
828 __iommu_flush_cache(iommu, entry, sizeof(*entry));
829 }
830 return &context[devfn];
831}
832
4ed6a540
DW		 833static int iommu_dummy(struct device *dev)
834{
835 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
836}
837
156baca8 838static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 839{
840 struct dmar_drhd_unit *drhd = NULL;
b683b230 841 struct intel_iommu *iommu;
156baca8
DW		 842 struct device *tmp;
843 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 844 u16 segment = 0;
c7151a8d
WH		 845 int i;
846
4ed6a540
DW		 847 if (iommu_dummy(dev))
848 return NULL;
849
156baca8
DW		 850 if (dev_is_pci(dev)) {
851 pdev = to_pci_dev(dev);
852 segment = pci_domain_nr(pdev->bus);
ca5b74d2 853 } else if (has_acpi_companion(dev))
156baca8
DW		 854 dev = &ACPI_COMPANION(dev)->dev;
855
0e242612 856 rcu_read_lock();
b683b230 857 for_each_active_iommu(iommu, drhd) {
156baca8 858 if (pdev && segment != drhd->segment)
276dbf99 859 continue;
c7151a8d 860
b683b230 861 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 862 drhd->devices_cnt, i, tmp) {
863 if (tmp == dev) {
864 *bus = drhd->devices[i].bus;
865 *devfn = drhd->devices[i].devfn;
b683b230 866 goto out;
156baca8
DW		 867 }
868
869 if (!pdev || !dev_is_pci(tmp))
870 continue;
871
872 ptmp = to_pci_dev(tmp);
873 if (ptmp->subordinate &&
874 ptmp->subordinate->number <= pdev->bus->number &&
875 ptmp->subordinate->busn_res.end >= pdev->bus->number)
876 goto got_pdev;
924b6231 877 }
c7151a8d 878
156baca8
DW		 879 if (pdev && drhd->include_all) {
880 got_pdev:
881 *bus = pdev->bus->number;
882 *devfn = pdev->devfn;
b683b230 883 goto out;
156baca8 884 }
c7151a8d 885 }
b683b230 886 iommu = NULL;
156baca8 887 out:
0e242612 888 rcu_read_unlock();
c7151a8d 889
b683b230 890 return iommu;
c7151a8d
WH		 891}
892
5331fe6f
WH		 893static void domain_flush_cache(struct dmar_domain *domain,
894 void *addr, int size)
895{
896 if (!domain->iommu_coherency)
897 clflush_cache_range(addr, size);
898}
899
ba395927
KA		 900static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
901{
ba395927 902 struct context_entry *context;
03ecc32c 903 int ret = 0;
ba395927
KA		 904 unsigned long flags;
905
906 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 907 context = iommu_context_addr(iommu, bus, devfn, 0);
908 if (context)
909 ret = context_present(context);
ba395927
KA		 910 spin_unlock_irqrestore(&iommu->lock, flags);
911 return ret;
912}
913
914static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
915{
ba395927
KA		 916 struct context_entry *context;
917 unsigned long flags;
918
919 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 920 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 921 if (context) {
03ecc32c
DW		 922 context_clear_entry(context);
923 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 924 }
925 spin_unlock_irqrestore(&iommu->lock, flags);
926}
927
928static void free_context_table(struct intel_iommu *iommu)
929{
ba395927
KA		 930 int i;
931 unsigned long flags;
932 struct context_entry *context;
933
934 spin_lock_irqsave(&iommu->lock, flags);
935 if (!iommu->root_entry) {
936 goto out;
937 }
938 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 939 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 940 if (context)
941 free_pgtable_page(context);
03ecc32c 942
c83b2f20 943 if (!ecs_enabled(iommu))
03ecc32c
DW		 944 continue;
945
946 context = iommu_context_addr(iommu, i, 0x80, 0);
947 if (context)
948 free_pgtable_page(context);
949
ba395927
KA		 950 }
951 free_pgtable_page(iommu->root_entry);
952 iommu->root_entry = NULL;
953out:
954 spin_unlock_irqrestore(&iommu->lock, flags);
955}
956
b026fd28 957static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 958 unsigned long pfn, int *target_level)
ba395927 959{
ba395927
KA		 960 struct dma_pte *parent, *pte = NULL;
961 int level = agaw_to_level(domain->agaw);
4399c8bf 962 int offset;
ba395927
KA		 963
964 BUG_ON(!domain->pgd);
f9423606 965
162d1b10 966 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 967 /* Address beyond IOMMU's addressing capabilities. */
968 return NULL;
969
ba395927
KA		 970 parent = domain->pgd;
971
5cf0a76f 972 while (1) {
ba395927
KA		 973 void *tmp_page;
974
b026fd28 975 offset = pfn_level_offset(pfn, level);
ba395927 976 pte = &parent[offset];
5cf0a76f 977 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 978 break;
5cf0a76f 979 if (level == *target_level)
ba395927
KA		 980 break;
981
19c239ce 982 if (!dma_pte_present(pte)) {
c85994e4
DW		 983 uint64_t pteval;
984
4c923d47 985 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 986
206a73c1 987 if (!tmp_page)
ba395927 988 return NULL;
206a73c1 989
c85994e4 990 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 991 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 992 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 993 /* Someone else set it while we were thinking; use theirs. */
994 free_pgtable_page(tmp_page);
effad4b5 995 else
c85994e4 996 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 997 }
5cf0a76f
DW		 998 if (level == 1)
999 break;
1000
19c239ce 1001 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1002 level--;
1003 }
1004
5cf0a76f
DW		 1005 if (!*target_level)
1006 *target_level = level;
1007
ba395927
KA		 1008 return pte;
1009}
1010
6dd9a7c7 1011
ba395927 1012/* return address's pte at specific level */
90dcfb5e
DW		 1013static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1014 unsigned long pfn,
6dd9a7c7 1015 int level, int *large_page)
ba395927
KA		 1016{
1017 struct dma_pte *parent, *pte = NULL;
1018 int total = agaw_to_level(domain->agaw);
1019 int offset;
1020
1021 parent = domain->pgd;
1022 while (level <= total) {
90dcfb5e 1023 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1024 pte = &parent[offset];
1025 if (level == total)
1026 return pte;
1027
6dd9a7c7
YS		 1028 if (!dma_pte_present(pte)) {
1029 *large_page = total;
ba395927 1030 break;
6dd9a7c7
YS		 1031 }
1032
e16922af 1033 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1034 *large_page = total;
1035 return pte;
1036 }
1037
19c239ce 1038 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1039 total--;
1040 }
1041 return NULL;
1042}
1043
ba395927 1044/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1045static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1046 unsigned long start_pfn,
1047 unsigned long last_pfn)
ba395927 1048{
6dd9a7c7 1049 unsigned int large_page = 1;
310a5ab9 1050 struct dma_pte *first_pte, *pte;
66eae846 1051
162d1b10
JL		 1052 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1053 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1054 BUG_ON(start_pfn > last_pfn);
ba395927 1055
04b18e65 1056 /* we don't need lock here; nobody else touches the iova range */
59c36286 1057 do {
6dd9a7c7
YS		 1058 large_page = 1;
1059 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1060 if (!pte) {
6dd9a7c7 1061 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1062 continue;
1063 }
6dd9a7c7 1064 do {
310a5ab9 1065 dma_clear_pte(pte);
6dd9a7c7 1066 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1067 pte++;
75e6bf96
DW		 1068 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1069
310a5ab9
DW		 1070 domain_flush_cache(domain, first_pte,
1071 (void *)pte - (void *)first_pte);
59c36286
DW		 1072
1073 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1074}
1075
3269ee0b
AW		 1076static void dma_pte_free_level(struct dmar_domain *domain, int level,
1077 struct dma_pte *pte, unsigned long pfn,
1078 unsigned long start_pfn, unsigned long last_pfn)
1079{
1080 pfn = max(start_pfn, pfn);
1081 pte = &pte[pfn_level_offset(pfn, level)];
1082
1083 do {
1084 unsigned long level_pfn;
1085 struct dma_pte *level_pte;
1086
1087 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1088 goto next;
1089
1090 level_pfn = pfn & level_mask(level - 1);
1091 level_pte = phys_to_virt(dma_pte_addr(pte));
1092
1093 if (level > 2)
1094 dma_pte_free_level(domain, level - 1, level_pte,
1095 level_pfn, start_pfn, last_pfn);
1096
1097 /* If range covers entire pagetable, free it */
1098 if (!(start_pfn > level_pfn ||
08336fd2 1099 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1100 dma_clear_pte(pte);
1101 domain_flush_cache(domain, pte, sizeof(*pte));
1102 free_pgtable_page(level_pte);
1103 }
1104next:
1105 pfn += level_size(level);
1106 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1107}
1108
ba395927
KA		 1109/* free page table pages. last level pte should already be cleared */
1110static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1111 unsigned long start_pfn,
1112 unsigned long last_pfn)
ba395927 1113{
162d1b10
JL		 1114 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1115 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1116 BUG_ON(start_pfn > last_pfn);
ba395927 1117
d41a4adb
JL		 1118 dma_pte_clear_range(domain, start_pfn, last_pfn);
1119
f3a0a52f 1120 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1121 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1122 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1123
ba395927 1124 /* free pgd */
d794dc9b 1125 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1126 free_pgtable_page(domain->pgd);
1127 domain->pgd = NULL;
1128 }
1129}
1130
ea8ea460
DW		 1131/* When a page at a given level is being unlinked from its parent, we don't
1132 need to *modify* it at all. All we need to do is make a list of all the
1133 pages which can be freed just as soon as we've flushed the IOTLB and we
1134 know the hardware page-walk will no longer touch them.
1135 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1136 be freed. */
1137static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1138 int level, struct dma_pte *pte,
1139 struct page *freelist)
1140{
1141 struct page *pg;
1142
1143 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1144 pg->freelist = freelist;
1145 freelist = pg;
1146
1147 if (level == 1)
1148 return freelist;
1149
adeb2590
JL		 1150 pte = page_address(pg);
1151 do {
ea8ea460
DW		 1152 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1153 freelist = dma_pte_list_pagetables(domain, level - 1,
1154 pte, freelist);
adeb2590
JL		 1155 pte++;
1156 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1157
1158 return freelist;
1159}
1160
1161static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1162 struct dma_pte *pte, unsigned long pfn,
1163 unsigned long start_pfn,
1164 unsigned long last_pfn,
1165 struct page *freelist)
1166{
1167 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1168
1169 pfn = max(start_pfn, pfn);
1170 pte = &pte[pfn_level_offset(pfn, level)];
1171
1172 do {
1173 unsigned long level_pfn;
1174
1175 if (!dma_pte_present(pte))
1176 goto next;
1177
1178 level_pfn = pfn & level_mask(level);
1179
1180 /* If range covers entire pagetable, free it */
1181 if (start_pfn <= level_pfn &&
1182 last_pfn >= level_pfn + level_size(level) - 1) {
1183 /* These suborbinate page tables are going away entirely. Don't
1184 bother to clear them; we're just going to *free* them. */
1185 if (level > 1 && !dma_pte_superpage(pte))
1186 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1187
1188 dma_clear_pte(pte);
1189 if (!first_pte)
1190 first_pte = pte;
1191 last_pte = pte;
1192 } else if (level > 1) {
1193 /* Recurse down into a level that isn't *entirely* obsolete */
1194 freelist = dma_pte_clear_level(domain, level - 1,
1195 phys_to_virt(dma_pte_addr(pte)),
1196 level_pfn, start_pfn, last_pfn,
1197 freelist);
1198 }
1199next:
1200 pfn += level_size(level);
1201 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1202
1203 if (first_pte)
1204 domain_flush_cache(domain, first_pte,
1205 (void *)++last_pte - (void *)first_pte);
1206
1207 return freelist;
1208}
1209
1210/* We can't just free the pages because the IOMMU may still be walking
1211 the page tables, and may have cached the intermediate levels. The
1212 pages can only be freed after the IOTLB flush has been done. */
1213struct page *domain_unmap(struct dmar_domain *domain,
1214 unsigned long start_pfn,
1215 unsigned long last_pfn)
1216{
ea8ea460
DW		 1217 struct page *freelist = NULL;
1218
162d1b10
JL		 1219 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1220 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1221 BUG_ON(start_pfn > last_pfn);
1222
1223 /* we don't need lock here; nobody else touches the iova range */
1224 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1225 domain->pgd, 0, start_pfn, last_pfn, NULL);
1226
1227 /* free pgd */
1228 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1229 struct page *pgd_page = virt_to_page(domain->pgd);
1230 pgd_page->freelist = freelist;
1231 freelist = pgd_page;
1232
1233 domain->pgd = NULL;
1234 }
1235
1236 return freelist;
1237}
1238
1239void dma_free_pagelist(struct page *freelist)
1240{
1241 struct page *pg;
1242
1243 while ((pg = freelist)) {
1244 freelist = pg->freelist;
1245 free_pgtable_page(page_address(pg));
1246 }
1247}
1248
ba395927
KA		 1249/* iommu handling */
1250static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1251{
1252 struct root_entry *root;
1253 unsigned long flags;
1254
4c923d47 1255 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1256 if (!root) {
9f10e5bf 1257 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1258 iommu->name);
ba395927 1259 return -ENOMEM;
ffebeb46 1260 }
ba395927 1261
5b6985ce 1262 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1263
1264 spin_lock_irqsave(&iommu->lock, flags);
1265 iommu->root_entry = root;
1266 spin_unlock_irqrestore(&iommu->lock, flags);
1267
1268 return 0;
1269}
1270
ba395927
KA		 1271static void iommu_set_root_entry(struct intel_iommu *iommu)
1272{
03ecc32c 1273 u64 addr;
c416daa9 1274 u32 sts;
ba395927
KA		 1275 unsigned long flag;
1276
03ecc32c 1277 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1278 if (ecs_enabled(iommu))
03ecc32c 1279 addr |= DMA_RTADDR_RTT;
ba395927 1280
1f5b3c3f 1281 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1282 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1283
c416daa9 1284 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1285
1286 /* Make sure hardware complete it */
1287 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1288 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1289
1f5b3c3f 1290 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1291}
1292
1293static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1294{
1295 u32 val;
1296 unsigned long flag;
1297
9af88143 1298 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1299 return;
ba395927 1300
1f5b3c3f 1301 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1302 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1303
1304 /* Make sure hardware complete it */
1305 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1306 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1307
1f5b3c3f 1308 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1309}
1310
1311/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1312static void __iommu_flush_context(struct intel_iommu *iommu,
1313 u16 did, u16 source_id, u8 function_mask,
1314 u64 type)
ba395927
KA		 1315{
1316 u64 val = 0;
1317 unsigned long flag;
1318
ba395927
KA		 1319 switch (type) {
1320 case DMA_CCMD_GLOBAL_INVL:
1321 val = DMA_CCMD_GLOBAL_INVL;
1322 break;
1323 case DMA_CCMD_DOMAIN_INVL:
1324 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1325 break;
1326 case DMA_CCMD_DEVICE_INVL:
1327 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1328 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1329 break;
1330 default:
1331 BUG();
1332 }
1333 val |= DMA_CCMD_ICC;
1334
1f5b3c3f 1335 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1336 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1337
1338 /* Make sure hardware complete it */
1339 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1340 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1341
1f5b3c3f 1342 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1343}
1344
ba395927 1345/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1346static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1347 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1348{
1349 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1350 u64 val = 0, val_iva = 0;
1351 unsigned long flag;
1352
ba395927
KA		 1353 switch (type) {
1354 case DMA_TLB_GLOBAL_FLUSH:
1355 /* global flush doesn't need set IVA_REG */
1356 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1357 break;
1358 case DMA_TLB_DSI_FLUSH:
1359 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1360 break;
1361 case DMA_TLB_PSI_FLUSH:
1362 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1363 /* IH bit is passed in as part of address */
ba395927
KA		 1364 val_iva = size_order | addr;
1365 break;
1366 default:
1367 BUG();
1368 }
1369 /* Note: set drain read/write */
1370#if 0
1371 /*
1372 * This is probably to be super secure.. Looks like we can
1373 * ignore it without any impact.
1374 */
1375 if (cap_read_drain(iommu->cap))
1376 val |= DMA_TLB_READ_DRAIN;
1377#endif
1378 if (cap_write_drain(iommu->cap))
1379 val |= DMA_TLB_WRITE_DRAIN;
1380
1f5b3c3f 1381 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1382 /* Note: Only uses first TLB reg currently */
1383 if (val_iva)
1384 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1385 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1386
1387 /* Make sure hardware complete it */
1388 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1389 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1390
1f5b3c3f 1391 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1392
1393 /* check IOTLB invalidation granularity */
1394 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1395 pr_err("Flush IOTLB failed\n");
ba395927 1396 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1397 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1398 (unsigned long long)DMA_TLB_IIRG(type),
1399 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1400}
1401
64ae892b
DW		 1402static struct device_domain_info *
1403iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1404 u8 bus, u8 devfn)
93a23a72 1405{
2f119c78 1406 bool found = false;
93a23a72
YZ		 1407 unsigned long flags;
1408 struct device_domain_info *info;
0bcb3e28 1409 struct pci_dev *pdev;
93a23a72
YZ		 1410
1411 if (!ecap_dev_iotlb_support(iommu->ecap))
1412 return NULL;
1413
1414 if (!iommu->qi)
1415 return NULL;
1416
1417 spin_lock_irqsave(&device_domain_lock, flags);
1418 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1419 if (info->iommu == iommu && info->bus == bus &&
1420 info->devfn == devfn) {
2f119c78 1421 found = true;
93a23a72
YZ		 1422 break;
1423 }
1424 spin_unlock_irqrestore(&device_domain_lock, flags);
1425
0bcb3e28 1426 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1427 return NULL;
1428
0bcb3e28
DW		 1429 pdev = to_pci_dev(info->dev);
1430
1431 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1432 return NULL;
1433
0bcb3e28 1434 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1435 return NULL;
1436
93a23a72
YZ		 1437 return info;
1438}
1439
1440static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1441{
0bcb3e28 1442 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1443 return;
1444
0bcb3e28 1445 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1446}
1447
1448static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1449{
0bcb3e28
DW		 1450 if (!info->dev || !dev_is_pci(info->dev) ||
1451 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1452 return;
1453
0bcb3e28 1454 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1455}
1456
1457static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1458 u64 addr, unsigned mask)
1459{
1460 u16 sid, qdep;
1461 unsigned long flags;
1462 struct device_domain_info *info;
1463
1464 spin_lock_irqsave(&device_domain_lock, flags);
1465 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1466 struct pci_dev *pdev;
1467 if (!info->dev || !dev_is_pci(info->dev))
1468 continue;
1469
1470 pdev = to_pci_dev(info->dev);
1471 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1472 continue;
1473
1474 sid = info->bus << 8 | info->devfn;
0bcb3e28 1475 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1476 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1477 }
1478 spin_unlock_irqrestore(&device_domain_lock, flags);
1479}
1480
a1ddcbe9
JR		 1481static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1482 struct dmar_domain *domain,
1483 unsigned long pfn, unsigned int pages,
1484 int ih, int map)
ba395927 1485{
9dd2fe89 1486 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1487 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1488 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1489
ba395927
KA		 1490 BUG_ON(pages == 0);
1491
ea8ea460
DW		 1492 if (ih)
1493 ih = 1 << 6;
ba395927 1494 /*
9dd2fe89
YZ		 1495 * Fallback to domain selective flush if no PSI support or the size is
1496 * too big.
ba395927
KA		 1497 * PSI requires page size to be 2 ^ x, and the base address is naturally
1498 * aligned to the size
1499 */
9dd2fe89
YZ		 1500 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1501 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1502 DMA_TLB_DSI_FLUSH);
9dd2fe89 1503 else
ea8ea460 1504 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1505 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1506
1507 /*
82653633
NA		 1508 * In caching mode, changes of pages from non-present to present require
1509 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1510 */
82653633 1511 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1512 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1513 addr, mask);
ba395927
KA		 1514}
1515
f8bab735 1516static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1517{
1518 u32 pmen;
1519 unsigned long flags;
1520
1f5b3c3f 1521 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1522 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1523 pmen &= ~DMA_PMEN_EPM;
1524 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1525
1526 /* wait for the protected region status bit to clear */
1527 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1528 readl, !(pmen & DMA_PMEN_PRS), pmen);
1529
1f5b3c3f 1530 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1531}
1532
2a41ccee 1533static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1534{
1535 u32 sts;
1536 unsigned long flags;
1537
1f5b3c3f 1538 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1539 iommu->gcmd |= DMA_GCMD_TE;
1540 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1541
1542 /* Make sure hardware complete it */
1543 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1544 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1545
1f5b3c3f 1546 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1547}
1548
2a41ccee 1549static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1550{
1551 u32 sts;
1552 unsigned long flag;
1553
1f5b3c3f 1554 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1555 iommu->gcmd &= ~DMA_GCMD_TE;
1556 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1557
1558 /* Make sure hardware complete it */
1559 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1560 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1561
1f5b3c3f 1562 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1563}
1564
3460a6d9 1565
ba395927
KA		 1566static int iommu_init_domains(struct intel_iommu *iommu)
1567{
8bf47816
JR		 1568 u32 ndomains, nlongs;
1569 size_t size;
ba395927
KA		 1570
1571 ndomains = cap_ndoms(iommu->cap);
8bf47816 1572 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1573 iommu->name, ndomains);
ba395927
KA		 1574 nlongs = BITS_TO_LONGS(ndomains);
1575
94a91b50
DD		 1576 spin_lock_init(&iommu->lock);
1577
ba395927
KA		 1578 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1579 if (!iommu->domain_ids) {
9f10e5bf
JR		 1580 pr_err("%s: Allocating domain id array failed\n",
1581 iommu->name);
ba395927
KA		 1582 return -ENOMEM;
1583 }
8bf47816
JR		 1584
1585 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1586 iommu->domains = kzalloc(size, GFP_KERNEL);
1587
1588 if (iommu->domains) {
1589 size = 256 * sizeof(struct dmar_domain *);
1590 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1591 }
1592
1593 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1594 pr_err("%s: Allocating domain array failed\n",
1595 iommu->name);
852bdb04 1596 kfree(iommu->domain_ids);
8bf47816 1597 kfree(iommu->domains);
852bdb04 1598 iommu->domain_ids = NULL;
8bf47816 1599 iommu->domains = NULL;
ba395927
KA		 1600 return -ENOMEM;
1601 }
1602
8bf47816
JR		 1603
1604
ba395927 1605 /*
c0e8a6c8
JR		 1606 * If Caching mode is set, then invalid translations are tagged
1607 * with domain-id 0, hence we need to pre-allocate it. We also
1608 * use domain-id 0 as a marker for non-allocated domain-id, so
1609 * make sure it is not used for a real domain.
ba395927 1610 */
c0e8a6c8
JR		 1611 set_bit(0, iommu->domain_ids);
1612
ba395927
KA		 1613 return 0;
1614}
ba395927 1615
ffebeb46 1616static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1617{
29a27719 1618 struct device_domain_info *info, *tmp;
ba395927 1619
29a27719
JR		 1620 if (!iommu->domains || !iommu->domain_ids)
1621 return;
a4eaa86c 1622
29a27719
JR		 1623 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1624 struct dmar_domain *domain;
1625
1626 if (info->iommu != iommu)
1627 continue;
1628
1629 if (!info->dev || !info->domain)
1630 continue;
1631
1632 domain = info->domain;
1633
e6de0f8d 1634 dmar_remove_one_dev_info(domain, info->dev);
29a27719
JR		 1635
1636 if (!domain_type_is_vm_or_si(domain))
1637 domain_exit(domain);
ba395927
KA		 1638 }
1639
1640 if (iommu->gcmd & DMA_GCMD_TE)
1641 iommu_disable_translation(iommu);
ffebeb46 1642}
ba395927 1643
ffebeb46
JL		 1644static void free_dmar_iommu(struct intel_iommu *iommu)
1645{
1646 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1647 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1648 int i;
1649
1650 for (i = 0; i < elems; i++)
1651 kfree(iommu->domains[i]);
ffebeb46
JL		 1652 kfree(iommu->domains);
1653 kfree(iommu->domain_ids);
1654 iommu->domains = NULL;
1655 iommu->domain_ids = NULL;
1656 }
ba395927 1657
d9630fe9
WH		 1658 g_iommus[iommu->seq_id] = NULL;
1659
ba395927
KA		 1660 /* free context mapping */
1661 free_context_table(iommu);
ba395927
KA		 1662}
1663
ab8dfe25 1664static struct dmar_domain *alloc_domain(int flags)
ba395927 1665{
ba395927 1666 struct dmar_domain *domain;
ba395927
KA		 1667
1668 domain = alloc_domain_mem();
1669 if (!domain)
1670 return NULL;
1671
ab8dfe25 1672 memset(domain, 0, sizeof(*domain));
4c923d47 1673 domain->nid = -1;
ab8dfe25 1674 domain->flags = flags;
92d03cc8
JL		 1675 spin_lock_init(&domain->iommu_lock);
1676 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1677
1678 return domain;
1679}
1680
fb170fb4
JL		 1681static int __iommu_attach_domain(struct dmar_domain *domain,
1682 struct intel_iommu *iommu)
2c2e2c38
FY		 1683{
1684 int num;
1685 unsigned long ndomains;
2c2e2c38 1686
e2411427
JR		 1687 num = domain->iommu_did[iommu->seq_id];
1688 if (num)
1689 return num;
1690
ba395927 1691 ndomains = cap_ndoms(iommu->cap);
e2411427
JR		 1692 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1693
fb170fb4
JL		 1694 if (num < ndomains) {
1695 set_bit(num, iommu->domain_ids);
9452d5bf 1696 set_iommu_domain(iommu, num, domain);
c0e8a6c8 1697 domain->iommu_did[iommu->seq_id] = num;
fb170fb4
JL		 1698 } else {
1699 num = -ENOSPC;
ba395927
KA		 1700 }
1701
e2411427
JR		 1702 if (num < 0)
1703 pr_err("%s: No free domain ids\n", iommu->name);
1704
fb170fb4
JL		 1705 return num;
1706}
1707
1708static int iommu_attach_domain(struct dmar_domain *domain,
1709 struct intel_iommu *iommu)
1710{
1711 int num;
1712 unsigned long flags;
1713
1714 spin_lock_irqsave(&iommu->lock, flags);
1715 num = __iommu_attach_domain(domain, iommu);
44bde614 1716 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1717
fb170fb4 1718 return num;
ba395927
KA		 1719}
1720
2c2e2c38
FY		 1721static void iommu_detach_domain(struct dmar_domain *domain,
1722 struct intel_iommu *iommu)
ba395927
KA		 1723{
1724 unsigned long flags;
c0e8a6c8 1725 int num;
ba395927 1726
8c11e798 1727 spin_lock_irqsave(&iommu->lock, flags);
c0e8a6c8
JR		 1728
1729 num = domain->iommu_did[iommu->seq_id];
1730
1731 if (num == 0)
1732 return;
1733
1734 clear_bit(num, iommu->domain_ids);
9452d5bf 1735 set_iommu_domain(iommu, num, NULL);
c0e8a6c8 1736
8c11e798 1737 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1738}
1739
fb170fb4
JL		 1740static void domain_attach_iommu(struct dmar_domain *domain,
1741 struct intel_iommu *iommu)
1742{
1743 unsigned long flags;
1744
1745 spin_lock_irqsave(&domain->iommu_lock, flags);
29a27719
JR		 1746 domain->iommu_refcnt[iommu->seq_id] += 1;
1747 domain->iommu_count += 1;
1748 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1749 domain->nid = iommu->node;
fb170fb4
JL		 1750 domain_update_iommu_cap(domain);
1751 }
1752 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1753}
1754
1755static int domain_detach_iommu(struct dmar_domain *domain,
1756 struct intel_iommu *iommu)
1757{
1758 unsigned long flags;
1759 int count = INT_MAX;
1760
1761 spin_lock_irqsave(&domain->iommu_lock, flags);
29a27719
JR		 1762 domain->iommu_refcnt[iommu->seq_id] -= 1;
1763 count = --domain->iommu_count;
1764 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
fb170fb4 1765 domain_update_iommu_cap(domain);
c0e8a6c8 1766 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4
JL		 1767 }
1768 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1769
1770 return count;
1771}
1772
ba395927 1773static struct iova_domain reserved_iova_list;
8a443df4 1774static struct lock_class_key reserved_rbtree_key;
ba395927 1775
51a63e67 1776static int dmar_init_reserved_ranges(void)
ba395927
KA		 1777{
1778 struct pci_dev *pdev = NULL;
1779 struct iova *iova;
1780 int i;
ba395927 1781
0fb5fe87
RM		 1782 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1783 DMA_32BIT_PFN);
ba395927 1784
8a443df4
MG		 1785 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1786 &reserved_rbtree_key);
1787
ba395927
KA		 1788 /* IOAPIC ranges shouldn't be accessed by DMA */
1789 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1790 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1791 if (!iova) {
9f10e5bf 1792 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1793 return -ENODEV;
1794 }
ba395927
KA		 1795
1796 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1797 for_each_pci_dev(pdev) {
1798 struct resource *r;
1799
1800 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1801 r = &pdev->resource[i];
1802 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1803 continue;
1a4a4551
DW		 1804 iova = reserve_iova(&reserved_iova_list,
1805 IOVA_PFN(r->start),
1806 IOVA_PFN(r->end));
51a63e67 1807 if (!iova) {
9f10e5bf 1808 pr_err("Reserve iova failed\n");
51a63e67
JC		 1809 return -ENODEV;
1810 }
ba395927
KA		 1811 }
1812 }
51a63e67 1813 return 0;
ba395927
KA		 1814}
1815
1816static void domain_reserve_special_ranges(struct dmar_domain *domain)
1817{
1818 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1819}
1820
1821static inline int guestwidth_to_adjustwidth(int gaw)
1822{
1823 int agaw;
1824 int r = (gaw - 12) % 9;
1825
1826 if (r == 0)
1827 agaw = gaw;
1828 else
1829 agaw = gaw + 9 - r;
1830 if (agaw > 64)
1831 agaw = 64;
1832 return agaw;
1833}
1834
1835static int domain_init(struct dmar_domain *domain, int guest_width)
1836{
1837 struct intel_iommu *iommu;
1838 int adjust_width, agaw;
1839 unsigned long sagaw;
1840
0fb5fe87
RM		 1841 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1842 DMA_32BIT_PFN);
ba395927
KA		 1843 domain_reserve_special_ranges(domain);
1844
1845 /* calculate AGAW */
8c11e798 1846 iommu = domain_get_iommu(domain);
ba395927
KA		 1847 if (guest_width > cap_mgaw(iommu->cap))
1848 guest_width = cap_mgaw(iommu->cap);
1849 domain->gaw = guest_width;
1850 adjust_width = guestwidth_to_adjustwidth(guest_width);
1851 agaw = width_to_agaw(adjust_width);
1852 sagaw = cap_sagaw(iommu->cap);
1853 if (!test_bit(agaw, &sagaw)) {
1854 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1855 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1856 agaw = find_next_bit(&sagaw, 5, agaw);
1857 if (agaw >= 5)
1858 return -ENODEV;
1859 }
1860 domain->agaw = agaw;
ba395927 1861
8e604097
WH		 1862 if (ecap_coherent(iommu->ecap))
1863 domain->iommu_coherency = 1;
1864 else
1865 domain->iommu_coherency = 0;
1866
58c610bd
SY		 1867 if (ecap_sc_support(iommu->ecap))
1868 domain->iommu_snooping = 1;
1869 else
1870 domain->iommu_snooping = 0;
1871
214e39aa
DW		 1872 if (intel_iommu_superpage)
1873 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1874 else
1875 domain->iommu_superpage = 0;
1876
4c923d47 1877 domain->nid = iommu->node;
c7151a8d 1878
ba395927 1879 /* always allocate the top pgd */
4c923d47 1880 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1881 if (!domain->pgd)
1882 return -ENOMEM;
5b6985ce 1883 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1884 return 0;
1885}
1886
1887static void domain_exit(struct dmar_domain *domain)
1888{
ea8ea460 1889 struct page *freelist = NULL;
29a27719 1890 int i;
ba395927
KA		 1891
1892 /* Domain 0 is reserved, so dont process it */
1893 if (!domain)
1894 return;
1895
7b668357
AW		 1896 /* Flush any lazy unmaps that may reference this domain */
1897 if (!intel_iommu_strict)
1898 flush_unmaps_timeout(0);
1899
92d03cc8 1900 /* remove associated devices */
ba395927 1901 domain_remove_dev_info(domain);
92d03cc8 1902
ba395927
KA		 1903 /* destroy iovas */
1904 put_iova_domain(&domain->iovad);
ba395927 1905
ea8ea460 1906 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1907
92d03cc8 1908 /* clear attached or cached domains */
0e242612 1909 rcu_read_lock();
29a27719
JR		 1910 for_each_domain_iommu(i, domain)
1911 iommu_detach_domain(domain, g_iommus[i]);
0e242612 1912 rcu_read_unlock();
2c2e2c38 1913
ea8ea460
DW		 1914 dma_free_pagelist(freelist);
1915
ba395927
KA		 1916 free_domain_mem(domain);
1917}
1918
64ae892b
DW		 1919static int domain_context_mapping_one(struct dmar_domain *domain,
1920 struct intel_iommu *iommu,
28ccce0d 1921 u8 bus, u8 devfn)
ba395927 1922{
28ccce0d
JR		 1923 int translation = CONTEXT_TT_MULTI_LEVEL;
1924 struct device_domain_info *info = NULL;
ba395927 1925 struct context_entry *context;
ba395927 1926 unsigned long flags;
ea6606b0 1927 struct dma_pte *pgd;
ea6606b0
WH		 1928 int id;
1929 int agaw;
28ccce0d
JR		 1930
1931 if (hw_pass_through && domain_type_is_si(domain))
1932 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1933
1934 pr_debug("Set context mapping for %02x:%02x.%d\n",
1935 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1936
ba395927 1937 BUG_ON(!domain->pgd);
5331fe6f 1938
03ecc32c
DW		 1939 spin_lock_irqsave(&iommu->lock, flags);
1940 context = iommu_context_addr(iommu, bus, devfn, 1);
1941 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1942 if (!context)
1943 return -ENOMEM;
1944 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1945 if (context_present(context)) {
ba395927
KA		 1946 spin_unlock_irqrestore(&iommu->lock, flags);
1947 return 0;
1948 }
1949
ea6606b0
WH		 1950 pgd = domain->pgd;
1951
de24e553
JR		 1952 id = __iommu_attach_domain(domain, iommu);
1953 if (id < 0) {
1954 spin_unlock_irqrestore(&iommu->lock, flags);
1955 pr_err("%s: No free domain ids\n", iommu->name);
1956 return -EFAULT;
ea6606b0
WH		 1957 }
1958
de24e553 1959 context_clear_entry(context);
ea6606b0 1960 context_set_domain_id(context, id);
4ed0d3e6 1961
de24e553
JR		 1962 /*
1963 * Skip top levels of page tables for iommu which has less agaw
1964 * than default. Unnecessary for PT mode.
1965 */
93a23a72 1966 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553
JR		 1967 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1968 pgd = phys_to_virt(dma_pte_addr(pgd));
1969 if (!dma_pte_present(pgd)) {
1970 spin_unlock_irqrestore(&iommu->lock, flags);
1971 return -ENOMEM;
1972 }
1973 }
1974
64ae892b 1975 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1976 translation = info ? CONTEXT_TT_DEV_IOTLB :
1977 CONTEXT_TT_MULTI_LEVEL;
de24e553 1978
93a23a72
YZ		 1979 context_set_address_root(context, virt_to_phys(pgd));
1980 context_set_address_width(context, iommu->agaw);
de24e553
JR		 1981 } else {
1982 /*
1983 * In pass through mode, AW must be programmed to
1984 * indicate the largest AGAW value supported by
1985 * hardware. And ASR is ignored by hardware.
1986 */
1987 context_set_address_width(context, iommu->msagaw);
93a23a72 1988 }
4ed0d3e6
FY		 1989
1990 context_set_translation_type(context, translation);
c07e7d21
MM		 1991 context_set_fault_enable(context);
1992 context_set_present(context);
5331fe6f 1993 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1994
4c25a2c1
DW		 1995 /*
1996 * It's a non-present to present mapping. If hardware doesn't cache
1997 * non-present entry we only need to flush the write-buffer. If the
1998 * _does_ cache non-present entries, then it does so in the special
1999 * domain #0, which we have to flush:
2000 */
2001 if (cap_caching_mode(iommu->cap)) {
2002 iommu->flush.flush_context(iommu, 0,
2003 (((u16)bus) << 8) | devfn,
2004 DMA_CCMD_MASK_NOBIT,
2005 DMA_CCMD_DEVICE_INVL);
18fd779a 2006 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 2007 } else {
ba395927 2008 iommu_flush_write_buffer(iommu);
4c25a2c1 2009 }
93a23a72 2010 iommu_enable_dev_iotlb(info);
ba395927 2011 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 2012
fb170fb4
JL		 2013 domain_attach_iommu(domain, iommu);
2014
ba395927
KA		 2015 return 0;
2016}
2017
579305f7
AW		 2018struct domain_context_mapping_data {
2019 struct dmar_domain *domain;
2020 struct intel_iommu *iommu;
579305f7
AW		 2021};
2022
2023static int domain_context_mapping_cb(struct pci_dev *pdev,
2024 u16 alias, void *opaque)
2025{
2026 struct domain_context_mapping_data *data = opaque;
2027
2028 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2029 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2030}
2031
ba395927 2032static int
28ccce0d 2033domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2034{
64ae892b 2035 struct intel_iommu *iommu;
156baca8 2036 u8 bus, devfn;
579305f7 2037 struct domain_context_mapping_data data;
64ae892b 2038
e1f167f3 2039 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2040 if (!iommu)
2041 return -ENODEV;
ba395927 2042
579305f7 2043 if (!dev_is_pci(dev))
28ccce0d 2044 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2045
2046 data.domain = domain;
2047 data.iommu = iommu;
579305f7
AW		 2048
2049 return pci_for_each_dma_alias(to_pci_dev(dev),
2050 &domain_context_mapping_cb, &data);
2051}
2052
2053static int domain_context_mapped_cb(struct pci_dev *pdev,
2054 u16 alias, void *opaque)
2055{
2056 struct intel_iommu *iommu = opaque;
2057
2058 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2059}
2060
e1f167f3 2061static int domain_context_mapped(struct device *dev)
ba395927 2062{
5331fe6f 2063 struct intel_iommu *iommu;
156baca8 2064 u8 bus, devfn;
5331fe6f 2065
e1f167f3 2066 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2067 if (!iommu)
2068 return -ENODEV;
ba395927 2069
579305f7
AW		 2070 if (!dev_is_pci(dev))
2071 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2072
579305f7
AW		 2073 return !pci_for_each_dma_alias(to_pci_dev(dev),
2074 domain_context_mapped_cb, iommu);
ba395927
KA		 2075}
2076
f532959b
FY		 2077/* Returns a number of VTD pages, but aligned to MM page size */
2078static inline unsigned long aligned_nrpages(unsigned long host_addr,
2079 size_t size)
2080{
2081 host_addr &= ~PAGE_MASK;
2082 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2083}
2084
6dd9a7c7
YS		 2085/* Return largest possible superpage level for a given mapping */
2086static inline int hardware_largepage_caps(struct dmar_domain *domain,
2087 unsigned long iov_pfn,
2088 unsigned long phy_pfn,
2089 unsigned long pages)
2090{
2091 int support, level = 1;
2092 unsigned long pfnmerge;
2093
2094 support = domain->iommu_superpage;
2095
2096 /* To use a large page, the virtual *and* physical addresses
2097 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2098 of them will mean we have to use smaller pages. So just
2099 merge them and check both at once. */
2100 pfnmerge = iov_pfn | phy_pfn;
2101
2102 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2103 pages >>= VTD_STRIDE_SHIFT;
2104 if (!pages)
2105 break;
2106 pfnmerge >>= VTD_STRIDE_SHIFT;
2107 level++;
2108 support--;
2109 }
2110 return level;
2111}
2112
9051aa02
DW		 2113static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2114 struct scatterlist *sg, unsigned long phys_pfn,
2115 unsigned long nr_pages, int prot)
e1605495
DW		 2116{
2117 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2118 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2119 unsigned long sg_res = 0;
6dd9a7c7
YS		 2120 unsigned int largepage_lvl = 0;
2121 unsigned long lvl_pages = 0;
e1605495 2122
162d1b10 2123 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2124
2125 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2126 return -EINVAL;
2127
2128 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2129
cc4f14aa
JL		 2130 if (!sg) {
2131 sg_res = nr_pages;
9051aa02
DW		 2132 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2133 }
2134
6dd9a7c7 2135 while (nr_pages > 0) {
c85994e4
DW		 2136 uint64_t tmp;
2137
e1605495 2138 if (!sg_res) {
f532959b 2139 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2140 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2141 sg->dma_length = sg->length;
2142 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2143 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2144 }
6dd9a7c7 2145
e1605495 2146 if (!pte) {
6dd9a7c7
YS		 2147 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2148
5cf0a76f 2149 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2150 if (!pte)
2151 return -ENOMEM;
6dd9a7c7 2152 /* It is large page*/
6491d4d0 2153 if (largepage_lvl > 1) {
6dd9a7c7 2154 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2155 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2156 /*
2157 * Ensure that old small page tables are
2158 * removed to make room for superpage,
2159 * if they exist.
2160 */
6491d4d0 2161 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2162 iov_pfn + lvl_pages - 1);
6491d4d0 2163 } else {
6dd9a7c7 2164 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2165 }
6dd9a7c7 2166
e1605495
DW		 2167 }
2168 /* We don't need lock here, nobody else
2169 * touches the iova range
2170 */
7766a3fb 2171 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2172 if (tmp) {
1bf20f0d 2173 static int dumps = 5;
9f10e5bf
JR		 2174 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2175 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2176 if (dumps) {
2177 dumps--;
2178 debug_dma_dump_mappings(NULL);
2179 }
2180 WARN_ON(1);
2181 }
6dd9a7c7
YS		 2182
2183 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2184
2185 BUG_ON(nr_pages < lvl_pages);
2186 BUG_ON(sg_res < lvl_pages);
2187
2188 nr_pages -= lvl_pages;
2189 iov_pfn += lvl_pages;
2190 phys_pfn += lvl_pages;
2191 pteval += lvl_pages * VTD_PAGE_SIZE;
2192 sg_res -= lvl_pages;
2193
2194 /* If the next PTE would be the first in a new page, then we
2195 need to flush the cache on the entries we've just written.
2196 And then we'll need to recalculate 'pte', so clear it and
2197 let it get set again in the if (!pte) block above.
2198
2199 If we're done (!nr_pages) we need to flush the cache too.
2200
2201 Also if we've been setting superpages, we may need to
2202 recalculate 'pte' and switch back to smaller pages for the
2203 end of the mapping, if the trailing size is not enough to
2204 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2205 pte++;
6dd9a7c7
YS		 2206 if (!nr_pages || first_pte_in_page(pte) ||
2207 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2208 domain_flush_cache(domain, first_pte,
2209 (void *)pte - (void *)first_pte);
2210 pte = NULL;
2211 }
6dd9a7c7
YS		 2212
2213 if (!sg_res && nr_pages)
e1605495
DW		 2214 sg = sg_next(sg);
2215 }
2216 return 0;
2217}
2218
9051aa02
DW		 2219static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2220 struct scatterlist *sg, unsigned long nr_pages,
2221 int prot)
ba395927 2222{
9051aa02
DW		 2223 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2224}
6f6a00e4 2225
9051aa02
DW		 2226static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2227 unsigned long phys_pfn, unsigned long nr_pages,
2228 int prot)
2229{
2230 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2231}
2232
c7151a8d 2233static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2234{
c7151a8d
WH		 2235 if (!iommu)
2236 return;
8c11e798
WH		 2237
2238 clear_context_table(iommu, bus, devfn);
2239 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2240 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2241 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2242}
2243
109b9b04
DW		 2244static inline void unlink_domain_info(struct device_domain_info *info)
2245{
2246 assert_spin_locked(&device_domain_lock);
2247 list_del(&info->link);
2248 list_del(&info->global);
2249 if (info->dev)
0bcb3e28 2250 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2251}
2252
ba395927
KA		 2253static void domain_remove_dev_info(struct dmar_domain *domain)
2254{
3a74ca01 2255 struct device_domain_info *info, *tmp;
ba395927 2256
76f45fe3 2257 list_for_each_entry_safe(info, tmp, &domain->devices, link)
e6de0f8d 2258 dmar_remove_one_dev_info(domain, info->dev);
ba395927
KA		 2259}
2260
2261/*
2262 * find_domain
1525a29a 2263 * Note: we use struct device->archdata.iommu stores the info
ba395927 2264 */
1525a29a 2265static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2266{
2267 struct device_domain_info *info;
2268
2269 /* No lock here, assumes no domain exit in normal case */
1525a29a 2270 info = dev->archdata.iommu;
ba395927
KA		 2271 if (info)
2272 return info->domain;
2273 return NULL;
2274}
2275
5a8f40e8 2276static inline struct device_domain_info *
745f2586
JL		 2277dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2278{
2279 struct device_domain_info *info;
2280
2281 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2282 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2283 info->devfn == devfn)
5a8f40e8 2284 return info;
745f2586
JL		 2285
2286 return NULL;
2287}
2288
5db31569
JR		 2289static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2290 int bus, int devfn,
2291 struct device *dev,
2292 struct dmar_domain *domain)
745f2586 2293{
5a8f40e8 2294 struct dmar_domain *found = NULL;
745f2586
JL		 2295 struct device_domain_info *info;
2296 unsigned long flags;
2297
2298 info = alloc_devinfo_mem();
2299 if (!info)
b718cd3d 2300 return NULL;
745f2586 2301
745f2586
JL		 2302 info->bus = bus;
2303 info->devfn = devfn;
2304 info->dev = dev;
2305 info->domain = domain;
5a8f40e8 2306 info->iommu = iommu;
745f2586
JL		 2307
2308 spin_lock_irqsave(&device_domain_lock, flags);
2309 if (dev)
0bcb3e28 2310 found = find_domain(dev);
5a8f40e8
DW		 2311 else {
2312 struct device_domain_info *info2;
41e80dca 2313 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2314 if (info2)
2315 found = info2->domain;
2316 }
745f2586
JL		 2317 if (found) {
2318 spin_unlock_irqrestore(&device_domain_lock, flags);
2319 free_devinfo_mem(info);
b718cd3d
DW		 2320 /* Caller must free the original domain */
2321 return found;
745f2586
JL		 2322 }
2323
b718cd3d
DW		 2324 list_add(&info->link, &domain->devices);
2325 list_add(&info->global, &device_domain_list);
2326 if (dev)
2327 dev->archdata.iommu = info;
2328 spin_unlock_irqrestore(&device_domain_lock, flags);
2329
cc4e2575
JR		 2330 if (dev && domain_context_mapping(domain, dev)) {
2331 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2332 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2333 return NULL;
2334 }
2335
b718cd3d 2336 return domain;
745f2586
JL		 2337}
2338
579305f7
AW		 2339static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2340{
2341 *(u16 *)opaque = alias;
2342 return 0;
2343}
2344
ba395927 2345/* domain is initialized */
146922ec 2346static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2347{
cc4e2575 2348 struct device_domain_info *info = NULL;
579305f7
AW		 2349 struct dmar_domain *domain, *tmp;
2350 struct intel_iommu *iommu;
ba395927 2351 unsigned long flags;
cc4e2575 2352 u16 dma_alias;
aa4d066a 2353 u8 bus, devfn;
ba395927 2354
146922ec 2355 domain = find_domain(dev);
ba395927
KA		 2356 if (domain)
2357 return domain;
2358
579305f7
AW		 2359 iommu = device_to_iommu(dev, &bus, &devfn);
2360 if (!iommu)
2361 return NULL;
2362
146922ec
DW		 2363 if (dev_is_pci(dev)) {
2364 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2365
579305f7
AW		 2366 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2367
2368 spin_lock_irqsave(&device_domain_lock, flags);
2369 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2370 PCI_BUS_NUM(dma_alias),
2371 dma_alias & 0xff);
2372 if (info) {
2373 iommu = info->iommu;
2374 domain = info->domain;
5a8f40e8 2375 }
579305f7 2376 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2377
579305f7
AW		 2378 /* DMA alias already has a domain, uses it */
2379 if (info)
2380 goto found_domain;
2381 }
ba395927 2382
146922ec 2383 /* Allocate and initialize new domain for the device */
ab8dfe25 2384 domain = alloc_domain(0);
745f2586 2385 if (!domain)
579305f7 2386 return NULL;
af1089ce 2387 if (iommu_attach_domain(domain, iommu) < 0) {
2fe9723d 2388 free_domain_mem(domain);
579305f7 2389 return NULL;
2c2e2c38 2390 }
fb170fb4 2391 domain_attach_iommu(domain, iommu);
579305f7
AW		 2392 if (domain_init(domain, gaw)) {
2393 domain_exit(domain);
2394 return NULL;
2c2e2c38 2395 }
ba395927 2396
579305f7
AW		 2397 /* register PCI DMA alias device */
2398 if (dev_is_pci(dev)) {
5db31569
JR		 2399 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2400 dma_alias & 0xff, NULL, domain);
579305f7
AW		 2401
2402 if (!tmp || tmp != domain) {
2403 domain_exit(domain);
2404 domain = tmp;
2405 }
2406
b718cd3d 2407 if (!domain)
579305f7 2408 return NULL;
ba395927
KA		 2409 }
2410
2411found_domain:
5db31569 2412 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
579305f7
AW		 2413
2414 if (!tmp || tmp != domain) {
2415 domain_exit(domain);
2416 domain = tmp;
2417 }
b718cd3d
DW		 2418
2419 return domain;
ba395927
KA		 2420}
2421
2c2e2c38 2422static int iommu_identity_mapping;
e0fc7e0b
DW		 2423#define IDENTMAP_ALL 1
2424#define IDENTMAP_GFX 2
2425#define IDENTMAP_AZALIA 4
2c2e2c38 2426
b213203e
DW		 2427static int iommu_domain_identity_map(struct dmar_domain *domain,
2428 unsigned long long start,
2429 unsigned long long end)
ba395927 2430{
c5395d5c
DW		 2431 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2432 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2433
2434 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2435 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2436 pr_err("Reserving iova failed\n");
b213203e 2437 return -ENOMEM;
ba395927
KA		 2438 }
2439
af1089ce 2440 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2441 /*
2442 * RMRR range might have overlap with physical memory range,
2443 * clear it first
2444 */
c5395d5c 2445 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2446
c5395d5c
DW		 2447 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2448 last_vpfn - first_vpfn + 1,
61df7443 2449 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2450}
2451
0b9d9753 2452static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2453 unsigned long long start,
2454 unsigned long long end)
2455{
2456 struct dmar_domain *domain;
2457 int ret;
2458
0b9d9753 2459 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2460 if (!domain)
2461 return -ENOMEM;
2462
19943b0e
DW		 2463 /* For _hardware_ passthrough, don't bother. But for software
2464 passthrough, we do it anyway -- it may indicate a memory
2465 range which is reserved in E820, so which didn't get set
2466 up to start with in si_domain */
2467 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2468 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2469 dev_name(dev), start, end);
19943b0e
DW		 2470 return 0;
2471 }
2472
9f10e5bf
JR		 2473 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2474 dev_name(dev), start, end);
2475
5595b528
DW		 2476 if (end < start) {
2477 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2478 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2479 dmi_get_system_info(DMI_BIOS_VENDOR),
2480 dmi_get_system_info(DMI_BIOS_VERSION),
2481 dmi_get_system_info(DMI_PRODUCT_VERSION));
2482 ret = -EIO;
2483 goto error;
2484 }
2485
2ff729f5
DW		 2486 if (end >> agaw_to_width(domain->agaw)) {
2487 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2488 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2489 agaw_to_width(domain->agaw),
2490 dmi_get_system_info(DMI_BIOS_VENDOR),
2491 dmi_get_system_info(DMI_BIOS_VERSION),
2492 dmi_get_system_info(DMI_PRODUCT_VERSION));
2493 ret = -EIO;
2494 goto error;
2495 }
19943b0e 2496
b213203e 2497 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2498 if (ret)
2499 goto error;
2500
b213203e
DW		 2501 return 0;
2502
2503 error:
ba395927
KA		 2504 domain_exit(domain);
2505 return ret;
ba395927
KA		 2506}
2507
2508static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2509 struct device *dev)
ba395927 2510{
0b9d9753 2511 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2512 return 0;
0b9d9753
DW		 2513 return iommu_prepare_identity_map(dev, rmrr->base_address,
2514 rmrr->end_address);
ba395927
KA		 2515}
2516
d3f13810 2517#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2518static inline void iommu_prepare_isa(void)
2519{
2520 struct pci_dev *pdev;
2521 int ret;
2522
2523 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2524 if (!pdev)
2525 return;
2526
9f10e5bf 2527 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2528 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2529
2530 if (ret)
9f10e5bf 2531 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2532
9b27e82d 2533 pci_dev_put(pdev);
49a0429e
KA		 2534}
2535#else
2536static inline void iommu_prepare_isa(void)
2537{
2538 return;
2539}
d3f13810 2540#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2541
2c2e2c38 2542static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2543
071e1374 2544static int __init si_domain_init(int hw)
2c2e2c38 2545{
c7ab48d2 2546 int nid, ret = 0;
2c2e2c38 2547
ab8dfe25 2548 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2549 if (!si_domain)
2550 return -EFAULT;
2551
2c2e2c38
FY		 2552 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2553 domain_exit(si_domain);
2554 return -EFAULT;
2555 }
2556
0dc79715 2557 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2558
19943b0e
DW		 2559 if (hw)
2560 return 0;
2561
c7ab48d2 2562 for_each_online_node(nid) {
5dfe8660
TH		 2563 unsigned long start_pfn, end_pfn;
2564 int i;
2565
2566 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2567 ret = iommu_domain_identity_map(si_domain,
2568 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2569 if (ret)
2570 return ret;
2571 }
c7ab48d2
DW		 2572 }
2573
2c2e2c38
FY		 2574 return 0;
2575}
2576
9b226624 2577static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2578{
2579 struct device_domain_info *info;
2580
2581 if (likely(!iommu_identity_mapping))
2582 return 0;
2583
9b226624 2584 info = dev->archdata.iommu;
cb452a40
MT		 2585 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2586 return (info->domain == si_domain);
2c2e2c38 2587
2c2e2c38
FY		 2588 return 0;
2589}
2590
28ccce0d 2591static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2592{
0ac72664 2593 struct dmar_domain *ndomain;
5a8f40e8 2594 struct intel_iommu *iommu;
156baca8 2595 u8 bus, devfn;
2c2e2c38 2596
5913c9bf 2597 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2598 if (!iommu)
2599 return -ENODEV;
2600
5db31569 2601 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2602 if (ndomain != domain)
2603 return -EBUSY;
2c2e2c38
FY		 2604
2605 return 0;
2606}
2607
0b9d9753 2608static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2609{
2610 struct dmar_rmrr_unit *rmrr;
832bd858 2611 struct device *tmp;
ea2447f7
TM		 2612 int i;
2613
0e242612 2614 rcu_read_lock();
ea2447f7 2615 for_each_rmrr_units(rmrr) {
b683b230
JL		 2616 /*
2617 * Return TRUE if this RMRR contains the device that
2618 * is passed in.
2619 */
2620 for_each_active_dev_scope(rmrr->devices,
2621 rmrr->devices_cnt, i, tmp)
0b9d9753 2622 if (tmp == dev) {
0e242612 2623 rcu_read_unlock();
ea2447f7 2624 return true;
b683b230 2625 }
ea2447f7 2626 }
0e242612 2627 rcu_read_unlock();
ea2447f7
TM		 2628 return false;
2629}
2630
c875d2c1
AW		 2631/*
2632 * There are a couple cases where we need to restrict the functionality of
2633 * devices associated with RMRRs. The first is when evaluating a device for
2634 * identity mapping because problems exist when devices are moved in and out
2635 * of domains and their respective RMRR information is lost. This means that
2636 * a device with associated RMRRs will never be in a "passthrough" domain.
2637 * The second is use of the device through the IOMMU API. This interface
2638 * expects to have full control of the IOVA space for the device. We cannot
2639 * satisfy both the requirement that RMRR access is maintained and have an
2640 * unencumbered IOVA space. We also have no ability to quiesce the device's
2641 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2642 * We therefore prevent devices associated with an RMRR from participating in
2643 * the IOMMU API, which eliminates them from device assignment.
2644 *
2645 * In both cases we assume that PCI USB devices with RMRRs have them largely
2646 * for historical reasons and that the RMRR space is not actively used post
2647 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2648 *
2649 * The same exception is made for graphics devices, with the requirement that
2650 * any use of the RMRR regions will be torn down before assigning the device
2651 * to a guest.
c875d2c1
AW		 2652 */
2653static bool device_is_rmrr_locked(struct device *dev)
2654{
2655 if (!device_has_rmrr(dev))
2656 return false;
2657
2658 if (dev_is_pci(dev)) {
2659 struct pci_dev *pdev = to_pci_dev(dev);
2660
18436afd 2661 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2662 return false;
2663 }
2664
2665 return true;
2666}
2667
3bdb2591 2668static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2669{
ea2447f7 2670
3bdb2591
DW		 2671 if (dev_is_pci(dev)) {
2672 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2673
c875d2c1 2674 if (device_is_rmrr_locked(dev))
3bdb2591 2675 return 0;
e0fc7e0b 2676
3bdb2591
DW		 2677 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2678 return 1;
e0fc7e0b 2679
3bdb2591
DW		 2680 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2681 return 1;
6941af28 2682
3bdb2591 2683 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2684 return 0;
3bdb2591
DW		 2685
2686 /*
2687 * We want to start off with all devices in the 1:1 domain, and
2688 * take them out later if we find they can't access all of memory.
2689 *
2690 * However, we can't do this for PCI devices behind bridges,
2691 * because all PCI devices behind the same bridge will end up
2692 * with the same source-id on their transactions.
2693 *
2694 * Practically speaking, we can't change things around for these
2695 * devices at run-time, because we can't be sure there'll be no
2696 * DMA transactions in flight for any of their siblings.
2697 *
2698 * So PCI devices (unless they're on the root bus) as well as
2699 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2700 * the 1:1 domain, just in _case_ one of their siblings turns out
2701 * not to be able to map all of memory.
2702 */
2703 if (!pci_is_pcie(pdev)) {
2704 if (!pci_is_root_bus(pdev->bus))
2705 return 0;
2706 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2707 return 0;
2708 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2709 return 0;
3bdb2591
DW		 2710 } else {
2711 if (device_has_rmrr(dev))
2712 return 0;
2713 }
3dfc813d 2714
3bdb2591 2715 /*
3dfc813d 2716 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2717 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2718 * take them out of the 1:1 domain later.
2719 */
8fcc5372
CW		 2720 if (!startup) {
2721 /*
2722 * If the device's dma_mask is less than the system's memory
2723 * size then this is not a candidate for identity mapping.
2724 */
3bdb2591 2725 u64 dma_mask = *dev->dma_mask;
8fcc5372 2726
3bdb2591
DW		 2727 if (dev->coherent_dma_mask &&
2728 dev->coherent_dma_mask < dma_mask)
2729 dma_mask = dev->coherent_dma_mask;
8fcc5372 2730
3bdb2591 2731 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2732 }
6941af28
DW		 2733
2734 return 1;
2735}
2736
cf04eee8
DW		 2737static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2738{
2739 int ret;
2740
2741 if (!iommu_should_identity_map(dev, 1))
2742 return 0;
2743
28ccce0d 2744 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2745 if (!ret)
9f10e5bf
JR		 2746 pr_info("%s identity mapping for device %s\n",
2747 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2748 else if (ret == -ENODEV)
2749 /* device not associated with an iommu */
2750 ret = 0;
2751
2752 return ret;
2753}
2754
2755
071e1374 2756static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2757{
2c2e2c38 2758 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2759 struct dmar_drhd_unit *drhd;
2760 struct intel_iommu *iommu;
2761 struct device *dev;
2762 int i;
2763 int ret = 0;
2c2e2c38 2764
2c2e2c38 2765 for_each_pci_dev(pdev) {
cf04eee8
DW		 2766 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2767 if (ret)
2768 return ret;
2769 }
2770
2771 for_each_active_iommu(iommu, drhd)
2772 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2773 struct acpi_device_physical_node *pn;
2774 struct acpi_device *adev;
2775
2776 if (dev->bus != &acpi_bus_type)
2777 continue;
86080ccc 2778
cf04eee8
DW		 2779 adev= to_acpi_device(dev);
2780 mutex_lock(&adev->physical_node_lock);
2781 list_for_each_entry(pn, &adev->physical_node_list, node) {
2782 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2783 if (ret)
2784 break;
eae460b6 2785 }
cf04eee8
DW		 2786 mutex_unlock(&adev->physical_node_lock);
2787 if (ret)
2788 return ret;
62edf5dc 2789 }
2c2e2c38
FY		 2790
2791 return 0;
2792}
2793
ffebeb46
JL		 2794static void intel_iommu_init_qi(struct intel_iommu *iommu)
2795{
2796 /*
2797 * Start from the sane iommu hardware state.
2798 * If the queued invalidation is already initialized by us
2799 * (for example, while enabling interrupt-remapping) then
2800 * we got the things already rolling from a sane state.
2801 */
2802 if (!iommu->qi) {
2803 /*
2804 * Clear any previous faults.
2805 */
2806 dmar_fault(-1, iommu);
2807 /*
2808 * Disable queued invalidation if supported and already enabled
2809 * before OS handover.
2810 */
2811 dmar_disable_qi(iommu);
2812 }
2813
2814 if (dmar_enable_qi(iommu)) {
2815 /*
2816 * Queued Invalidate not enabled, use Register Based Invalidate
2817 */
2818 iommu->flush.flush_context = __iommu_flush_context;
2819 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2820 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2821 iommu->name);
2822 } else {
2823 iommu->flush.flush_context = qi_flush_context;
2824 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2825 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2826 }
2827}
2828
091d42e4
JR		 2829static int copy_context_table(struct intel_iommu *iommu,
2830 struct root_entry *old_re,
2831 struct context_entry **tbl,
2832 int bus, bool ext)
2833{
2834 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2835 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2836 phys_addr_t old_ce_phys;
2837
2838 tbl_idx = ext ? bus * 2 : bus;
2839
2840 for (devfn = 0; devfn < 256; devfn++) {
2841 /* First calculate the correct index */
2842 idx = (ext ? devfn * 2 : devfn) % 256;
2843
2844 if (idx == 0) {
2845 /* First save what we may have and clean up */
2846 if (new_ce) {
2847 tbl[tbl_idx] = new_ce;
2848 __iommu_flush_cache(iommu, new_ce,
2849 VTD_PAGE_SIZE);
2850 pos = 1;
2851 }
2852
2853 if (old_ce)
2854 iounmap(old_ce);
2855
2856 ret = 0;
2857 if (devfn < 0x80)
2858 old_ce_phys = root_entry_lctp(old_re);
2859 else
2860 old_ce_phys = root_entry_uctp(old_re);
2861
2862 if (!old_ce_phys) {
2863 if (ext && devfn == 0) {
2864 /* No LCTP, try UCTP */
2865 devfn = 0x7f;
2866 continue;
2867 } else {
2868 goto out;
2869 }
2870 }
2871
2872 ret = -ENOMEM;
2873 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2874 if (!old_ce)
2875 goto out;
2876
2877 new_ce = alloc_pgtable_page(iommu->node);
2878 if (!new_ce)
2879 goto out_unmap;
2880
2881 ret = 0;
2882 }
2883
2884 /* Now copy the context entry */
2885 ce = old_ce[idx];
2886
cf484d0e 2887 if (!__context_present(&ce))
091d42e4
JR		 2888 continue;
2889
dbcd861f
JR		 2890 did = context_domain_id(&ce);
2891 if (did >= 0 && did < cap_ndoms(iommu->cap))
2892 set_bit(did, iommu->domain_ids);
2893
cf484d0e
JR		 2894 /*
2895 * We need a marker for copied context entries. This
2896 * marker needs to work for the old format as well as
2897 * for extended context entries.
2898 *
2899 * Bit 67 of the context entry is used. In the old
2900 * format this bit is available to software, in the
2901 * extended format it is the PGE bit, but PGE is ignored
2902 * by HW if PASIDs are disabled (and thus still
2903 * available).
2904 *
2905 * So disable PASIDs first and then mark the entry
2906 * copied. This means that we don't copy PASID
2907 * translations from the old kernel, but this is fine as
2908 * faults there are not fatal.
2909 */
2910 context_clear_pasid_enable(&ce);
2911 context_set_copied(&ce);
2912
091d42e4
JR		 2913 new_ce[idx] = ce;
2914 }
2915
2916 tbl[tbl_idx + pos] = new_ce;
2917
2918 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2919
2920out_unmap:
2921 iounmap(old_ce);
2922
2923out:
2924 return ret;
2925}
2926
2927static int copy_translation_tables(struct intel_iommu *iommu)
2928{
2929 struct context_entry **ctxt_tbls;
2930 struct root_entry *old_rt;
2931 phys_addr_t old_rt_phys;
2932 int ctxt_table_entries;
2933 unsigned long flags;
2934 u64 rtaddr_reg;
2935 int bus, ret;
c3361f2f 2936 bool new_ext, ext;
091d42e4
JR		 2937
2938 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2939 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2940 new_ext = !!ecap_ecs(iommu->ecap);
2941
2942 /*
2943 * The RTT bit can only be changed when translation is disabled,
2944 * but disabling translation means to open a window for data
2945 * corruption. So bail out and don't copy anything if we would
2946 * have to change the bit.
2947 */
2948 if (new_ext != ext)
2949 return -EINVAL;
091d42e4
JR		 2950
2951 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2952 if (!old_rt_phys)
2953 return -EINVAL;
2954
2955 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2956 if (!old_rt)
2957 return -ENOMEM;
2958
2959 /* This is too big for the stack - allocate it from slab */
2960 ctxt_table_entries = ext ? 512 : 256;
2961 ret = -ENOMEM;
2962 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2963 if (!ctxt_tbls)
2964 goto out_unmap;
2965
2966 for (bus = 0; bus < 256; bus++) {
2967 ret = copy_context_table(iommu, &old_rt[bus],
2968 ctxt_tbls, bus, ext);
2969 if (ret) {
2970 pr_err("%s: Failed to copy context table for bus %d\n",
2971 iommu->name, bus);
2972 continue;
2973 }
2974 }
2975
2976 spin_lock_irqsave(&iommu->lock, flags);
2977
2978 /* Context tables are copied, now write them to the root_entry table */
2979 for (bus = 0; bus < 256; bus++) {
2980 int idx = ext ? bus * 2 : bus;
2981 u64 val;
2982
2983 if (ctxt_tbls[idx]) {
2984 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2985 iommu->root_entry[bus].lo = val;
2986 }
2987
2988 if (!ext || !ctxt_tbls[idx + 1])
2989 continue;
2990
2991 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2992 iommu->root_entry[bus].hi = val;
2993 }
2994
2995 spin_unlock_irqrestore(&iommu->lock, flags);
2996
2997 kfree(ctxt_tbls);
2998
2999 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3000
3001 ret = 0;
3002
3003out_unmap:
3004 iounmap(old_rt);
3005
3006 return ret;
3007}
3008
b779260b 3009static int __init init_dmars(void)
ba395927
KA		 3010{
3011 struct dmar_drhd_unit *drhd;
3012 struct dmar_rmrr_unit *rmrr;
a87f4918 3013 bool copied_tables = false;
832bd858 3014 struct device *dev;
ba395927 3015 struct intel_iommu *iommu;
9d783ba0 3016 int i, ret;
2c2e2c38 3017
ba395927
KA		 3018 /*
3019 * for each drhd
3020 * allocate root
3021 * initialize and program root entry to not present
3022 * endfor
3023 */
3024 for_each_drhd_unit(drhd) {
5e0d2a6f 3025 /*
3026 * lock not needed as this is only incremented in the single
3027 * threaded kernel __init code path all other access are read
3028 * only
3029 */
78d8e704 3030 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3031 g_num_of_iommus++;
3032 continue;
3033 }
9f10e5bf 3034 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3035 }
3036
ffebeb46
JL		 3037 /* Preallocate enough resources for IOMMU hot-addition */
3038 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3039 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3040
d9630fe9
WH		 3041 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3042 GFP_KERNEL);
3043 if (!g_iommus) {
9f10e5bf 3044 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3045 ret = -ENOMEM;
3046 goto error;
3047 }
3048
80b20dd8 3049 deferred_flush = kzalloc(g_num_of_iommus *
3050 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3051 if (!deferred_flush) {
5e0d2a6f 3052 ret = -ENOMEM;
989d51fc 3053 goto free_g_iommus;
5e0d2a6f 3054 }
3055
7c919779 3056 for_each_active_iommu(iommu, drhd) {
d9630fe9 3057 g_iommus[iommu->seq_id] = iommu;
ba395927 3058
b63d80d1
JR		 3059 intel_iommu_init_qi(iommu);
3060
e61d98d8
SS		 3061 ret = iommu_init_domains(iommu);
3062 if (ret)
989d51fc 3063 goto free_iommu;
e61d98d8 3064
4158c2ec
JR		 3065 init_translation_status(iommu);
3066
091d42e4
JR		 3067 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3068 iommu_disable_translation(iommu);
3069 clear_translation_pre_enabled(iommu);
3070 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3071 iommu->name);
3072 }
4158c2ec 3073
ba395927
KA		 3074 /*
3075 * TBD:
3076 * we could share the same root & context tables
25985edc 3077 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3078 */
3079 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3080 if (ret)
989d51fc 3081 goto free_iommu;
5f0a7f76 3082
091d42e4
JR		 3083 if (translation_pre_enabled(iommu)) {
3084 pr_info("Translation already enabled - trying to copy translation structures\n");
3085
3086 ret = copy_translation_tables(iommu);
3087 if (ret) {
3088 /*
3089 * We found the IOMMU with translation
3090 * enabled - but failed to copy over the
3091 * old root-entry table. Try to proceed
3092 * by disabling translation now and
3093 * allocating a clean root-entry table.
3094 * This might cause DMAR faults, but
3095 * probably the dump will still succeed.
3096 */
3097 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3098 iommu->name);
3099 iommu_disable_translation(iommu);
3100 clear_translation_pre_enabled(iommu);
3101 } else {
3102 pr_info("Copied translation tables from previous kernel for %s\n",
3103 iommu->name);
a87f4918 3104 copied_tables = true;
091d42e4
JR		 3105 }
3106 }
3107
5f0a7f76
JR		 3108 iommu_flush_write_buffer(iommu);
3109 iommu_set_root_entry(iommu);
3110 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3111 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3112
4ed0d3e6 3113 if (!ecap_pass_through(iommu->ecap))
19943b0e 3114 hw_pass_through = 0;
ba395927
KA		 3115 }
3116
19943b0e 3117 if (iommu_pass_through)
e0fc7e0b
DW		 3118 iommu_identity_mapping |= IDENTMAP_ALL;
3119
d3f13810 3120#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3121 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3122#endif
e0fc7e0b 3123
86080ccc
JR		 3124 if (iommu_identity_mapping) {
3125 ret = si_domain_init(hw_pass_through);
3126 if (ret)
3127 goto free_iommu;
3128 }
3129
e0fc7e0b
DW		 3130 check_tylersburg_isoch();
3131
a87f4918
JR		 3132 /*
3133 * If we copied translations from a previous kernel in the kdump
3134 * case, we can not assign the devices to domains now, as that
3135 * would eliminate the old mappings. So skip this part and defer
3136 * the assignment to device driver initialization time.
3137 */
3138 if (copied_tables)
3139 goto domains_done;
3140
ba395927 3141 /*
19943b0e
DW		 3142 * If pass through is not set or not enabled, setup context entries for
3143 * identity mappings for rmrr, gfx, and isa and may fall back to static
3144 * identity mapping if iommu_identity_mapping is set.
ba395927 3145 */
19943b0e
DW		 3146 if (iommu_identity_mapping) {
3147 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3148 if (ret) {
9f10e5bf 3149 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3150 goto free_iommu;
ba395927
KA		 3151 }
3152 }
ba395927 3153 /*
19943b0e
DW		 3154 * For each rmrr
3155 * for each dev attached to rmrr
3156 * do
3157 * locate drhd for dev, alloc domain for dev
3158 * allocate free domain
3159 * allocate page table entries for rmrr
3160 * if context not allocated for bus
3161 * allocate and init context
3162 * set present in root table for this bus
3163 * init context with domain, translation etc
3164 * endfor
3165 * endfor
ba395927 3166 */
9f10e5bf 3167 pr_info("Setting RMRR:\n");
19943b0e 3168 for_each_rmrr_units(rmrr) {
b683b230
JL		 3169 /* some BIOS lists non-exist devices in DMAR table. */
3170 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3171 i, dev) {
0b9d9753 3172 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3173 if (ret)
9f10e5bf 3174 pr_err("Mapping reserved region failed\n");
ba395927 3175 }
4ed0d3e6 3176 }
49a0429e 3177
19943b0e
DW		 3178 iommu_prepare_isa();
3179
a87f4918
JR		 3180domains_done:
3181
ba395927
KA		 3182 /*
3183 * for each drhd
3184 * enable fault log
3185 * global invalidate context cache
3186 * global invalidate iotlb
3187 * enable translation
3188 */
7c919779 3189 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3190 if (drhd->ignored) {
3191 /*
3192 * we always have to disable PMRs or DMA may fail on
3193 * this device
3194 */
3195 if (force_on)
7c919779 3196 iommu_disable_protect_mem_regions(iommu);
ba395927 3197 continue;
51a63e67 3198 }
ba395927
KA		 3199
3200 iommu_flush_write_buffer(iommu);
3201
3460a6d9
KA		 3202 ret = dmar_set_interrupt(iommu);
3203 if (ret)
989d51fc 3204 goto free_iommu;
3460a6d9 3205
8939ddf6
JR		 3206 if (!translation_pre_enabled(iommu))
3207 iommu_enable_translation(iommu);
3208
b94996c9 3209 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3210 }
3211
3212 return 0;
989d51fc
JL		 3213
3214free_iommu:
ffebeb46
JL		 3215 for_each_active_iommu(iommu, drhd) {
3216 disable_dmar_iommu(iommu);
a868e6b7 3217 free_dmar_iommu(iommu);
ffebeb46 3218 }
9bdc531e 3219 kfree(deferred_flush);
989d51fc 3220free_g_iommus:
d9630fe9 3221 kfree(g_iommus);
989d51fc 3222error:
ba395927
KA		 3223 return ret;
3224}
3225
5a5e02a6 3226/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3227static struct iova *intel_alloc_iova(struct device *dev,
3228 struct dmar_domain *domain,
3229 unsigned long nrpages, uint64_t dma_mask)
ba395927 3230{
ba395927 3231 struct iova *iova = NULL;
ba395927 3232
875764de
DW		 3233 /* Restrict dma_mask to the width that the iommu can handle */
3234 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3235
3236 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3237 /*
3238 * First try to allocate an io virtual address in
284901a9 3239 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3240 * from higher range
ba395927 3241 */
875764de
DW		 3242 iova = alloc_iova(&domain->iovad, nrpages,
3243 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3244 if (iova)
3245 return iova;
3246 }
3247 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3248 if (unlikely(!iova)) {
9f10e5bf 3249 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3250 nrpages, dev_name(dev));
f76aec76
KA		 3251 return NULL;
3252 }
3253
3254 return iova;
3255}
3256
d4b709f4 3257static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3258{
3259 struct dmar_domain *domain;
f76aec76 3260
d4b709f4 3261 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3262 if (!domain) {
9f10e5bf 3263 pr_err("Allocating domain for %s failed\n",
d4b709f4 3264 dev_name(dev));
4fe05bbc 3265 return NULL;
ba395927
KA		 3266 }
3267
f76aec76
KA		 3268 return domain;
3269}
3270
d4b709f4 3271static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3272{
3273 struct device_domain_info *info;
3274
3275 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3276 info = dev->archdata.iommu;
147202aa
DW		 3277 if (likely(info))
3278 return info->domain;
3279
3280 return __get_valid_domain_for_dev(dev);
3281}
3282
ecb509ec 3283/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3284static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3285{
3286 int found;
3287
3d89194a 3288 if (iommu_dummy(dev))
1e4c64c4
DW		 3289 return 1;
3290
2c2e2c38 3291 if (!iommu_identity_mapping)
1e4c64c4 3292 return 0;
2c2e2c38 3293
9b226624 3294 found = identity_mapping(dev);
2c2e2c38 3295 if (found) {
ecb509ec 3296 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3297 return 1;
3298 else {
3299 /*
3300 * 32 bit DMA is removed from si_domain and fall back
3301 * to non-identity mapping.
3302 */
e6de0f8d 3303 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3304 pr_info("32bit %s uses non-identity mapping\n",
3305 dev_name(dev));
2c2e2c38
FY		 3306 return 0;
3307 }
3308 } else {
3309 /*
3310 * In case of a detached 64 bit DMA device from vm, the device
3311 * is put into si_domain for identity mapping.
3312 */
ecb509ec 3313 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3314 int ret;
28ccce0d 3315 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3316 if (!ret) {
9f10e5bf
JR		 3317 pr_info("64bit %s uses identity mapping\n",
3318 dev_name(dev));
2c2e2c38
FY		 3319 return 1;
3320 }
3321 }
3322 }
3323
1e4c64c4 3324 return 0;
2c2e2c38
FY		 3325}
3326
5040a918 3327static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3328 size_t size, int dir, u64 dma_mask)
f76aec76 3329{
f76aec76 3330 struct dmar_domain *domain;
5b6985ce 3331 phys_addr_t start_paddr;
f76aec76
KA		 3332 struct iova *iova;
3333 int prot = 0;
6865f0d1 3334 int ret;
8c11e798 3335 struct intel_iommu *iommu;
33041ec0 3336 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3337
3338 BUG_ON(dir == DMA_NONE);
2c2e2c38 3339
5040a918 3340 if (iommu_no_mapping(dev))
6865f0d1 3341 return paddr;
f76aec76 3342
5040a918 3343 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3344 if (!domain)
3345 return 0;
3346
8c11e798 3347 iommu = domain_get_iommu(domain);
88cb6a74 3348 size = aligned_nrpages(paddr, size);
f76aec76 3349
5040a918 3350 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3351 if (!iova)
3352 goto error;
3353
ba395927
KA		 3354 /*
3355 * Check if DMAR supports zero-length reads on write only
3356 * mappings..
3357 */
3358 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3359 !cap_zlr(iommu->cap))
ba395927
KA		 3360 prot |= DMA_PTE_READ;
3361 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3362 prot |= DMA_PTE_WRITE;
3363 /*
6865f0d1 3364 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3365 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3366 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3367 * is not a big problem
3368 */
0ab36de2 3369 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3370 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3371 if (ret)
3372 goto error;
3373
1f0ef2aa
DW		 3374 /* it's a non-present to present mapping. Only flush if caching mode */
3375 if (cap_caching_mode(iommu->cap))
a1ddcbe9
JR		 3376 iommu_flush_iotlb_psi(iommu, domain,
3377 mm_to_dma_pfn(iova->pfn_lo),
3378 size, 0, 1);
1f0ef2aa 3379 else
8c11e798 3380 iommu_flush_write_buffer(iommu);
f76aec76 3381
03d6a246
DW		 3382 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3383 start_paddr += paddr & ~PAGE_MASK;
3384 return start_paddr;
ba395927 3385
ba395927 3386error:
f76aec76
KA		 3387 if (iova)
3388 __free_iova(&domain->iovad, iova);
9f10e5bf 3389 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3390 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3391 return 0;
3392}
3393
ffbbef5c
FT		 3394static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3395 unsigned long offset, size_t size,
3396 enum dma_data_direction dir,
3397 struct dma_attrs *attrs)
bb9e6d65 3398{
ffbbef5c 3399 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3400 dir, *dev->dma_mask);
bb9e6d65
FT		 3401}
3402
5e0d2a6f 3403static void flush_unmaps(void)
3404{
80b20dd8 3405 int i, j;
5e0d2a6f 3406
5e0d2a6f 3407 timer_on = 0;
3408
3409 /* just flush them all */
3410 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3411 struct intel_iommu *iommu = g_iommus[i];
3412 if (!iommu)
3413 continue;
c42d9f32 3414
9dd2fe89
YZ		 3415 if (!deferred_flush[i].next)
3416 continue;
3417
78d5f0f5
NA		 3418 /* In caching mode, global flushes turn emulation expensive */
3419 if (!cap_caching_mode(iommu->cap))
3420 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3421 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3422 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3423 unsigned long mask;
3424 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3425 struct dmar_domain *domain = deferred_flush[i].domain[j];
3426
3427 /* On real hardware multiple invalidations are expensive */
3428 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3429 iommu_flush_iotlb_psi(iommu, domain,
a156ef99 3430 iova->pfn_lo, iova_size(iova),
ea8ea460 3431 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3432 else {
a156ef99 3433 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3434 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3435 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3436 }
93a23a72 3437 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3438 if (deferred_flush[i].freelist[j])
3439 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3440 }
9dd2fe89 3441 deferred_flush[i].next = 0;
5e0d2a6f 3442 }
3443
5e0d2a6f 3444 list_size = 0;
5e0d2a6f 3445}
3446
3447static void flush_unmaps_timeout(unsigned long data)
3448{
80b20dd8 3449 unsigned long flags;
3450
3451 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3452 flush_unmaps();
80b20dd8 3453 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3454}
3455
ea8ea460 3456static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3457{
3458 unsigned long flags;
80b20dd8 3459 int next, iommu_id;
8c11e798 3460 struct intel_iommu *iommu;
5e0d2a6f 3461
3462 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3463 if (list_size == HIGH_WATER_MARK)
3464 flush_unmaps();
3465
8c11e798
WH		 3466 iommu = domain_get_iommu(dom);
3467 iommu_id = iommu->seq_id;
c42d9f32 3468
80b20dd8 3469 next = deferred_flush[iommu_id].next;
3470 deferred_flush[iommu_id].domain[next] = dom;
3471 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3472 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3473 deferred_flush[iommu_id].next++;
5e0d2a6f 3474
3475 if (!timer_on) {
3476 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3477 timer_on = 1;
3478 }
3479 list_size++;
3480 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3481}
3482
d41a4adb 3483static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3484{
f76aec76 3485 struct dmar_domain *domain;
d794dc9b 3486 unsigned long start_pfn, last_pfn;
ba395927 3487 struct iova *iova;
8c11e798 3488 struct intel_iommu *iommu;
ea8ea460 3489 struct page *freelist;
ba395927 3490
73676832 3491 if (iommu_no_mapping(dev))
f76aec76 3492 return;
2c2e2c38 3493
1525a29a 3494 domain = find_domain(dev);
ba395927
KA		 3495 BUG_ON(!domain);
3496
8c11e798
WH		 3497 iommu = domain_get_iommu(domain);
3498
ba395927 3499 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3500 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3501 (unsigned long long)dev_addr))
ba395927 3502 return;
ba395927 3503
d794dc9b
DW		 3504 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3505 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3506
d794dc9b 3507 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3508 dev_name(dev), start_pfn, last_pfn);
ba395927 3509
ea8ea460 3510 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3511
5e0d2a6f 3512 if (intel_iommu_strict) {
a1ddcbe9 3513 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
ea8ea460 3514 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3515 /* free iova */
3516 __free_iova(&domain->iovad, iova);
ea8ea460 3517 dma_free_pagelist(freelist);
5e0d2a6f 3518 } else {
ea8ea460 3519 add_unmap(domain, iova, freelist);
5e0d2a6f 3520 /*
3521 * queue up the release of the unmap to save the 1/6th of the
3522 * cpu used up by the iotlb flush operation...
3523 */
5e0d2a6f 3524 }
ba395927
KA		 3525}
3526
d41a4adb
JL		 3527static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3528 size_t size, enum dma_data_direction dir,
3529 struct dma_attrs *attrs)
3530{
3531 intel_unmap(dev, dev_addr);
3532}
3533
5040a918 3534static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3535 dma_addr_t *dma_handle, gfp_t flags,
3536 struct dma_attrs *attrs)
ba395927 3537{
36746436 3538 struct page *page = NULL;
ba395927
KA		 3539 int order;
3540
5b6985ce 3541 size = PAGE_ALIGN(size);
ba395927 3542 order = get_order(size);
e8bb910d 3543
5040a918 3544 if (!iommu_no_mapping(dev))
e8bb910d 3545 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3546 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3547 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3548 flags |= GFP_DMA;
3549 else
3550 flags |= GFP_DMA32;
3551 }
ba395927 3552
36746436
AM		 3553 if (flags & __GFP_WAIT) {
3554 unsigned int count = size >> PAGE_SHIFT;
3555
3556 page = dma_alloc_from_contiguous(dev, count, order);
3557 if (page && iommu_no_mapping(dev) &&
3558 page_to_phys(page) + size > dev->coherent_dma_mask) {
3559 dma_release_from_contiguous(dev, page, count);
3560 page = NULL;
3561 }
3562 }
3563
3564 if (!page)
3565 page = alloc_pages(flags, order);
3566 if (!page)
ba395927 3567 return NULL;
36746436 3568 memset(page_address(page), 0, size);
ba395927 3569
36746436 3570 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3571 DMA_BIDIRECTIONAL,
5040a918 3572 dev->coherent_dma_mask);
ba395927 3573 if (*dma_handle)
36746436
AM		 3574 return page_address(page);
3575 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3576 __free_pages(page, order);
3577
ba395927
KA		 3578 return NULL;
3579}
3580
5040a918 3581static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3582 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3583{
3584 int order;
36746436 3585 struct page *page = virt_to_page(vaddr);
ba395927 3586
5b6985ce 3587 size = PAGE_ALIGN(size);
ba395927
KA		 3588 order = get_order(size);
3589
d41a4adb 3590 intel_unmap(dev, dma_handle);
36746436
AM		 3591 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3592 __free_pages(page, order);
ba395927
KA		 3593}
3594
5040a918 3595static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3596 int nelems, enum dma_data_direction dir,
3597 struct dma_attrs *attrs)
ba395927 3598{
d41a4adb 3599 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3600}
3601
ba395927 3602static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3603 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3604{
3605 int i;
c03ab37c 3606 struct scatterlist *sg;
ba395927 3607
c03ab37c 3608 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3609 BUG_ON(!sg_page(sg));
4cf2e75d 3610 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3611 sg->dma_length = sg->length;
ba395927
KA		 3612 }
3613 return nelems;
3614}
3615
5040a918 3616static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3617 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3618{
ba395927 3619 int i;
ba395927 3620 struct dmar_domain *domain;
f76aec76
KA		 3621 size_t size = 0;
3622 int prot = 0;
f76aec76
KA		 3623 struct iova *iova = NULL;
3624 int ret;
c03ab37c 3625 struct scatterlist *sg;
b536d24d 3626 unsigned long start_vpfn;
8c11e798 3627 struct intel_iommu *iommu;
ba395927
KA		 3628
3629 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3630 if (iommu_no_mapping(dev))
3631 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3632
5040a918 3633 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3634 if (!domain)
3635 return 0;
3636
8c11e798
WH		 3637 iommu = domain_get_iommu(domain);
3638
b536d24d 3639 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3640 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3641
5040a918
DW		 3642 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3643 *dev->dma_mask);
f76aec76 3644 if (!iova) {
c03ab37c 3645 sglist->dma_length = 0;
f76aec76
KA		 3646 return 0;
3647 }
3648
3649 /*
3650 * Check if DMAR supports zero-length reads on write only
3651 * mappings..
3652 */
3653 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3654 !cap_zlr(iommu->cap))
f76aec76
KA		 3655 prot |= DMA_PTE_READ;
3656 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3657 prot |= DMA_PTE_WRITE;
3658
b536d24d 3659 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3660
f532959b 3661 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3662 if (unlikely(ret)) {
e1605495
DW		 3663 dma_pte_free_pagetable(domain, start_vpfn,
3664 start_vpfn + size - 1);
e1605495
DW		 3665 __free_iova(&domain->iovad, iova);
3666 return 0;
ba395927
KA		 3667 }
3668
1f0ef2aa
DW		 3669 /* it's a non-present to present mapping. Only flush if caching mode */
3670 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3671 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3672 else
8c11e798 3673 iommu_flush_write_buffer(iommu);
1f0ef2aa 3674
ba395927
KA		 3675 return nelems;
3676}
3677
dfb805e8
FT		 3678static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3679{
3680 return !dma_addr;
3681}
3682
160c1d8e 3683struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3684 .alloc = intel_alloc_coherent,
3685 .free = intel_free_coherent,
ba395927
KA		 3686 .map_sg = intel_map_sg,
3687 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3688 .map_page = intel_map_page,
3689 .unmap_page = intel_unmap_page,
dfb805e8 3690 .mapping_error = intel_mapping_error,
ba395927
KA		 3691};
3692
3693static inline int iommu_domain_cache_init(void)
3694{
3695 int ret = 0;
3696
3697 iommu_domain_cache = kmem_cache_create("iommu_domain",
3698 sizeof(struct dmar_domain),
3699 0,
3700 SLAB_HWCACHE_ALIGN,
3701
3702 NULL);
3703 if (!iommu_domain_cache) {
9f10e5bf 3704 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3705 ret = -ENOMEM;
3706 }
3707
3708 return ret;
3709}
3710
3711static inline int iommu_devinfo_cache_init(void)
3712{
3713 int ret = 0;
3714
3715 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3716 sizeof(struct device_domain_info),
3717 0,
3718 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3719 NULL);
3720 if (!iommu_devinfo_cache) {
9f10e5bf 3721 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3722 ret = -ENOMEM;
3723 }
3724
3725 return ret;
3726}
3727
ba395927
KA		 3728static int __init iommu_init_mempool(void)
3729{
3730 int ret;
3731 ret = iommu_iova_cache_init();
3732 if (ret)
3733 return ret;
3734
3735 ret = iommu_domain_cache_init();
3736 if (ret)
3737 goto domain_error;
3738
3739 ret = iommu_devinfo_cache_init();
3740 if (!ret)
3741 return ret;
3742
3743 kmem_cache_destroy(iommu_domain_cache);
3744domain_error:
85b45456 3745 iommu_iova_cache_destroy();
ba395927
KA		 3746
3747 return -ENOMEM;
3748}
3749
3750static void __init iommu_exit_mempool(void)
3751{
3752 kmem_cache_destroy(iommu_devinfo_cache);
3753 kmem_cache_destroy(iommu_domain_cache);
85b45456 3754 iommu_iova_cache_destroy();
ba395927
KA		 3755}
3756
556ab45f
DW		 3757static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3758{
3759 struct dmar_drhd_unit *drhd;
3760 u32 vtbar;
3761 int rc;
3762
3763 /* We know that this device on this chipset has its own IOMMU.
3764 * If we find it under a different IOMMU, then the BIOS is lying
3765 * to us. Hope that the IOMMU for this device is actually
3766 * disabled, and it needs no translation...
3767 */
3768 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3769 if (rc) {
3770 /* "can't" happen */
3771 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3772 return;
3773 }
3774 vtbar &= 0xffff0000;
3775
3776 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3777 drhd = dmar_find_matched_drhd_unit(pdev);
3778 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3779 TAINT_FIRMWARE_WORKAROUND,
3780 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3781 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3782}
3783DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3784
ba395927
KA		 3785static void __init init_no_remapping_devices(void)
3786{
3787 struct dmar_drhd_unit *drhd;
832bd858 3788 struct device *dev;
b683b230 3789 int i;
ba395927
KA		 3790
3791 for_each_drhd_unit(drhd) {
3792 if (!drhd->include_all) {
b683b230
JL		 3793 for_each_active_dev_scope(drhd->devices,
3794 drhd->devices_cnt, i, dev)
3795 break;
832bd858 3796 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3797 if (i == drhd->devices_cnt)
3798 drhd->ignored = 1;
3799 }
3800 }
3801
7c919779 3802 for_each_active_drhd_unit(drhd) {
7c919779 3803 if (drhd->include_all)
ba395927
KA		 3804 continue;
3805
b683b230
JL		 3806 for_each_active_dev_scope(drhd->devices,
3807 drhd->devices_cnt, i, dev)
832bd858 3808 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3809 break;
ba395927
KA		 3810 if (i < drhd->devices_cnt)
3811 continue;
3812
c0771df8
DW		 3813 /* This IOMMU has *only* gfx devices. Either bypass it or
3814 set the gfx_mapped flag, as appropriate */
3815 if (dmar_map_gfx) {
3816 intel_iommu_gfx_mapped = 1;
3817 } else {
3818 drhd->ignored = 1;
b683b230
JL		 3819 for_each_active_dev_scope(drhd->devices,
3820 drhd->devices_cnt, i, dev)
832bd858 3821 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3822 }
3823 }
3824}
3825
f59c7b69
FY		 3826#ifdef CONFIG_SUSPEND
3827static int init_iommu_hw(void)
3828{
3829 struct dmar_drhd_unit *drhd;
3830 struct intel_iommu *iommu = NULL;
3831
3832 for_each_active_iommu(iommu, drhd)
3833 if (iommu->qi)
3834 dmar_reenable_qi(iommu);
3835
b779260b
JC		 3836 for_each_iommu(iommu, drhd) {
3837 if (drhd->ignored) {
3838 /*
3839 * we always have to disable PMRs or DMA may fail on
3840 * this device
3841 */
3842 if (force_on)
3843 iommu_disable_protect_mem_regions(iommu);
3844 continue;
3845 }
3846
f59c7b69
FY		 3847 iommu_flush_write_buffer(iommu);
3848
3849 iommu_set_root_entry(iommu);
3850
3851 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3852 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3853 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3854 iommu_enable_translation(iommu);
b94996c9 3855 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3856 }
3857
3858 return 0;
3859}
3860
3861static void iommu_flush_all(void)
3862{
3863 struct dmar_drhd_unit *drhd;
3864 struct intel_iommu *iommu;
3865
3866 for_each_active_iommu(iommu, drhd) {
3867 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3868 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3869 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3870 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3871 }
3872}
3873
134fac3f 3874static int iommu_suspend(void)
f59c7b69
FY		 3875{
3876 struct dmar_drhd_unit *drhd;
3877 struct intel_iommu *iommu = NULL;
3878 unsigned long flag;
3879
3880 for_each_active_iommu(iommu, drhd) {
3881 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3882 GFP_ATOMIC);
3883 if (!iommu->iommu_state)
3884 goto nomem;
3885 }
3886
3887 iommu_flush_all();
3888
3889 for_each_active_iommu(iommu, drhd) {
3890 iommu_disable_translation(iommu);
3891
1f5b3c3f 3892 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3893
3894 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3895 readl(iommu->reg + DMAR_FECTL_REG);
3896 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3897 readl(iommu->reg + DMAR_FEDATA_REG);
3898 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3899 readl(iommu->reg + DMAR_FEADDR_REG);
3900 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3901 readl(iommu->reg + DMAR_FEUADDR_REG);
3902
1f5b3c3f 3903 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3904 }
3905 return 0;
3906
3907nomem:
3908 for_each_active_iommu(iommu, drhd)
3909 kfree(iommu->iommu_state);
3910
3911 return -ENOMEM;
3912}
3913
134fac3f 3914static void iommu_resume(void)
f59c7b69
FY		 3915{
3916 struct dmar_drhd_unit *drhd;
3917 struct intel_iommu *iommu = NULL;
3918 unsigned long flag;
3919
3920 if (init_iommu_hw()) {
b779260b
JC		 3921 if (force_on)
3922 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3923 else
3924 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3925 return;
f59c7b69
FY		 3926 }
3927
3928 for_each_active_iommu(iommu, drhd) {
3929
1f5b3c3f 3930 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3931
3932 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3933 iommu->reg + DMAR_FECTL_REG);
3934 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3935 iommu->reg + DMAR_FEDATA_REG);
3936 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3937 iommu->reg + DMAR_FEADDR_REG);
3938 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3939 iommu->reg + DMAR_FEUADDR_REG);
3940
1f5b3c3f 3941 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3942 }
3943
3944 for_each_active_iommu(iommu, drhd)
3945 kfree(iommu->iommu_state);
f59c7b69
FY		 3946}
3947
134fac3f 3948static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3949 .resume = iommu_resume,
3950 .suspend = iommu_suspend,
3951};
3952
134fac3f 3953static void __init init_iommu_pm_ops(void)
f59c7b69 3954{
134fac3f 3955 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3956}
3957
3958#else
99592ba4 3959static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3960#endif /* CONFIG_PM */
3961
318fe7df 3962
c2a0b538 3963int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3964{
3965 struct acpi_dmar_reserved_memory *rmrr;
3966 struct dmar_rmrr_unit *rmrru;
3967
3968 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3969 if (!rmrru)
3970 return -ENOMEM;
3971
3972 rmrru->hdr = header;
3973 rmrr = (struct acpi_dmar_reserved_memory *)header;
3974 rmrru->base_address = rmrr->base_address;
3975 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3976 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3977 ((void *)rmrr) + rmrr->header.length,
3978 &rmrru->devices_cnt);
3979 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3980 kfree(rmrru);
3981 return -ENOMEM;
3982 }
318fe7df 3983
2e455289 3984 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3985
2e455289 3986 return 0;
318fe7df
SS		 3987}
3988
6b197249
JL		 3989static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3990{
3991 struct dmar_atsr_unit *atsru;
3992 struct acpi_dmar_atsr *tmp;
3993
3994 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3995 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3996 if (atsr->segment != tmp->segment)
3997 continue;
3998 if (atsr->header.length != tmp->header.length)
3999 continue;
4000 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4001 return atsru;
4002 }
4003
4004 return NULL;
4005}
4006
4007int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4008{
4009 struct acpi_dmar_atsr *atsr;
4010 struct dmar_atsr_unit *atsru;
4011
6b197249
JL		 4012 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4013 return 0;
4014
318fe7df 4015 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4016 atsru = dmar_find_atsr(atsr);
4017 if (atsru)
4018 return 0;
4019
4020 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4021 if (!atsru)
4022 return -ENOMEM;
4023
6b197249
JL		 4024 /*
4025 * If memory is allocated from slab by ACPI _DSM method, we need to
4026 * copy the memory content because the memory buffer will be freed
4027 * on return.
4028 */
4029 atsru->hdr = (void *)(atsru + 1);
4030 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4031 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4032 if (!atsru->include_all) {
4033 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4034 (void *)atsr + atsr->header.length,
4035 &atsru->devices_cnt);
4036 if (atsru->devices_cnt && atsru->devices == NULL) {
4037 kfree(atsru);
4038 return -ENOMEM;
4039 }
4040 }
318fe7df 4041
0e242612 4042 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4043
4044 return 0;
4045}
4046
9bdc531e
JL		 4047static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4048{
4049 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4050 kfree(atsru);
4051}
4052
6b197249
JL		 4053int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4054{
4055 struct acpi_dmar_atsr *atsr;
4056 struct dmar_atsr_unit *atsru;
4057
4058 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4059 atsru = dmar_find_atsr(atsr);
4060 if (atsru) {
4061 list_del_rcu(&atsru->list);
4062 synchronize_rcu();
4063 intel_iommu_free_atsr(atsru);
4064 }
4065
4066 return 0;
4067}
4068
4069int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4070{
4071 int i;
4072 struct device *dev;
4073 struct acpi_dmar_atsr *atsr;
4074 struct dmar_atsr_unit *atsru;
4075
4076 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4077 atsru = dmar_find_atsr(atsr);
4078 if (!atsru)
4079 return 0;
4080
4081 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4082 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4083 i, dev)
4084 return -EBUSY;
4085
4086 return 0;
4087}
4088
ffebeb46
JL		 4089static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4090{
4091 int sp, ret = 0;
4092 struct intel_iommu *iommu = dmaru->iommu;
4093
4094 if (g_iommus[iommu->seq_id])
4095 return 0;
4096
4097 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4098 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4099 iommu->name);
4100 return -ENXIO;
4101 }
4102 if (!ecap_sc_support(iommu->ecap) &&
4103 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4104 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4105 iommu->name);
4106 return -ENXIO;
4107 }
4108 sp = domain_update_iommu_superpage(iommu) - 1;
4109 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4110 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4111 iommu->name);
4112 return -ENXIO;
4113 }
4114
4115 /*
4116 * Disable translation if already enabled prior to OS handover.
4117 */
4118 if (iommu->gcmd & DMA_GCMD_TE)
4119 iommu_disable_translation(iommu);
4120
4121 g_iommus[iommu->seq_id] = iommu;
4122 ret = iommu_init_domains(iommu);
4123 if (ret == 0)
4124 ret = iommu_alloc_root_entry(iommu);
4125 if (ret)
4126 goto out;
4127
4128 if (dmaru->ignored) {
4129 /*
4130 * we always have to disable PMRs or DMA may fail on this device
4131 */
4132 if (force_on)
4133 iommu_disable_protect_mem_regions(iommu);
4134 return 0;
4135 }
4136
4137 intel_iommu_init_qi(iommu);
4138 iommu_flush_write_buffer(iommu);
4139 ret = dmar_set_interrupt(iommu);
4140 if (ret)
4141 goto disable_iommu;
4142
4143 iommu_set_root_entry(iommu);
4144 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4145 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4146 iommu_enable_translation(iommu);
4147
ffebeb46
JL		 4148 iommu_disable_protect_mem_regions(iommu);
4149 return 0;
4150
4151disable_iommu:
4152 disable_dmar_iommu(iommu);
4153out:
4154 free_dmar_iommu(iommu);
4155 return ret;
4156}
4157
6b197249
JL		 4158int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4159{
ffebeb46
JL		 4160 int ret = 0;
4161 struct intel_iommu *iommu = dmaru->iommu;
4162
4163 if (!intel_iommu_enabled)
4164 return 0;
4165 if (iommu == NULL)
4166 return -EINVAL;
4167
4168 if (insert) {
4169 ret = intel_iommu_add(dmaru);
4170 } else {
4171 disable_dmar_iommu(iommu);
4172 free_dmar_iommu(iommu);
4173 }
4174
4175 return ret;
6b197249
JL		 4176}
4177
9bdc531e
JL		 4178static void intel_iommu_free_dmars(void)
4179{
4180 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4181 struct dmar_atsr_unit *atsru, *atsr_n;
4182
4183 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4184 list_del(&rmrru->list);
4185 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4186 kfree(rmrru);
318fe7df
SS		 4187 }
4188
9bdc531e
JL		 4189 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4190 list_del(&atsru->list);
4191 intel_iommu_free_atsr(atsru);
4192 }
318fe7df
SS		 4193}
4194
4195int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4196{
b683b230 4197 int i, ret = 1;
318fe7df 4198 struct pci_bus *bus;
832bd858
DW		 4199 struct pci_dev *bridge = NULL;
4200 struct device *tmp;
318fe7df
SS		 4201 struct acpi_dmar_atsr *atsr;
4202 struct dmar_atsr_unit *atsru;
4203
4204 dev = pci_physfn(dev);
318fe7df 4205 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4206 bridge = bus->self;
318fe7df 4207 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4208 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4209 return 0;
b5f82ddf 4210 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4211 break;
318fe7df 4212 }
b5f82ddf
JL		 4213 if (!bridge)
4214 return 0;
318fe7df 4215
0e242612 4216 rcu_read_lock();
b5f82ddf
JL		 4217 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4218 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4219 if (atsr->segment != pci_domain_nr(dev->bus))
4220 continue;
4221
b683b230 4222 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4223 if (tmp == &bridge->dev)
b683b230 4224 goto out;
b5f82ddf
JL		 4225
4226 if (atsru->include_all)
b683b230 4227 goto out;
b5f82ddf 4228 }
b683b230
JL		 4229 ret = 0;
4230out:
0e242612 4231 rcu_read_unlock();
318fe7df 4232
b683b230 4233 return ret;
318fe7df
SS		 4234}
4235
59ce0515
JL		 4236int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4237{
4238 int ret = 0;
4239 struct dmar_rmrr_unit *rmrru;
4240 struct dmar_atsr_unit *atsru;
4241 struct acpi_dmar_atsr *atsr;
4242 struct acpi_dmar_reserved_memory *rmrr;
4243
4244 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4245 return 0;
4246
4247 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4248 rmrr = container_of(rmrru->hdr,
4249 struct acpi_dmar_reserved_memory, header);
4250 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4251 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4252 ((void *)rmrr) + rmrr->header.length,
4253 rmrr->segment, rmrru->devices,
4254 rmrru->devices_cnt);
27e24950 4255 if(ret < 0)
59ce0515
JL		 4256 return ret;
4257 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4258 dmar_remove_dev_scope(info, rmrr->segment,
4259 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4260 }
4261 }
4262
4263 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4264 if (atsru->include_all)
4265 continue;
4266
4267 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4268 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4269 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4270 (void *)atsr + atsr->header.length,
4271 atsr->segment, atsru->devices,
4272 atsru->devices_cnt);
4273 if (ret > 0)
4274 break;
4275 else if(ret < 0)
4276 return ret;
4277 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4278 if (dmar_remove_dev_scope(info, atsr->segment,
4279 atsru->devices, atsru->devices_cnt))
4280 break;
4281 }
4282 }
4283
4284 return 0;
4285}
4286
99dcaded
FY		 4287/*
4288 * Here we only respond to action of unbound device from driver.
4289 *
4290 * Added device is not attached to its DMAR domain here yet. That will happen
4291 * when mapping the device to iova.
4292 */
4293static int device_notifier(struct notifier_block *nb,
4294 unsigned long action, void *data)
4295{
4296 struct device *dev = data;
99dcaded
FY		 4297 struct dmar_domain *domain;
4298
3d89194a 4299 if (iommu_dummy(dev))
44cd613c
DW		 4300 return 0;
4301
1196c2fb 4302 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4303 return 0;
4304
1525a29a 4305 domain = find_domain(dev);
99dcaded
FY		 4306 if (!domain)
4307 return 0;
4308
3a5670e8 4309 down_read(&dmar_global_lock);
e6de0f8d 4310 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4311 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4312 domain_exit(domain);
3a5670e8 4313 up_read(&dmar_global_lock);
a97590e5 4314
99dcaded
FY		 4315 return 0;
4316}
4317
4318static struct notifier_block device_nb = {
4319 .notifier_call = device_notifier,
4320};
4321
75f05569
JL		 4322static int intel_iommu_memory_notifier(struct notifier_block *nb,
4323 unsigned long val, void *v)
4324{
4325 struct memory_notify *mhp = v;
4326 unsigned long long start, end;
4327 unsigned long start_vpfn, last_vpfn;
4328
4329 switch (val) {
4330 case MEM_GOING_ONLINE:
4331 start = mhp->start_pfn << PAGE_SHIFT;
4332 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4333 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4334 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4335 start, end);
4336 return NOTIFY_BAD;
4337 }
4338 break;
4339
4340 case MEM_OFFLINE:
4341 case MEM_CANCEL_ONLINE:
4342 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4343 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4344 while (start_vpfn <= last_vpfn) {
4345 struct iova *iova;
4346 struct dmar_drhd_unit *drhd;
4347 struct intel_iommu *iommu;
ea8ea460 4348 struct page *freelist;
75f05569
JL		 4349
4350 iova = find_iova(&si_domain->iovad, start_vpfn);
4351 if (iova == NULL) {
9f10e5bf 4352 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4353 start_vpfn);
4354 break;
4355 }
4356
4357 iova = split_and_remove_iova(&si_domain->iovad, iova,
4358 start_vpfn, last_vpfn);
4359 if (iova == NULL) {
9f10e5bf 4360 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4361 start_vpfn, last_vpfn);
4362 return NOTIFY_BAD;
4363 }
4364
ea8ea460
DW		 4365 freelist = domain_unmap(si_domain, iova->pfn_lo,
4366 iova->pfn_hi);
4367
75f05569
JL		 4368 rcu_read_lock();
4369 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4370 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4371 iova->pfn_lo, iova_size(iova),
ea8ea460 4372 !freelist, 0);
75f05569 4373 rcu_read_unlock();
ea8ea460 4374 dma_free_pagelist(freelist);
75f05569
JL		 4375
4376 start_vpfn = iova->pfn_hi + 1;
4377 free_iova_mem(iova);
4378 }
4379 break;
4380 }
4381
4382 return NOTIFY_OK;
4383}
4384
4385static struct notifier_block intel_iommu_memory_nb = {
4386 .notifier_call = intel_iommu_memory_notifier,
4387 .priority = 0
4388};
4389
a5459cfe
AW		 4390
4391static ssize_t intel_iommu_show_version(struct device *dev,
4392 struct device_attribute *attr,
4393 char *buf)
4394{
4395 struct intel_iommu *iommu = dev_get_drvdata(dev);
4396 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4397 return sprintf(buf, "%d:%d\n",
4398 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4399}
4400static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4401
4402static ssize_t intel_iommu_show_address(struct device *dev,
4403 struct device_attribute *attr,
4404 char *buf)
4405{
4406 struct intel_iommu *iommu = dev_get_drvdata(dev);
4407 return sprintf(buf, "%llx\n", iommu->reg_phys);
4408}
4409static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4410
4411static ssize_t intel_iommu_show_cap(struct device *dev,
4412 struct device_attribute *attr,
4413 char *buf)
4414{
4415 struct intel_iommu *iommu = dev_get_drvdata(dev);
4416 return sprintf(buf, "%llx\n", iommu->cap);
4417}
4418static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4419
4420static ssize_t intel_iommu_show_ecap(struct device *dev,
4421 struct device_attribute *attr,
4422 char *buf)
4423{
4424 struct intel_iommu *iommu = dev_get_drvdata(dev);
4425 return sprintf(buf, "%llx\n", iommu->ecap);
4426}
4427static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4428
2238c082
AW		 4429static ssize_t intel_iommu_show_ndoms(struct device *dev,
4430 struct device_attribute *attr,
4431 char *buf)
4432{
4433 struct intel_iommu *iommu = dev_get_drvdata(dev);
4434 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4435}
4436static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4437
4438static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4439 struct device_attribute *attr,
4440 char *buf)
4441{
4442 struct intel_iommu *iommu = dev_get_drvdata(dev);
4443 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4444 cap_ndoms(iommu->cap)));
4445}
4446static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4447
a5459cfe
AW		 4448static struct attribute *intel_iommu_attrs[] = {
4449 &dev_attr_version.attr,
4450 &dev_attr_address.attr,
4451 &dev_attr_cap.attr,
4452 &dev_attr_ecap.attr,
2238c082
AW		 4453 &dev_attr_domains_supported.attr,
4454 &dev_attr_domains_used.attr,
a5459cfe
AW		 4455 NULL,
4456};
4457
4458static struct attribute_group intel_iommu_group = {
4459 .name = "intel-iommu",
4460 .attrs = intel_iommu_attrs,
4461};
4462
4463const struct attribute_group *intel_iommu_groups[] = {
4464 &intel_iommu_group,
4465 NULL,
4466};
4467
ba395927
KA		 4468int __init intel_iommu_init(void)
4469{
9bdc531e 4470 int ret = -ENODEV;
3a93c841 4471 struct dmar_drhd_unit *drhd;
7c919779 4472 struct intel_iommu *iommu;
ba395927 4473
a59b50e9
JC		 4474 /* VT-d is required for a TXT/tboot launch, so enforce that */
4475 force_on = tboot_force_iommu();
4476
3a5670e8
JL		 4477 if (iommu_init_mempool()) {
4478 if (force_on)
4479 panic("tboot: Failed to initialize iommu memory\n");
4480 return -ENOMEM;
4481 }
4482
4483 down_write(&dmar_global_lock);
a59b50e9
JC		 4484 if (dmar_table_init()) {
4485 if (force_on)
4486 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4487 goto out_free_dmar;
a59b50e9 4488 }
ba395927 4489
c2c7286a 4490 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4491 if (force_on)
4492 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4493 goto out_free_dmar;
a59b50e9 4494 }
1886e8a9 4495
75f1cdf1 4496 if (no_iommu || dmar_disabled)
9bdc531e 4497 goto out_free_dmar;
2ae21010 4498
318fe7df 4499 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4500 pr_info("No RMRR found\n");
318fe7df
SS		 4501
4502 if (list_empty(&dmar_atsr_units))
9f10e5bf 4503 pr_info("No ATSR found\n");
318fe7df 4504
51a63e67
JC		 4505 if (dmar_init_reserved_ranges()) {
4506 if (force_on)
4507 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4508 goto out_free_reserved_range;
51a63e67 4509 }
ba395927
KA		 4510
4511 init_no_remapping_devices();
4512
b779260b 4513 ret = init_dmars();
ba395927 4514 if (ret) {
a59b50e9
JC		 4515 if (force_on)
4516 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4517 pr_err("Initialization failed\n");
9bdc531e 4518 goto out_free_reserved_range;
ba395927 4519 }
3a5670e8 4520 up_write(&dmar_global_lock);
9f10e5bf 4521 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4522
5e0d2a6f 4523 init_timer(&unmap_timer);
75f1cdf1
FT		 4524#ifdef CONFIG_SWIOTLB
4525 swiotlb = 0;
4526#endif
19943b0e 4527 dma_ops = &intel_dma_ops;
4ed0d3e6 4528
134fac3f 4529 init_iommu_pm_ops();
a8bcbb0d 4530
a5459cfe
AW		 4531 for_each_active_iommu(iommu, drhd)
4532 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4533 intel_iommu_groups,
2439d4aa 4534 "%s", iommu->name);
a5459cfe 4535
4236d97d 4536 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4537 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4538 if (si_domain && !hw_pass_through)
4539 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4540
8bc1f85c
ED		 4541 intel_iommu_enabled = 1;
4542
ba395927 4543 return 0;
9bdc531e
JL		 4544
4545out_free_reserved_range:
4546 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4547out_free_dmar:
4548 intel_iommu_free_dmars();
3a5670e8
JL		 4549 up_write(&dmar_global_lock);
4550 iommu_exit_mempool();
9bdc531e 4551 return ret;
ba395927 4552}
e820482c 4553
579305f7
AW		 4554static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4555{
4556 struct intel_iommu *iommu = opaque;
4557
4558 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4559 return 0;
4560}
4561
4562/*
4563 * NB - intel-iommu lacks any sort of reference counting for the users of
4564 * dependent devices. If multiple endpoints have intersecting dependent
4565 * devices, unbinding the driver from any one of them will possibly leave
4566 * the others unable to operate.
4567 */
3199aa6b 4568static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4569 struct device *dev)
3199aa6b 4570{
0bcb3e28 4571 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4572 return;
4573
579305f7 4574 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4575}
4576
e6de0f8d
JR		 4577static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4578 struct device *dev)
c7151a8d 4579{
b608ac3b 4580 struct device_domain_info *info;
c7151a8d
WH		 4581 struct intel_iommu *iommu;
4582 unsigned long flags;
156baca8 4583 u8 bus, devfn;
c7151a8d 4584
bf9c9eda 4585 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4586 if (!iommu)
4587 return;
4588
b608ac3b 4589 info = dev->archdata.iommu;
c7151a8d 4590
b608ac3b
JR		 4591 if (WARN_ON(!info))
4592 return;
c7151a8d 4593
b608ac3b
JR		 4594 spin_lock_irqsave(&device_domain_lock, flags);
4595 unlink_domain_info(info);
3e7abe25
RD		 4596 spin_unlock_irqrestore(&device_domain_lock, flags);
4597
b608ac3b
JR		 4598 iommu_disable_dev_iotlb(info);
4599 iommu_detach_dev(iommu, info->bus, info->devfn);
4600 iommu_detach_dependent_devices(iommu, dev);
4601 free_devinfo_mem(info);
4602 domain_detach_iommu(domain, iommu);
4603
4604 spin_lock_irqsave(&domain->iommu_lock, flags);
4605 if (!domain->iommu_refcnt[iommu->seq_id])
4606 iommu_detach_domain(domain, iommu);
4607 spin_unlock_irqrestore(&domain->iommu_lock, flags);
c7151a8d
WH		 4608}
4609
2c2e2c38 4610static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4611{
4612 int adjust_width;
4613
0fb5fe87
RM		 4614 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4615 DMA_32BIT_PFN);
5e98c4b1
WH		 4616 domain_reserve_special_ranges(domain);
4617
4618 /* calculate AGAW */
4619 domain->gaw = guest_width;
4620 adjust_width = guestwidth_to_adjustwidth(guest_width);
4621 domain->agaw = width_to_agaw(adjust_width);
4622
5e98c4b1 4623 domain->iommu_coherency = 0;
c5b15255 4624 domain->iommu_snooping = 0;
6dd9a7c7 4625 domain->iommu_superpage = 0;
fe40f1e0 4626 domain->max_addr = 0;
5e98c4b1
WH		 4627
4628 /* always allocate the top pgd */
4c923d47 4629 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4630 if (!domain->pgd)
4631 return -ENOMEM;
4632 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4633 return 0;
4634}
4635
00a77deb 4636static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4637{
5d450806 4638 struct dmar_domain *dmar_domain;
00a77deb
JR		 4639 struct iommu_domain *domain;
4640
4641 if (type != IOMMU_DOMAIN_UNMANAGED)
4642 return NULL;
38717946 4643
ab8dfe25 4644 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4645 if (!dmar_domain) {
9f10e5bf 4646 pr_err("Can't allocate dmar_domain\n");
00a77deb 4647 return NULL;
38717946 4648 }
2c2e2c38 4649 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4650 pr_err("Domain initialization failed\n");
92d03cc8 4651 domain_exit(dmar_domain);
00a77deb 4652 return NULL;
38717946 4653 }
8140a95d 4654 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4655
00a77deb 4656 domain = &dmar_domain->domain;
8a0e715b
JR		 4657 domain->geometry.aperture_start = 0;
4658 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4659 domain->geometry.force_aperture = true;
4660
00a77deb 4661 return domain;
38717946 4662}
38717946 4663
00a77deb 4664static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4665{
00a77deb 4666 domain_exit(to_dmar_domain(domain));
38717946 4667}
38717946 4668
4c5478c9
JR		 4669static int intel_iommu_attach_device(struct iommu_domain *domain,
4670 struct device *dev)
38717946 4671{
00a77deb 4672 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4673 struct intel_iommu *iommu;
4674 int addr_width;
156baca8 4675 u8 bus, devfn;
faa3d6f5 4676
c875d2c1
AW		 4677 if (device_is_rmrr_locked(dev)) {
4678 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4679 return -EPERM;
4680 }
4681
7207d8f9
DW		 4682 /* normally dev is not mapped */
4683 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4684 struct dmar_domain *old_domain;
4685
1525a29a 4686 old_domain = find_domain(dev);
faa3d6f5 4687 if (old_domain) {
ab8dfe25 4688 if (domain_type_is_vm_or_si(dmar_domain))
e6de0f8d 4689 dmar_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4690 else
4691 domain_remove_dev_info(old_domain);
62c22167
JR		 4692
4693 if (!domain_type_is_vm_or_si(old_domain) &&
4694 list_empty(&old_domain->devices))
4695 domain_exit(old_domain);
faa3d6f5
WH		 4696 }
4697 }
4698
156baca8 4699 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4700 if (!iommu)
4701 return -ENODEV;
4702
4703 /* check if this iommu agaw is sufficient for max mapped address */
4704 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4705 if (addr_width > cap_mgaw(iommu->cap))
4706 addr_width = cap_mgaw(iommu->cap);
4707
4708 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4709 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4710 "sufficient for the mapped address (%llx)\n",
a99c47a2 4711 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4712 return -EFAULT;
4713 }
a99c47a2
TL		 4714 dmar_domain->gaw = addr_width;
4715
4716 /*
4717 * Knock out extra levels of page tables if necessary
4718 */
4719 while (iommu->agaw < dmar_domain->agaw) {
4720 struct dma_pte *pte;
4721
4722 pte = dmar_domain->pgd;
4723 if (dma_pte_present(pte)) {
25cbff16
SY		 4724 dmar_domain->pgd = (struct dma_pte *)
4725 phys_to_virt(dma_pte_addr(pte));
7a661013 4726 free_pgtable_page(pte);
a99c47a2
TL		 4727 }
4728 dmar_domain->agaw--;
4729 }
fe40f1e0 4730
28ccce0d 4731 return domain_add_dev_info(dmar_domain, dev);
38717946 4732}
38717946 4733
4c5478c9
JR		 4734static void intel_iommu_detach_device(struct iommu_domain *domain,
4735 struct device *dev)
38717946 4736{
e6de0f8d 4737 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4738}
c7151a8d 4739
b146a1c9
JR		 4740static int intel_iommu_map(struct iommu_domain *domain,
4741 unsigned long iova, phys_addr_t hpa,
5009065d 4742 size_t size, int iommu_prot)
faa3d6f5 4743{
00a77deb 4744 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4745 u64 max_addr;
dde57a21 4746 int prot = 0;
faa3d6f5 4747 int ret;
fe40f1e0 4748
dde57a21
JR		 4749 if (iommu_prot & IOMMU_READ)
4750 prot |= DMA_PTE_READ;
4751 if (iommu_prot & IOMMU_WRITE)
4752 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4753 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4754 prot |= DMA_PTE_SNP;
dde57a21 4755
163cc52c 4756 max_addr = iova + size;
dde57a21 4757 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4758 u64 end;
4759
4760 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4761 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4762 if (end < max_addr) {
9f10e5bf 4763 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4764 "sufficient for the mapped address (%llx)\n",
8954da1f 4765 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4766 return -EFAULT;
4767 }
dde57a21 4768 dmar_domain->max_addr = max_addr;
fe40f1e0 4769 }
ad051221
DW		 4770 /* Round up size to next multiple of PAGE_SIZE, if it and
4771 the low bits of hpa would take us onto the next page */
88cb6a74 4772 size = aligned_nrpages(hpa, size);
ad051221
DW		 4773 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4774 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4775 return ret;
38717946 4776}
38717946 4777
5009065d 4778static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4779 unsigned long iova, size_t size)
38717946 4780{
00a77deb 4781 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4782 struct page *freelist = NULL;
4783 struct intel_iommu *iommu;
4784 unsigned long start_pfn, last_pfn;
4785 unsigned int npages;
42e8c186 4786 int iommu_id, level = 0;
5cf0a76f
DW		 4787
4788 /* Cope with horrid API which requires us to unmap more than the
4789 size argument if it happens to be a large-page mapping. */
4790 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4791 BUG();
4792
4793 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4794 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4795
ea8ea460
DW		 4796 start_pfn = iova >> VTD_PAGE_SHIFT;
4797 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4798
4799 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4800
4801 npages = last_pfn - start_pfn + 1;
4802
29a27719 4803 for_each_domain_iommu(iommu_id, dmar_domain) {
a1ddcbe9
JR		 4804 iommu = g_iommus[iommu_id];
4805
42e8c186
JR		 4806 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4807 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 4808 }
4809
4810 dma_free_pagelist(freelist);
fe40f1e0 4811
163cc52c
DW		 4812 if (dmar_domain->max_addr == iova + size)
4813 dmar_domain->max_addr = iova;
b146a1c9 4814
5cf0a76f 4815 return size;
38717946 4816}
38717946 4817
d14d6577 4818static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4819 dma_addr_t iova)
38717946 4820{
00a77deb 4821 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4822 struct dma_pte *pte;
5cf0a76f 4823 int level = 0;
faa3d6f5 4824 u64 phys = 0;
38717946 4825
5cf0a76f 4826 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4827 if (pte)
faa3d6f5 4828 phys = dma_pte_addr(pte);
38717946 4829
faa3d6f5 4830 return phys;
38717946 4831}
a8bcbb0d 4832
5d587b8d 4833static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4834{
dbb9fd86 4835 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4836 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4837 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4838 return irq_remapping_enabled == 1;
dbb9fd86 4839
5d587b8d 4840 return false;
dbb9fd86
SY		 4841}
4842
abdfdde2
AW		 4843static int intel_iommu_add_device(struct device *dev)
4844{
a5459cfe 4845 struct intel_iommu *iommu;
abdfdde2 4846 struct iommu_group *group;
156baca8 4847 u8 bus, devfn;
70ae6f0d 4848
a5459cfe
AW		 4849 iommu = device_to_iommu(dev, &bus, &devfn);
4850 if (!iommu)
70ae6f0d
AW		 4851 return -ENODEV;
4852
a5459cfe 4853 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4854
e17f9ff4 4855 group = iommu_group_get_for_dev(dev);
783f157b 4856
e17f9ff4
AW		 4857 if (IS_ERR(group))
4858 return PTR_ERR(group);
bcb71abe 4859
abdfdde2 4860 iommu_group_put(group);
e17f9ff4 4861 return 0;
abdfdde2 4862}
70ae6f0d 4863
abdfdde2
AW		 4864static void intel_iommu_remove_device(struct device *dev)
4865{
a5459cfe
AW		 4866 struct intel_iommu *iommu;
4867 u8 bus, devfn;
4868
4869 iommu = device_to_iommu(dev, &bus, &devfn);
4870 if (!iommu)
4871 return;
4872
abdfdde2 4873 iommu_group_remove_device(dev);
a5459cfe
AW		 4874
4875 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4876}
4877
b22f6434 4878static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4879 .capable = intel_iommu_capable,
00a77deb
JR		 4880 .domain_alloc = intel_iommu_domain_alloc,
4881 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4882 .attach_dev = intel_iommu_attach_device,
4883 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4884 .map = intel_iommu_map,
4885 .unmap = intel_iommu_unmap,
315786eb 4886 .map_sg = default_iommu_map_sg,
a8bcbb0d 4887 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4888 .add_device = intel_iommu_add_device,
4889 .remove_device = intel_iommu_remove_device,
6d1c56a9 4890 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4891};
9af88143 4892
9452618e
DV		 4893static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4894{
4895 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4896 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4897 dmar_map_gfx = 0;
4898}
4899
4900DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4901DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4902DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4903DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4904DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4905DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4906DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4907
d34d6517 4908static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4909{
4910 /*
4911 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4912 * but needs it. Same seems to hold for the desktop versions.
9af88143 4913 */
9f10e5bf 4914 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4915 rwbf_quirk = 1;
4916}
4917
4918DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4919DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4920DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4921DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4922DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4923DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4924DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4925
eecfd57f
AJ		 4926#define GGC 0x52
4927#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4928#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4929#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4930#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4931#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4932#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4933#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4934#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4935
d34d6517 4936static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4937{
4938 unsigned short ggc;
4939
eecfd57f 4940 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4941 return;
4942
eecfd57f 4943 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4944 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4945 dmar_map_gfx = 0;
6fbcfb3e
DW		 4946 } else if (dmar_map_gfx) {
4947 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4948 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4949 intel_iommu_strict = 1;
4950 }
9eecabcb
DW		 4951}
4952DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4953DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4954DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4955DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4956
e0fc7e0b
DW		 4957/* On Tylersburg chipsets, some BIOSes have been known to enable the
4958 ISOCH DMAR unit for the Azalia sound device, but not give it any
4959 TLB entries, which causes it to deadlock. Check for that. We do
4960 this in a function called from init_dmars(), instead of in a PCI
4961 quirk, because we don't want to print the obnoxious "BIOS broken"
4962 message if VT-d is actually disabled.
4963*/
4964static void __init check_tylersburg_isoch(void)
4965{
4966 struct pci_dev *pdev;
4967 uint32_t vtisochctrl;
4968
4969 /* If there's no Azalia in the system anyway, forget it. */
4970 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4971 if (!pdev)
4972 return;
4973 pci_dev_put(pdev);
4974
4975 /* System Management Registers. Might be hidden, in which case
4976 we can't do the sanity check. But that's OK, because the
4977 known-broken BIOSes _don't_ actually hide it, so far. */
4978 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4979 if (!pdev)
4980 return;
4981
4982 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4983 pci_dev_put(pdev);
4984 return;
4985 }
4986
4987 pci_dev_put(pdev);
4988
4989 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4990 if (vtisochctrl & 1)
4991 return;
4992
4993 /* Drop all bits other than the number of TLB entries */
4994 vtisochctrl &= 0x1c;
4995
4996 /* If we have the recommended number of TLB entries (16), fine. */
4997 if (vtisochctrl == 0x10)
4998 return;
4999
5000 /* Zero TLB entries? You get to ride the short bus to school. */
5001 if (!vtisochctrl) {
5002 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5003 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5004 dmi_get_system_info(DMI_BIOS_VENDOR),
5005 dmi_get_system_info(DMI_BIOS_VERSION),
5006 dmi_get_system_info(DMI_PRODUCT_VERSION));
5007 iommu_identity_mapping |= IDENTMAP_AZALIA;
5008 return;
5009 }
9f10e5bf
JR		 5010
5011 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5012 vtisochctrl);
5013}
Linux 6.x block layer and io_uring tree(s)