projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Return false instead of 0 in irq_remapping_cap()
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
29a27719
JR		 376#define for_each_domain_iommu(idx, domain) \
377 for (idx = 0; idx < g_num_of_iommus; idx++) \
378 if (domain->iommu_refcnt[idx])
379
99126f7c 380struct dmar_domain {
4c923d47 381 int nid; /* node id */
29a27719
JR		 382
383 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
384 /* Refcount of devices per iommu */
385
99126f7c 386
c0e8a6c8
JR		 387 u16 iommu_did[DMAR_UNITS_SUPPORTED];
388 /* Domain ids per IOMMU. Use u16 since
389 * domain ids are 16 bit wide according
390 * to VT-d spec, section 9.3 */
391
00a77deb 392 struct list_head devices; /* all devices' list */
99126f7c
MM		 393 struct iova_domain iovad; /* iova's that belong to this domain */
394
395 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 396 int gaw; /* max guest address width */
397
398 /* adjusted guest address width, 0 is level 2 30-bit */
399 int agaw;
400
3b5410e7 401 int flags; /* flags to find out type of domain */
8e604097
WH		 402
403 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 404 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 405 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 406 int iommu_superpage;/* Level of superpages supported:
407 0 == 4KiB (no superpages), 1 == 2MiB,
408 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
fe40f1e0 409 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 410
411 struct iommu_domain domain; /* generic domain data structure for
412 iommu core */
99126f7c
MM		 413};
414
a647dacb
MM		 415/* PCI domain-device relationship */
416struct device_domain_info {
417 struct list_head link; /* link to domain siblings */
418 struct list_head global; /* link to global list */
276dbf99 419 u8 bus; /* PCI bus number */
a647dacb 420 u8 devfn; /* PCI devfn number */
0bcb3e28 421 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 422 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 423 struct dmar_domain *domain; /* pointer to domain */
424};
425
b94e4117
JL		 426struct dmar_rmrr_unit {
427 struct list_head list; /* list of rmrr units */
428 struct acpi_dmar_header *hdr; /* ACPI header */
429 u64 base_address; /* reserved base address*/
430 u64 end_address; /* reserved end address */
832bd858 431 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 432 int devices_cnt; /* target device count */
433};
434
435struct dmar_atsr_unit {
436 struct list_head list; /* list of ATSR units */
437 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 438 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 439 int devices_cnt; /* target device count */
440 u8 include_all:1; /* include all ports */
441};
442
443static LIST_HEAD(dmar_atsr_units);
444static LIST_HEAD(dmar_rmrr_units);
445
446#define for_each_rmrr_units(rmrr) \
447 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
448
5e0d2a6f 449static void flush_unmaps_timeout(unsigned long data);
450
b707cb02 451static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 452
80b20dd8 453#define HIGH_WATER_MARK 250
454struct deferred_flush_tables {
455 int next;
456 struct iova *iova[HIGH_WATER_MARK];
457 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 458 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 459};
460
461static struct deferred_flush_tables *deferred_flush;
462
5e0d2a6f 463/* bitmap for indexing intel_iommus */
5e0d2a6f 464static int g_num_of_iommus;
465
466static DEFINE_SPINLOCK(async_umap_flush_lock);
467static LIST_HEAD(unmaps_to_do);
468
469static int timer_on;
470static long list_size;
5e0d2a6f 471
92d03cc8 472static void domain_exit(struct dmar_domain *domain);
ba395927 473static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 474static void dmar_remove_one_dev_info(struct dmar_domain *domain,
475 struct device *dev);
127c7615 476static void __dmar_remove_one_dev_info(struct device_domain_info *info);
2452d9db
JR		 477static void domain_context_clear(struct intel_iommu *iommu,
478 struct device *dev);
2a46ddf7
JL		 479static int domain_detach_iommu(struct dmar_domain *domain,
480 struct intel_iommu *iommu);
ba395927 481
d3f13810 482#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 483int dmar_disabled = 0;
484#else
485int dmar_disabled = 1;
d3f13810 486#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 487
8bc1f85c
ED		 488int intel_iommu_enabled = 0;
489EXPORT_SYMBOL_GPL(intel_iommu_enabled);
490
2d9e667e 491static int dmar_map_gfx = 1;
7d3b03ce 492static int dmar_forcedac;
5e0d2a6f 493static int intel_iommu_strict;
6dd9a7c7 494static int intel_iommu_superpage = 1;
c83b2f20
DW		 495static int intel_iommu_ecs = 1;
496
497/* We only actually use ECS when PASID support (on the new bit 40)
498 * is also advertised. Some early implementations — the ones with
499 * PASID support on bit 28 — have issues even when we *only* use
500 * extended root/context tables. */
501#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
502 ecap_pasid(iommu->ecap))
ba395927 503
c0771df8
DW		 504int intel_iommu_gfx_mapped;
505EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
506
ba395927
KA		 507#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
508static DEFINE_SPINLOCK(device_domain_lock);
509static LIST_HEAD(device_domain_list);
510
b22f6434 511static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 512
4158c2ec
JR		 513static bool translation_pre_enabled(struct intel_iommu *iommu)
514{
515 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
516}
517
091d42e4
JR		 518static void clear_translation_pre_enabled(struct intel_iommu *iommu)
519{
520 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
521}
522
4158c2ec
JR		 523static void init_translation_status(struct intel_iommu *iommu)
524{
525 u32 gsts;
526
527 gsts = readl(iommu->reg + DMAR_GSTS_REG);
528 if (gsts & DMA_GSTS_TES)
529 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
530}
531
00a77deb
JR		 532/* Convert generic 'struct iommu_domain to private struct dmar_domain */
533static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
534{
535 return container_of(dom, struct dmar_domain, domain);
536}
537
ba395927
KA		 538static int __init intel_iommu_setup(char *str)
539{
540 if (!str)
541 return -EINVAL;
542 while (*str) {
0cd5c3c8
KM		 543 if (!strncmp(str, "on", 2)) {
544 dmar_disabled = 0;
9f10e5bf 545 pr_info("IOMMU enabled\n");
0cd5c3c8 546 } else if (!strncmp(str, "off", 3)) {
ba395927 547 dmar_disabled = 1;
9f10e5bf 548 pr_info("IOMMU disabled\n");
ba395927
KA		 549 } else if (!strncmp(str, "igfx_off", 8)) {
550 dmar_map_gfx = 0;
9f10e5bf 551 pr_info("Disable GFX device mapping\n");
7d3b03ce 552 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 553 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 554 dmar_forcedac = 1;
5e0d2a6f 555 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 556 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 557 intel_iommu_strict = 1;
6dd9a7c7 558 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 559 pr_info("Disable supported super page\n");
6dd9a7c7 560 intel_iommu_superpage = 0;
c83b2f20
DW		 561 } else if (!strncmp(str, "ecs_off", 7)) {
562 printk(KERN_INFO
563 "Intel-IOMMU: disable extended context table support\n");
564 intel_iommu_ecs = 0;
ba395927
KA		 565 }
566
567 str += strcspn(str, ",");
568 while (*str == ',')
569 str++;
570 }
571 return 0;
572}
573__setup("intel_iommu=", intel_iommu_setup);
574
575static struct kmem_cache *iommu_domain_cache;
576static struct kmem_cache *iommu_devinfo_cache;
ba395927 577
9452d5bf
JR		 578static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
579{
8bf47816
JR		 580 struct dmar_domain **domains;
581 int idx = did >> 8;
582
583 domains = iommu->domains[idx];
584 if (!domains)
585 return NULL;
586
587 return domains[did & 0xff];
9452d5bf
JR		 588}
589
590static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
591 struct dmar_domain *domain)
592{
8bf47816
JR		 593 struct dmar_domain **domains;
594 int idx = did >> 8;
595
596 if (!iommu->domains[idx]) {
597 size_t size = 256 * sizeof(struct dmar_domain *);
598 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
599 }
600
601 domains = iommu->domains[idx];
602 if (WARN_ON(!domains))
603 return;
604 else
605 domains[did & 0xff] = domain;
9452d5bf
JR		 606}
607
4c923d47 608static inline void *alloc_pgtable_page(int node)
eb3fa7cb 609{
4c923d47
SS		 610 struct page *page;
611 void *vaddr = NULL;
eb3fa7cb 612
4c923d47
SS		 613 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
614 if (page)
615 vaddr = page_address(page);
eb3fa7cb 616 return vaddr;
ba395927
KA		 617}
618
619static inline void free_pgtable_page(void *vaddr)
620{
621 free_page((unsigned long)vaddr);
622}
623
624static inline void *alloc_domain_mem(void)
625{
354bb65e 626 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 627}
628
38717946 629static void free_domain_mem(void *vaddr)
ba395927
KA		 630{
631 kmem_cache_free(iommu_domain_cache, vaddr);
632}
633
634static inline void * alloc_devinfo_mem(void)
635{
354bb65e 636 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 637}
638
639static inline void free_devinfo_mem(void *vaddr)
640{
641 kmem_cache_free(iommu_devinfo_cache, vaddr);
642}
643
ab8dfe25
JL		 644static inline int domain_type_is_vm(struct dmar_domain *domain)
645{
646 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
647}
648
28ccce0d
JR		 649static inline int domain_type_is_si(struct dmar_domain *domain)
650{
651 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
652}
653
ab8dfe25
JL		 654static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
655{
656 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
657 DOMAIN_FLAG_STATIC_IDENTITY);
658}
1b573683 659
162d1b10
JL		 660static inline int domain_pfn_supported(struct dmar_domain *domain,
661 unsigned long pfn)
662{
663 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
664
665 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
666}
667
4ed0d3e6 668static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 669{
670 unsigned long sagaw;
671 int agaw = -1;
672
673 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 674 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 675 agaw >= 0; agaw--) {
676 if (test_bit(agaw, &sagaw))
677 break;
678 }
679
680 return agaw;
681}
682
4ed0d3e6
FY		 683/*
684 * Calculate max SAGAW for each iommu.
685 */
686int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
687{
688 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
689}
690
691/*
692 * calculate agaw for each iommu.
693 * "SAGAW" may be different across iommus, use a default agaw, and
694 * get a supported less agaw for iommus that don't support the default agaw.
695 */
696int iommu_calculate_agaw(struct intel_iommu *iommu)
697{
698 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
699}
700
2c2e2c38 701/* This functionin only returns single iommu in a domain */
8c11e798
WH		 702static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
703{
704 int iommu_id;
705
2c2e2c38 706 /* si_domain and vm domain should not get here. */
ab8dfe25 707 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 708 for_each_domain_iommu(iommu_id, domain)
709 break;
710
8c11e798
WH		 711 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
712 return NULL;
713
714 return g_iommus[iommu_id];
715}
716
8e604097
WH		 717static void domain_update_iommu_coherency(struct dmar_domain *domain)
718{
d0501960
DW		 719 struct dmar_drhd_unit *drhd;
720 struct intel_iommu *iommu;
2f119c78
QL		 721 bool found = false;
722 int i;
2e12bc29 723
d0501960 724 domain->iommu_coherency = 1;
8e604097 725
29a27719 726 for_each_domain_iommu(i, domain) {
2f119c78 727 found = true;
8e604097
WH		 728 if (!ecap_coherent(g_iommus[i]->ecap)) {
729 domain->iommu_coherency = 0;
730 break;
731 }
8e604097 732 }
d0501960
DW		 733 if (found)
734 return;
735
736 /* No hardware attached; use lowest common denominator */
737 rcu_read_lock();
738 for_each_active_iommu(iommu, drhd) {
739 if (!ecap_coherent(iommu->ecap)) {
740 domain->iommu_coherency = 0;
741 break;
742 }
743 }
744 rcu_read_unlock();
8e604097
WH		 745}
746
161f6934 747static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 748{
161f6934
JL		 749 struct dmar_drhd_unit *drhd;
750 struct intel_iommu *iommu;
751 int ret = 1;
58c610bd 752
161f6934
JL		 753 rcu_read_lock();
754 for_each_active_iommu(iommu, drhd) {
755 if (iommu != skip) {
756 if (!ecap_sc_support(iommu->ecap)) {
757 ret = 0;
758 break;
759 }
58c610bd 760 }
58c610bd 761 }
161f6934
JL		 762 rcu_read_unlock();
763
764 return ret;
58c610bd
SY		 765}
766
161f6934 767static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 768{
8140a95d 769 struct dmar_drhd_unit *drhd;
161f6934 770 struct intel_iommu *iommu;
8140a95d 771 int mask = 0xf;
6dd9a7c7
YS		 772
773 if (!intel_iommu_superpage) {
161f6934 774 return 0;
6dd9a7c7
YS		 775 }
776
8140a95d 777 /* set iommu_superpage to the smallest common denominator */
0e242612 778 rcu_read_lock();
8140a95d 779 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 780 if (iommu != skip) {
781 mask &= cap_super_page_val(iommu->cap);
782 if (!mask)
783 break;
6dd9a7c7
YS		 784 }
785 }
0e242612
JL		 786 rcu_read_unlock();
787
161f6934 788 return fls(mask);
6dd9a7c7
YS		 789}
790
58c610bd
SY		 791/* Some capabilities may be different across iommus */
792static void domain_update_iommu_cap(struct dmar_domain *domain)
793{
794 domain_update_iommu_coherency(domain);
161f6934
JL		 795 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
796 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 797}
798
03ecc32c
DW		 799static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
800 u8 bus, u8 devfn, int alloc)
801{
802 struct root_entry *root = &iommu->root_entry[bus];
803 struct context_entry *context;
804 u64 *entry;
805
c83b2f20 806 if (ecs_enabled(iommu)) {
03ecc32c
DW		 807 if (devfn >= 0x80) {
808 devfn -= 0x80;
809 entry = &root->hi;
810 }
811 devfn *= 2;
812 }
813 entry = &root->lo;
814 if (*entry & 1)
815 context = phys_to_virt(*entry & VTD_PAGE_MASK);
816 else {
817 unsigned long phy_addr;
818 if (!alloc)
819 return NULL;
820
821 context = alloc_pgtable_page(iommu->node);
822 if (!context)
823 return NULL;
824
825 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
826 phy_addr = virt_to_phys((void *)context);
827 *entry = phy_addr | 1;
828 __iommu_flush_cache(iommu, entry, sizeof(*entry));
829 }
830 return &context[devfn];
831}
832
4ed6a540
DW		 833static int iommu_dummy(struct device *dev)
834{
835 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
836}
837
156baca8 838static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 839{
840 struct dmar_drhd_unit *drhd = NULL;
b683b230 841 struct intel_iommu *iommu;
156baca8
DW		 842 struct device *tmp;
843 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 844 u16 segment = 0;
c7151a8d
WH		 845 int i;
846
4ed6a540
DW		 847 if (iommu_dummy(dev))
848 return NULL;
849
156baca8
DW		 850 if (dev_is_pci(dev)) {
851 pdev = to_pci_dev(dev);
852 segment = pci_domain_nr(pdev->bus);
ca5b74d2 853 } else if (has_acpi_companion(dev))
156baca8
DW		 854 dev = &ACPI_COMPANION(dev)->dev;
855
0e242612 856 rcu_read_lock();
b683b230 857 for_each_active_iommu(iommu, drhd) {
156baca8 858 if (pdev && segment != drhd->segment)
276dbf99 859 continue;
c7151a8d 860
b683b230 861 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 862 drhd->devices_cnt, i, tmp) {
863 if (tmp == dev) {
864 *bus = drhd->devices[i].bus;
865 *devfn = drhd->devices[i].devfn;
b683b230 866 goto out;
156baca8
DW		 867 }
868
869 if (!pdev || !dev_is_pci(tmp))
870 continue;
871
872 ptmp = to_pci_dev(tmp);
873 if (ptmp->subordinate &&
874 ptmp->subordinate->number <= pdev->bus->number &&
875 ptmp->subordinate->busn_res.end >= pdev->bus->number)
876 goto got_pdev;
924b6231 877 }
c7151a8d 878
156baca8
DW		 879 if (pdev && drhd->include_all) {
880 got_pdev:
881 *bus = pdev->bus->number;
882 *devfn = pdev->devfn;
b683b230 883 goto out;
156baca8 884 }
c7151a8d 885 }
b683b230 886 iommu = NULL;
156baca8 887 out:
0e242612 888 rcu_read_unlock();
c7151a8d 889
b683b230 890 return iommu;
c7151a8d
WH		 891}
892
5331fe6f
WH		 893static void domain_flush_cache(struct dmar_domain *domain,
894 void *addr, int size)
895{
896 if (!domain->iommu_coherency)
897 clflush_cache_range(addr, size);
898}
899
ba395927
KA		 900static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
901{
ba395927 902 struct context_entry *context;
03ecc32c 903 int ret = 0;
ba395927
KA		 904 unsigned long flags;
905
906 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 907 context = iommu_context_addr(iommu, bus, devfn, 0);
908 if (context)
909 ret = context_present(context);
ba395927
KA		 910 spin_unlock_irqrestore(&iommu->lock, flags);
911 return ret;
912}
913
914static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
915{
ba395927
KA		 916 struct context_entry *context;
917 unsigned long flags;
918
919 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 920 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 921 if (context) {
03ecc32c
DW		 922 context_clear_entry(context);
923 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 924 }
925 spin_unlock_irqrestore(&iommu->lock, flags);
926}
927
928static void free_context_table(struct intel_iommu *iommu)
929{
ba395927
KA		 930 int i;
931 unsigned long flags;
932 struct context_entry *context;
933
934 spin_lock_irqsave(&iommu->lock, flags);
935 if (!iommu->root_entry) {
936 goto out;
937 }
938 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 939 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 940 if (context)
941 free_pgtable_page(context);
03ecc32c 942
c83b2f20 943 if (!ecs_enabled(iommu))
03ecc32c
DW		 944 continue;
945
946 context = iommu_context_addr(iommu, i, 0x80, 0);
947 if (context)
948 free_pgtable_page(context);
949
ba395927
KA		 950 }
951 free_pgtable_page(iommu->root_entry);
952 iommu->root_entry = NULL;
953out:
954 spin_unlock_irqrestore(&iommu->lock, flags);
955}
956
b026fd28 957static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 958 unsigned long pfn, int *target_level)
ba395927 959{
ba395927
KA		 960 struct dma_pte *parent, *pte = NULL;
961 int level = agaw_to_level(domain->agaw);
4399c8bf 962 int offset;
ba395927
KA		 963
964 BUG_ON(!domain->pgd);
f9423606 965
162d1b10 966 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 967 /* Address beyond IOMMU's addressing capabilities. */
968 return NULL;
969
ba395927
KA		 970 parent = domain->pgd;
971
5cf0a76f 972 while (1) {
ba395927
KA		 973 void *tmp_page;
974
b026fd28 975 offset = pfn_level_offset(pfn, level);
ba395927 976 pte = &parent[offset];
5cf0a76f 977 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 978 break;
5cf0a76f 979 if (level == *target_level)
ba395927
KA		 980 break;
981
19c239ce 982 if (!dma_pte_present(pte)) {
c85994e4
DW		 983 uint64_t pteval;
984
4c923d47 985 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 986
206a73c1 987 if (!tmp_page)
ba395927 988 return NULL;
206a73c1 989
c85994e4 990 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 991 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 992 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 993 /* Someone else set it while we were thinking; use theirs. */
994 free_pgtable_page(tmp_page);
effad4b5 995 else
c85994e4 996 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 997 }
5cf0a76f
DW		 998 if (level == 1)
999 break;
1000
19c239ce 1001 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1002 level--;
1003 }
1004
5cf0a76f
DW		 1005 if (!*target_level)
1006 *target_level = level;
1007
ba395927
KA		 1008 return pte;
1009}
1010
6dd9a7c7 1011
ba395927 1012/* return address's pte at specific level */
90dcfb5e
DW		 1013static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1014 unsigned long pfn,
6dd9a7c7 1015 int level, int *large_page)
ba395927
KA		 1016{
1017 struct dma_pte *parent, *pte = NULL;
1018 int total = agaw_to_level(domain->agaw);
1019 int offset;
1020
1021 parent = domain->pgd;
1022 while (level <= total) {
90dcfb5e 1023 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1024 pte = &parent[offset];
1025 if (level == total)
1026 return pte;
1027
6dd9a7c7
YS		 1028 if (!dma_pte_present(pte)) {
1029 *large_page = total;
ba395927 1030 break;
6dd9a7c7
YS		 1031 }
1032
e16922af 1033 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1034 *large_page = total;
1035 return pte;
1036 }
1037
19c239ce 1038 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1039 total--;
1040 }
1041 return NULL;
1042}
1043
ba395927 1044/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1045static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1046 unsigned long start_pfn,
1047 unsigned long last_pfn)
ba395927 1048{
6dd9a7c7 1049 unsigned int large_page = 1;
310a5ab9 1050 struct dma_pte *first_pte, *pte;
66eae846 1051
162d1b10
JL		 1052 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1053 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1054 BUG_ON(start_pfn > last_pfn);
ba395927 1055
04b18e65 1056 /* we don't need lock here; nobody else touches the iova range */
59c36286 1057 do {
6dd9a7c7
YS		 1058 large_page = 1;
1059 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1060 if (!pte) {
6dd9a7c7 1061 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1062 continue;
1063 }
6dd9a7c7 1064 do {
310a5ab9 1065 dma_clear_pte(pte);
6dd9a7c7 1066 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1067 pte++;
75e6bf96
DW		 1068 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1069
310a5ab9
DW		 1070 domain_flush_cache(domain, first_pte,
1071 (void *)pte - (void *)first_pte);
59c36286
DW		 1072
1073 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1074}
1075
3269ee0b
AW		 1076static void dma_pte_free_level(struct dmar_domain *domain, int level,
1077 struct dma_pte *pte, unsigned long pfn,
1078 unsigned long start_pfn, unsigned long last_pfn)
1079{
1080 pfn = max(start_pfn, pfn);
1081 pte = &pte[pfn_level_offset(pfn, level)];
1082
1083 do {
1084 unsigned long level_pfn;
1085 struct dma_pte *level_pte;
1086
1087 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1088 goto next;
1089
1090 level_pfn = pfn & level_mask(level - 1);
1091 level_pte = phys_to_virt(dma_pte_addr(pte));
1092
1093 if (level > 2)
1094 dma_pte_free_level(domain, level - 1, level_pte,
1095 level_pfn, start_pfn, last_pfn);
1096
1097 /* If range covers entire pagetable, free it */
1098 if (!(start_pfn > level_pfn ||
08336fd2 1099 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1100 dma_clear_pte(pte);
1101 domain_flush_cache(domain, pte, sizeof(*pte));
1102 free_pgtable_page(level_pte);
1103 }
1104next:
1105 pfn += level_size(level);
1106 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1107}
1108
ba395927
KA		 1109/* free page table pages. last level pte should already be cleared */
1110static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1111 unsigned long start_pfn,
1112 unsigned long last_pfn)
ba395927 1113{
162d1b10
JL		 1114 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1115 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1116 BUG_ON(start_pfn > last_pfn);
ba395927 1117
d41a4adb
JL		 1118 dma_pte_clear_range(domain, start_pfn, last_pfn);
1119
f3a0a52f 1120 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1121 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1122 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1123
ba395927 1124 /* free pgd */
d794dc9b 1125 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1126 free_pgtable_page(domain->pgd);
1127 domain->pgd = NULL;
1128 }
1129}
1130
ea8ea460
DW		 1131/* When a page at a given level is being unlinked from its parent, we don't
1132 need to *modify* it at all. All we need to do is make a list of all the
1133 pages which can be freed just as soon as we've flushed the IOTLB and we
1134 know the hardware page-walk will no longer touch them.
1135 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1136 be freed. */
1137static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1138 int level, struct dma_pte *pte,
1139 struct page *freelist)
1140{
1141 struct page *pg;
1142
1143 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1144 pg->freelist = freelist;
1145 freelist = pg;
1146
1147 if (level == 1)
1148 return freelist;
1149
adeb2590
JL		 1150 pte = page_address(pg);
1151 do {
ea8ea460
DW		 1152 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1153 freelist = dma_pte_list_pagetables(domain, level - 1,
1154 pte, freelist);
adeb2590
JL		 1155 pte++;
1156 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1157
1158 return freelist;
1159}
1160
1161static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1162 struct dma_pte *pte, unsigned long pfn,
1163 unsigned long start_pfn,
1164 unsigned long last_pfn,
1165 struct page *freelist)
1166{
1167 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1168
1169 pfn = max(start_pfn, pfn);
1170 pte = &pte[pfn_level_offset(pfn, level)];
1171
1172 do {
1173 unsigned long level_pfn;
1174
1175 if (!dma_pte_present(pte))
1176 goto next;
1177
1178 level_pfn = pfn & level_mask(level);
1179
1180 /* If range covers entire pagetable, free it */
1181 if (start_pfn <= level_pfn &&
1182 last_pfn >= level_pfn + level_size(level) - 1) {
1183 /* These suborbinate page tables are going away entirely. Don't
1184 bother to clear them; we're just going to *free* them. */
1185 if (level > 1 && !dma_pte_superpage(pte))
1186 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1187
1188 dma_clear_pte(pte);
1189 if (!first_pte)
1190 first_pte = pte;
1191 last_pte = pte;
1192 } else if (level > 1) {
1193 /* Recurse down into a level that isn't *entirely* obsolete */
1194 freelist = dma_pte_clear_level(domain, level - 1,
1195 phys_to_virt(dma_pte_addr(pte)),
1196 level_pfn, start_pfn, last_pfn,
1197 freelist);
1198 }
1199next:
1200 pfn += level_size(level);
1201 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1202
1203 if (first_pte)
1204 domain_flush_cache(domain, first_pte,
1205 (void *)++last_pte - (void *)first_pte);
1206
1207 return freelist;
1208}
1209
1210/* We can't just free the pages because the IOMMU may still be walking
1211 the page tables, and may have cached the intermediate levels. The
1212 pages can only be freed after the IOTLB flush has been done. */
1213struct page *domain_unmap(struct dmar_domain *domain,
1214 unsigned long start_pfn,
1215 unsigned long last_pfn)
1216{
ea8ea460
DW		 1217 struct page *freelist = NULL;
1218
162d1b10
JL		 1219 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1220 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1221 BUG_ON(start_pfn > last_pfn);
1222
1223 /* we don't need lock here; nobody else touches the iova range */
1224 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1225 domain->pgd, 0, start_pfn, last_pfn, NULL);
1226
1227 /* free pgd */
1228 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1229 struct page *pgd_page = virt_to_page(domain->pgd);
1230 pgd_page->freelist = freelist;
1231 freelist = pgd_page;
1232
1233 domain->pgd = NULL;
1234 }
1235
1236 return freelist;
1237}
1238
1239void dma_free_pagelist(struct page *freelist)
1240{
1241 struct page *pg;
1242
1243 while ((pg = freelist)) {
1244 freelist = pg->freelist;
1245 free_pgtable_page(page_address(pg));
1246 }
1247}
1248
ba395927
KA		 1249/* iommu handling */
1250static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1251{
1252 struct root_entry *root;
1253 unsigned long flags;
1254
4c923d47 1255 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1256 if (!root) {
9f10e5bf 1257 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1258 iommu->name);
ba395927 1259 return -ENOMEM;
ffebeb46 1260 }
ba395927 1261
5b6985ce 1262 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1263
1264 spin_lock_irqsave(&iommu->lock, flags);
1265 iommu->root_entry = root;
1266 spin_unlock_irqrestore(&iommu->lock, flags);
1267
1268 return 0;
1269}
1270
ba395927
KA		 1271static void iommu_set_root_entry(struct intel_iommu *iommu)
1272{
03ecc32c 1273 u64 addr;
c416daa9 1274 u32 sts;
ba395927
KA		 1275 unsigned long flag;
1276
03ecc32c 1277 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1278 if (ecs_enabled(iommu))
03ecc32c 1279 addr |= DMA_RTADDR_RTT;
ba395927 1280
1f5b3c3f 1281 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1282 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1283
c416daa9 1284 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1285
1286 /* Make sure hardware complete it */
1287 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1288 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1289
1f5b3c3f 1290 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1291}
1292
1293static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1294{
1295 u32 val;
1296 unsigned long flag;
1297
9af88143 1298 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1299 return;
ba395927 1300
1f5b3c3f 1301 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1302 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1303
1304 /* Make sure hardware complete it */
1305 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1306 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1307
1f5b3c3f 1308 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1309}
1310
1311/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1312static void __iommu_flush_context(struct intel_iommu *iommu,
1313 u16 did, u16 source_id, u8 function_mask,
1314 u64 type)
ba395927
KA		 1315{
1316 u64 val = 0;
1317 unsigned long flag;
1318
ba395927
KA		 1319 switch (type) {
1320 case DMA_CCMD_GLOBAL_INVL:
1321 val = DMA_CCMD_GLOBAL_INVL;
1322 break;
1323 case DMA_CCMD_DOMAIN_INVL:
1324 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1325 break;
1326 case DMA_CCMD_DEVICE_INVL:
1327 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1328 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1329 break;
1330 default:
1331 BUG();
1332 }
1333 val |= DMA_CCMD_ICC;
1334
1f5b3c3f 1335 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1336 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1337
1338 /* Make sure hardware complete it */
1339 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1340 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1341
1f5b3c3f 1342 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1343}
1344
ba395927 1345/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1346static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1347 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1348{
1349 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1350 u64 val = 0, val_iva = 0;
1351 unsigned long flag;
1352
ba395927
KA		 1353 switch (type) {
1354 case DMA_TLB_GLOBAL_FLUSH:
1355 /* global flush doesn't need set IVA_REG */
1356 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1357 break;
1358 case DMA_TLB_DSI_FLUSH:
1359 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1360 break;
1361 case DMA_TLB_PSI_FLUSH:
1362 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1363 /* IH bit is passed in as part of address */
ba395927
KA		 1364 val_iva = size_order | addr;
1365 break;
1366 default:
1367 BUG();
1368 }
1369 /* Note: set drain read/write */
1370#if 0
1371 /*
1372 * This is probably to be super secure.. Looks like we can
1373 * ignore it without any impact.
1374 */
1375 if (cap_read_drain(iommu->cap))
1376 val |= DMA_TLB_READ_DRAIN;
1377#endif
1378 if (cap_write_drain(iommu->cap))
1379 val |= DMA_TLB_WRITE_DRAIN;
1380
1f5b3c3f 1381 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1382 /* Note: Only uses first TLB reg currently */
1383 if (val_iva)
1384 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1385 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1386
1387 /* Make sure hardware complete it */
1388 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1389 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1390
1f5b3c3f 1391 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1392
1393 /* check IOTLB invalidation granularity */
1394 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1395 pr_err("Flush IOTLB failed\n");
ba395927 1396 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1397 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1398 (unsigned long long)DMA_TLB_IIRG(type),
1399 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1400}
1401
64ae892b
DW		 1402static struct device_domain_info *
1403iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1404 u8 bus, u8 devfn)
93a23a72 1405{
2f119c78 1406 bool found = false;
93a23a72 1407 struct device_domain_info *info;
0bcb3e28 1408 struct pci_dev *pdev;
93a23a72 1409
55d94043
JR		 1410 assert_spin_locked(&device_domain_lock);
1411
93a23a72
YZ		 1412 if (!ecap_dev_iotlb_support(iommu->ecap))
1413 return NULL;
1414
1415 if (!iommu->qi)
1416 return NULL;
1417
93a23a72 1418 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1419 if (info->iommu == iommu && info->bus == bus &&
1420 info->devfn == devfn) {
2f119c78 1421 found = true;
93a23a72
YZ		 1422 break;
1423 }
93a23a72 1424
0bcb3e28 1425 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1426 return NULL;
1427
0bcb3e28
DW		 1428 pdev = to_pci_dev(info->dev);
1429
1430 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1431 return NULL;
1432
0bcb3e28 1433 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1434 return NULL;
1435
93a23a72
YZ		 1436 return info;
1437}
1438
1439static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1440{
0bcb3e28 1441 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1442 return;
1443
0bcb3e28 1444 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1445}
1446
1447static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1448{
0bcb3e28
DW		 1449 if (!info->dev || !dev_is_pci(info->dev) ||
1450 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1451 return;
1452
0bcb3e28 1453 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1454}
1455
1456static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1457 u64 addr, unsigned mask)
1458{
1459 u16 sid, qdep;
1460 unsigned long flags;
1461 struct device_domain_info *info;
1462
1463 spin_lock_irqsave(&device_domain_lock, flags);
1464 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1465 struct pci_dev *pdev;
1466 if (!info->dev || !dev_is_pci(info->dev))
1467 continue;
1468
1469 pdev = to_pci_dev(info->dev);
1470 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1471 continue;
1472
1473 sid = info->bus << 8 | info->devfn;
0bcb3e28 1474 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1475 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1476 }
1477 spin_unlock_irqrestore(&device_domain_lock, flags);
1478}
1479
a1ddcbe9
JR		 1480static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1481 struct dmar_domain *domain,
1482 unsigned long pfn, unsigned int pages,
1483 int ih, int map)
ba395927 1484{
9dd2fe89 1485 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1486 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1487 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1488
ba395927
KA		 1489 BUG_ON(pages == 0);
1490
ea8ea460
DW		 1491 if (ih)
1492 ih = 1 << 6;
ba395927 1493 /*
9dd2fe89
YZ		 1494 * Fallback to domain selective flush if no PSI support or the size is
1495 * too big.
ba395927
KA		 1496 * PSI requires page size to be 2 ^ x, and the base address is naturally
1497 * aligned to the size
1498 */
9dd2fe89
YZ		 1499 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1500 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1501 DMA_TLB_DSI_FLUSH);
9dd2fe89 1502 else
ea8ea460 1503 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1504 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1505
1506 /*
82653633
NA		 1507 * In caching mode, changes of pages from non-present to present require
1508 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1509 */
82653633 1510 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1511 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1512 addr, mask);
ba395927
KA		 1513}
1514
f8bab735 1515static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1516{
1517 u32 pmen;
1518 unsigned long flags;
1519
1f5b3c3f 1520 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1521 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1522 pmen &= ~DMA_PMEN_EPM;
1523 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1524
1525 /* wait for the protected region status bit to clear */
1526 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1527 readl, !(pmen & DMA_PMEN_PRS), pmen);
1528
1f5b3c3f 1529 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1530}
1531
2a41ccee 1532static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1533{
1534 u32 sts;
1535 unsigned long flags;
1536
1f5b3c3f 1537 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1538 iommu->gcmd |= DMA_GCMD_TE;
1539 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1540
1541 /* Make sure hardware complete it */
1542 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1543 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1544
1f5b3c3f 1545 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1546}
1547
2a41ccee 1548static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1549{
1550 u32 sts;
1551 unsigned long flag;
1552
1f5b3c3f 1553 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1554 iommu->gcmd &= ~DMA_GCMD_TE;
1555 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1556
1557 /* Make sure hardware complete it */
1558 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1559 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1560
1f5b3c3f 1561 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1562}
1563
3460a6d9 1564
ba395927
KA		 1565static int iommu_init_domains(struct intel_iommu *iommu)
1566{
8bf47816
JR		 1567 u32 ndomains, nlongs;
1568 size_t size;
ba395927
KA		 1569
1570 ndomains = cap_ndoms(iommu->cap);
8bf47816 1571 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1572 iommu->name, ndomains);
ba395927
KA		 1573 nlongs = BITS_TO_LONGS(ndomains);
1574
94a91b50
DD		 1575 spin_lock_init(&iommu->lock);
1576
ba395927
KA		 1577 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1578 if (!iommu->domain_ids) {
9f10e5bf
JR		 1579 pr_err("%s: Allocating domain id array failed\n",
1580 iommu->name);
ba395927
KA		 1581 return -ENOMEM;
1582 }
8bf47816
JR		 1583
1584 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1585 iommu->domains = kzalloc(size, GFP_KERNEL);
1586
1587 if (iommu->domains) {
1588 size = 256 * sizeof(struct dmar_domain *);
1589 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1590 }
1591
1592 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1593 pr_err("%s: Allocating domain array failed\n",
1594 iommu->name);
852bdb04 1595 kfree(iommu->domain_ids);
8bf47816 1596 kfree(iommu->domains);
852bdb04 1597 iommu->domain_ids = NULL;
8bf47816 1598 iommu->domains = NULL;
ba395927
KA		 1599 return -ENOMEM;
1600 }
1601
8bf47816
JR		 1602
1603
ba395927 1604 /*
c0e8a6c8
JR		 1605 * If Caching mode is set, then invalid translations are tagged
1606 * with domain-id 0, hence we need to pre-allocate it. We also
1607 * use domain-id 0 as a marker for non-allocated domain-id, so
1608 * make sure it is not used for a real domain.
ba395927 1609 */
c0e8a6c8
JR		 1610 set_bit(0, iommu->domain_ids);
1611
ba395927
KA		 1612 return 0;
1613}
ba395927 1614
ffebeb46 1615static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1616{
29a27719 1617 struct device_domain_info *info, *tmp;
55d94043 1618 unsigned long flags;
ba395927 1619
29a27719
JR		 1620 if (!iommu->domains || !iommu->domain_ids)
1621 return;
a4eaa86c 1622
55d94043 1623 spin_lock_irqsave(&device_domain_lock, flags);
29a27719
JR		 1624 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1625 struct dmar_domain *domain;
1626
1627 if (info->iommu != iommu)
1628 continue;
1629
1630 if (!info->dev || !info->domain)
1631 continue;
1632
1633 domain = info->domain;
1634
e6de0f8d 1635 dmar_remove_one_dev_info(domain, info->dev);
29a27719
JR		 1636
1637 if (!domain_type_is_vm_or_si(domain))
1638 domain_exit(domain);
ba395927 1639 }
55d94043 1640 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1641
1642 if (iommu->gcmd & DMA_GCMD_TE)
1643 iommu_disable_translation(iommu);
ffebeb46 1644}
ba395927 1645
ffebeb46
JL		 1646static void free_dmar_iommu(struct intel_iommu *iommu)
1647{
1648 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1649 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1650 int i;
1651
1652 for (i = 0; i < elems; i++)
1653 kfree(iommu->domains[i]);
ffebeb46
JL		 1654 kfree(iommu->domains);
1655 kfree(iommu->domain_ids);
1656 iommu->domains = NULL;
1657 iommu->domain_ids = NULL;
1658 }
ba395927 1659
d9630fe9
WH		 1660 g_iommus[iommu->seq_id] = NULL;
1661
ba395927
KA		 1662 /* free context mapping */
1663 free_context_table(iommu);
ba395927
KA		 1664}
1665
ab8dfe25 1666static struct dmar_domain *alloc_domain(int flags)
ba395927 1667{
ba395927 1668 struct dmar_domain *domain;
ba395927
KA		 1669
1670 domain = alloc_domain_mem();
1671 if (!domain)
1672 return NULL;
1673
ab8dfe25 1674 memset(domain, 0, sizeof(*domain));
4c923d47 1675 domain->nid = -1;
ab8dfe25 1676 domain->flags = flags;
92d03cc8 1677 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1678
1679 return domain;
1680}
1681
d160aca5
JR		 1682/* Must be called with iommu->lock */
1683static int domain_attach_iommu(struct dmar_domain *domain,
fb170fb4
JL		 1684 struct intel_iommu *iommu)
1685{
d160aca5 1686 unsigned long ndomains;
55d94043 1687 int num;
c0e8a6c8 1688
55d94043 1689 assert_spin_locked(&device_domain_lock);
d160aca5 1690 assert_spin_locked(&iommu->lock);
fb170fb4 1691
29a27719
JR		 1692 domain->iommu_refcnt[iommu->seq_id] += 1;
1693 domain->iommu_count += 1;
1694 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
d160aca5
JR		 1695 ndomains = cap_ndoms(iommu->cap);
1696 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1697
1698 if (num >= ndomains) {
1699 pr_err("%s: No free domain ids\n", iommu->name);
1700 domain->iommu_refcnt[iommu->seq_id] -= 1;
1701 domain->iommu_count -= 1;
55d94043 1702 return -ENOSPC;
d160aca5
JR		 1703 }
1704
1705 set_bit(num, iommu->domain_ids);
1706 set_iommu_domain(iommu, num, domain);
1707
1708 domain->iommu_did[iommu->seq_id] = num;
1709 domain->nid = iommu->node;
1710
fb170fb4
JL		 1711 domain_update_iommu_cap(domain);
1712 }
d160aca5 1713
55d94043 1714 return 0;
fb170fb4
JL		 1715}
1716
1717static int domain_detach_iommu(struct dmar_domain *domain,
1718 struct intel_iommu *iommu)
1719{
d160aca5 1720 int num, count = INT_MAX;
d160aca5 1721
55d94043 1722 assert_spin_locked(&device_domain_lock);
d160aca5 1723 assert_spin_locked(&iommu->lock);
fb170fb4 1724
29a27719
JR		 1725 domain->iommu_refcnt[iommu->seq_id] -= 1;
1726 count = --domain->iommu_count;
1727 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
d160aca5
JR		 1728 num = domain->iommu_did[iommu->seq_id];
1729 clear_bit(num, iommu->domain_ids);
1730 set_iommu_domain(iommu, num, NULL);
1731
fb170fb4 1732 domain_update_iommu_cap(domain);
c0e8a6c8 1733 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4 1734 }
fb170fb4
JL		 1735
1736 return count;
1737}
1738
ba395927 1739static struct iova_domain reserved_iova_list;
8a443df4 1740static struct lock_class_key reserved_rbtree_key;
ba395927 1741
51a63e67 1742static int dmar_init_reserved_ranges(void)
ba395927
KA		 1743{
1744 struct pci_dev *pdev = NULL;
1745 struct iova *iova;
1746 int i;
ba395927 1747
0fb5fe87
RM		 1748 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1749 DMA_32BIT_PFN);
ba395927 1750
8a443df4
MG		 1751 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1752 &reserved_rbtree_key);
1753
ba395927
KA		 1754 /* IOAPIC ranges shouldn't be accessed by DMA */
1755 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1756 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1757 if (!iova) {
9f10e5bf 1758 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1759 return -ENODEV;
1760 }
ba395927
KA		 1761
1762 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1763 for_each_pci_dev(pdev) {
1764 struct resource *r;
1765
1766 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1767 r = &pdev->resource[i];
1768 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1769 continue;
1a4a4551
DW		 1770 iova = reserve_iova(&reserved_iova_list,
1771 IOVA_PFN(r->start),
1772 IOVA_PFN(r->end));
51a63e67 1773 if (!iova) {
9f10e5bf 1774 pr_err("Reserve iova failed\n");
51a63e67
JC		 1775 return -ENODEV;
1776 }
ba395927
KA		 1777 }
1778 }
51a63e67 1779 return 0;
ba395927
KA		 1780}
1781
1782static void domain_reserve_special_ranges(struct dmar_domain *domain)
1783{
1784 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1785}
1786
1787static inline int guestwidth_to_adjustwidth(int gaw)
1788{
1789 int agaw;
1790 int r = (gaw - 12) % 9;
1791
1792 if (r == 0)
1793 agaw = gaw;
1794 else
1795 agaw = gaw + 9 - r;
1796 if (agaw > 64)
1797 agaw = 64;
1798 return agaw;
1799}
1800
dc534b25
JR		 1801static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1802 int guest_width)
ba395927 1803{
ba395927
KA		 1804 int adjust_width, agaw;
1805 unsigned long sagaw;
1806
0fb5fe87
RM		 1807 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1808 DMA_32BIT_PFN);
ba395927
KA		 1809 domain_reserve_special_ranges(domain);
1810
1811 /* calculate AGAW */
ba395927
KA		 1812 if (guest_width > cap_mgaw(iommu->cap))
1813 guest_width = cap_mgaw(iommu->cap);
1814 domain->gaw = guest_width;
1815 adjust_width = guestwidth_to_adjustwidth(guest_width);
1816 agaw = width_to_agaw(adjust_width);
1817 sagaw = cap_sagaw(iommu->cap);
1818 if (!test_bit(agaw, &sagaw)) {
1819 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1820 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1821 agaw = find_next_bit(&sagaw, 5, agaw);
1822 if (agaw >= 5)
1823 return -ENODEV;
1824 }
1825 domain->agaw = agaw;
ba395927 1826
8e604097
WH		 1827 if (ecap_coherent(iommu->ecap))
1828 domain->iommu_coherency = 1;
1829 else
1830 domain->iommu_coherency = 0;
1831
58c610bd
SY		 1832 if (ecap_sc_support(iommu->ecap))
1833 domain->iommu_snooping = 1;
1834 else
1835 domain->iommu_snooping = 0;
1836
214e39aa
DW		 1837 if (intel_iommu_superpage)
1838 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1839 else
1840 domain->iommu_superpage = 0;
1841
4c923d47 1842 domain->nid = iommu->node;
c7151a8d 1843
ba395927 1844 /* always allocate the top pgd */
4c923d47 1845 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1846 if (!domain->pgd)
1847 return -ENOMEM;
5b6985ce 1848 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1849 return 0;
1850}
1851
1852static void domain_exit(struct dmar_domain *domain)
1853{
ea8ea460 1854 struct page *freelist = NULL;
ba395927
KA		 1855
1856 /* Domain 0 is reserved, so dont process it */
1857 if (!domain)
1858 return;
1859
7b668357
AW		 1860 /* Flush any lazy unmaps that may reference this domain */
1861 if (!intel_iommu_strict)
1862 flush_unmaps_timeout(0);
1863
d160aca5
JR		 1864 /* Remove associated devices and clear attached or cached domains */
1865 rcu_read_lock();
ba395927 1866 domain_remove_dev_info(domain);
d160aca5 1867 rcu_read_unlock();
92d03cc8 1868
ba395927
KA		 1869 /* destroy iovas */
1870 put_iova_domain(&domain->iovad);
ba395927 1871
ea8ea460 1872 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1873
ea8ea460
DW		 1874 dma_free_pagelist(freelist);
1875
ba395927
KA		 1876 free_domain_mem(domain);
1877}
1878
64ae892b
DW		 1879static int domain_context_mapping_one(struct dmar_domain *domain,
1880 struct intel_iommu *iommu,
28ccce0d 1881 u8 bus, u8 devfn)
ba395927 1882{
c6c2cebd 1883 u16 did = domain->iommu_did[iommu->seq_id];
28ccce0d
JR		 1884 int translation = CONTEXT_TT_MULTI_LEVEL;
1885 struct device_domain_info *info = NULL;
ba395927 1886 struct context_entry *context;
ba395927 1887 unsigned long flags;
ea6606b0 1888 struct dma_pte *pgd;
55d94043 1889 int ret, agaw;
28ccce0d 1890
c6c2cebd
JR		 1891 WARN_ON(did == 0);
1892
28ccce0d
JR		 1893 if (hw_pass_through && domain_type_is_si(domain))
1894 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1895
1896 pr_debug("Set context mapping for %02x:%02x.%d\n",
1897 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1898
ba395927 1899 BUG_ON(!domain->pgd);
5331fe6f 1900
55d94043
JR		 1901 spin_lock_irqsave(&device_domain_lock, flags);
1902 spin_lock(&iommu->lock);
1903
1904 ret = -ENOMEM;
03ecc32c 1905 context = iommu_context_addr(iommu, bus, devfn, 1);
ba395927 1906 if (!context)
55d94043
JR		 1907 goto out_unlock;
1908
1909 ret = 0;
1910 if (context_present(context))
1911 goto out_unlock;
ba395927 1912
ea6606b0
WH		 1913 pgd = domain->pgd;
1914
de24e553 1915 context_clear_entry(context);
c6c2cebd 1916 context_set_domain_id(context, did);
4ed0d3e6 1917
de24e553
JR		 1918 /*
1919 * Skip top levels of page tables for iommu which has less agaw
1920 * than default. Unnecessary for PT mode.
1921 */
93a23a72 1922 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553 1923 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
55d94043 1924 ret = -ENOMEM;
de24e553 1925 pgd = phys_to_virt(dma_pte_addr(pgd));
55d94043
JR		 1926 if (!dma_pte_present(pgd))
1927 goto out_unlock;
de24e553
JR		 1928 }
1929
64ae892b 1930 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1931 translation = info ? CONTEXT_TT_DEV_IOTLB :
1932 CONTEXT_TT_MULTI_LEVEL;
de24e553 1933
93a23a72
YZ		 1934 context_set_address_root(context, virt_to_phys(pgd));
1935 context_set_address_width(context, iommu->agaw);
de24e553
JR		 1936 } else {
1937 /*
1938 * In pass through mode, AW must be programmed to
1939 * indicate the largest AGAW value supported by
1940 * hardware. And ASR is ignored by hardware.
1941 */
1942 context_set_address_width(context, iommu->msagaw);
93a23a72 1943 }
4ed0d3e6
FY		 1944
1945 context_set_translation_type(context, translation);
c07e7d21
MM		 1946 context_set_fault_enable(context);
1947 context_set_present(context);
5331fe6f 1948 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1949
4c25a2c1
DW		 1950 /*
1951 * It's a non-present to present mapping. If hardware doesn't cache
1952 * non-present entry we only need to flush the write-buffer. If the
1953 * _does_ cache non-present entries, then it does so in the special
1954 * domain #0, which we have to flush:
1955 */
1956 if (cap_caching_mode(iommu->cap)) {
1957 iommu->flush.flush_context(iommu, 0,
1958 (((u16)bus) << 8) | devfn,
1959 DMA_CCMD_MASK_NOBIT,
1960 DMA_CCMD_DEVICE_INVL);
c6c2cebd 1961 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1962 } else {
ba395927 1963 iommu_flush_write_buffer(iommu);
4c25a2c1 1964 }
93a23a72 1965 iommu_enable_dev_iotlb(info);
55d94043
JR		 1966
1967 ret = 0;
1968
1969out_unlock:
1970 spin_unlock(&iommu->lock);
1971 spin_unlock_irqrestore(&device_domain_lock, flags);
c7151a8d 1972
ba395927
KA		 1973 return 0;
1974}
1975
579305f7
AW		 1976struct domain_context_mapping_data {
1977 struct dmar_domain *domain;
1978 struct intel_iommu *iommu;
579305f7
AW		 1979};
1980
1981static int domain_context_mapping_cb(struct pci_dev *pdev,
1982 u16 alias, void *opaque)
1983{
1984 struct domain_context_mapping_data *data = opaque;
1985
1986 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 1987 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 1988}
1989
ba395927 1990static int
28ccce0d 1991domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 1992{
64ae892b 1993 struct intel_iommu *iommu;
156baca8 1994 u8 bus, devfn;
579305f7 1995 struct domain_context_mapping_data data;
64ae892b 1996
e1f167f3 1997 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1998 if (!iommu)
1999 return -ENODEV;
ba395927 2000
579305f7 2001 if (!dev_is_pci(dev))
28ccce0d 2002 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2003
2004 data.domain = domain;
2005 data.iommu = iommu;
579305f7
AW		 2006
2007 return pci_for_each_dma_alias(to_pci_dev(dev),
2008 &domain_context_mapping_cb, &data);
2009}
2010
2011static int domain_context_mapped_cb(struct pci_dev *pdev,
2012 u16 alias, void *opaque)
2013{
2014 struct intel_iommu *iommu = opaque;
2015
2016 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2017}
2018
e1f167f3 2019static int domain_context_mapped(struct device *dev)
ba395927 2020{
5331fe6f 2021 struct intel_iommu *iommu;
156baca8 2022 u8 bus, devfn;
5331fe6f 2023
e1f167f3 2024 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2025 if (!iommu)
2026 return -ENODEV;
ba395927 2027
579305f7
AW		 2028 if (!dev_is_pci(dev))
2029 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2030
579305f7
AW		 2031 return !pci_for_each_dma_alias(to_pci_dev(dev),
2032 domain_context_mapped_cb, iommu);
ba395927
KA		 2033}
2034
f532959b
FY		 2035/* Returns a number of VTD pages, but aligned to MM page size */
2036static inline unsigned long aligned_nrpages(unsigned long host_addr,
2037 size_t size)
2038{
2039 host_addr &= ~PAGE_MASK;
2040 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2041}
2042
6dd9a7c7
YS		 2043/* Return largest possible superpage level for a given mapping */
2044static inline int hardware_largepage_caps(struct dmar_domain *domain,
2045 unsigned long iov_pfn,
2046 unsigned long phy_pfn,
2047 unsigned long pages)
2048{
2049 int support, level = 1;
2050 unsigned long pfnmerge;
2051
2052 support = domain->iommu_superpage;
2053
2054 /* To use a large page, the virtual *and* physical addresses
2055 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2056 of them will mean we have to use smaller pages. So just
2057 merge them and check both at once. */
2058 pfnmerge = iov_pfn | phy_pfn;
2059
2060 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2061 pages >>= VTD_STRIDE_SHIFT;
2062 if (!pages)
2063 break;
2064 pfnmerge >>= VTD_STRIDE_SHIFT;
2065 level++;
2066 support--;
2067 }
2068 return level;
2069}
2070
9051aa02
DW		 2071static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2072 struct scatterlist *sg, unsigned long phys_pfn,
2073 unsigned long nr_pages, int prot)
e1605495
DW		 2074{
2075 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2076 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2077 unsigned long sg_res = 0;
6dd9a7c7
YS		 2078 unsigned int largepage_lvl = 0;
2079 unsigned long lvl_pages = 0;
e1605495 2080
162d1b10 2081 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2082
2083 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2084 return -EINVAL;
2085
2086 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2087
cc4f14aa
JL		 2088 if (!sg) {
2089 sg_res = nr_pages;
9051aa02
DW		 2090 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2091 }
2092
6dd9a7c7 2093 while (nr_pages > 0) {
c85994e4
DW		 2094 uint64_t tmp;
2095
e1605495 2096 if (!sg_res) {
f532959b 2097 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2098 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2099 sg->dma_length = sg->length;
2100 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2101 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2102 }
6dd9a7c7 2103
e1605495 2104 if (!pte) {
6dd9a7c7
YS		 2105 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2106
5cf0a76f 2107 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2108 if (!pte)
2109 return -ENOMEM;
6dd9a7c7 2110 /* It is large page*/
6491d4d0 2111 if (largepage_lvl > 1) {
6dd9a7c7 2112 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2113 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2114 /*
2115 * Ensure that old small page tables are
2116 * removed to make room for superpage,
2117 * if they exist.
2118 */
6491d4d0 2119 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2120 iov_pfn + lvl_pages - 1);
6491d4d0 2121 } else {
6dd9a7c7 2122 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2123 }
6dd9a7c7 2124
e1605495
DW		 2125 }
2126 /* We don't need lock here, nobody else
2127 * touches the iova range
2128 */
7766a3fb 2129 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2130 if (tmp) {
1bf20f0d 2131 static int dumps = 5;
9f10e5bf
JR		 2132 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2133 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2134 if (dumps) {
2135 dumps--;
2136 debug_dma_dump_mappings(NULL);
2137 }
2138 WARN_ON(1);
2139 }
6dd9a7c7
YS		 2140
2141 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2142
2143 BUG_ON(nr_pages < lvl_pages);
2144 BUG_ON(sg_res < lvl_pages);
2145
2146 nr_pages -= lvl_pages;
2147 iov_pfn += lvl_pages;
2148 phys_pfn += lvl_pages;
2149 pteval += lvl_pages * VTD_PAGE_SIZE;
2150 sg_res -= lvl_pages;
2151
2152 /* If the next PTE would be the first in a new page, then we
2153 need to flush the cache on the entries we've just written.
2154 And then we'll need to recalculate 'pte', so clear it and
2155 let it get set again in the if (!pte) block above.
2156
2157 If we're done (!nr_pages) we need to flush the cache too.
2158
2159 Also if we've been setting superpages, we may need to
2160 recalculate 'pte' and switch back to smaller pages for the
2161 end of the mapping, if the trailing size is not enough to
2162 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2163 pte++;
6dd9a7c7
YS		 2164 if (!nr_pages || first_pte_in_page(pte) ||
2165 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2166 domain_flush_cache(domain, first_pte,
2167 (void *)pte - (void *)first_pte);
2168 pte = NULL;
2169 }
6dd9a7c7
YS		 2170
2171 if (!sg_res && nr_pages)
e1605495
DW		 2172 sg = sg_next(sg);
2173 }
2174 return 0;
2175}
2176
9051aa02
DW		 2177static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178 struct scatterlist *sg, unsigned long nr_pages,
2179 int prot)
ba395927 2180{
9051aa02
DW		 2181 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2182}
6f6a00e4 2183
9051aa02
DW		 2184static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2185 unsigned long phys_pfn, unsigned long nr_pages,
2186 int prot)
2187{
2188 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2189}
2190
2452d9db 2191static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2192{
c7151a8d
WH		 2193 if (!iommu)
2194 return;
8c11e798
WH		 2195
2196 clear_context_table(iommu, bus, devfn);
2197 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2198 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2199 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2200}
2201
109b9b04
DW		 2202static inline void unlink_domain_info(struct device_domain_info *info)
2203{
2204 assert_spin_locked(&device_domain_lock);
2205 list_del(&info->link);
2206 list_del(&info->global);
2207 if (info->dev)
0bcb3e28 2208 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2209}
2210
ba395927
KA		 2211static void domain_remove_dev_info(struct dmar_domain *domain)
2212{
3a74ca01 2213 struct device_domain_info *info, *tmp;
55d94043 2214 unsigned long flags;
ba395927 2215
55d94043 2216 spin_lock_irqsave(&device_domain_lock, flags);
76f45fe3 2217 list_for_each_entry_safe(info, tmp, &domain->devices, link)
127c7615 2218 __dmar_remove_one_dev_info(info);
55d94043 2219 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2220}
2221
2222/*
2223 * find_domain
1525a29a 2224 * Note: we use struct device->archdata.iommu stores the info
ba395927 2225 */
1525a29a 2226static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2227{
2228 struct device_domain_info *info;
2229
2230 /* No lock here, assumes no domain exit in normal case */
1525a29a 2231 info = dev->archdata.iommu;
ba395927
KA		 2232 if (info)
2233 return info->domain;
2234 return NULL;
2235}
2236
5a8f40e8 2237static inline struct device_domain_info *
745f2586
JL		 2238dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2239{
2240 struct device_domain_info *info;
2241
2242 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2243 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2244 info->devfn == devfn)
5a8f40e8 2245 return info;
745f2586
JL		 2246
2247 return NULL;
2248}
2249
5db31569
JR		 2250static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2251 int bus, int devfn,
2252 struct device *dev,
2253 struct dmar_domain *domain)
745f2586 2254{
5a8f40e8 2255 struct dmar_domain *found = NULL;
745f2586
JL		 2256 struct device_domain_info *info;
2257 unsigned long flags;
d160aca5 2258 int ret;
745f2586
JL		 2259
2260 info = alloc_devinfo_mem();
2261 if (!info)
b718cd3d 2262 return NULL;
745f2586 2263
745f2586
JL		 2264 info->bus = bus;
2265 info->devfn = devfn;
2266 info->dev = dev;
2267 info->domain = domain;
5a8f40e8 2268 info->iommu = iommu;
745f2586
JL		 2269
2270 spin_lock_irqsave(&device_domain_lock, flags);
2271 if (dev)
0bcb3e28 2272 found = find_domain(dev);
f303e507
JR		 2273
2274 if (!found) {
5a8f40e8 2275 struct device_domain_info *info2;
41e80dca 2276 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
f303e507
JR		 2277 if (info2) {
2278 found = info2->domain;
2279 info2->dev = dev;
2280 }
5a8f40e8 2281 }
f303e507 2282
745f2586
JL		 2283 if (found) {
2284 spin_unlock_irqrestore(&device_domain_lock, flags);
2285 free_devinfo_mem(info);
b718cd3d
DW		 2286 /* Caller must free the original domain */
2287 return found;
745f2586
JL		 2288 }
2289
d160aca5
JR		 2290 spin_lock(&iommu->lock);
2291 ret = domain_attach_iommu(domain, iommu);
2292 spin_unlock(&iommu->lock);
2293
2294 if (ret) {
c6c2cebd
JR		 2295 spin_unlock_irqrestore(&device_domain_lock, flags);
2296 return NULL;
2297 }
c6c2cebd 2298
b718cd3d
DW		 2299 list_add(&info->link, &domain->devices);
2300 list_add(&info->global, &device_domain_list);
2301 if (dev)
2302 dev->archdata.iommu = info;
2303 spin_unlock_irqrestore(&device_domain_lock, flags);
2304
cc4e2575
JR		 2305 if (dev && domain_context_mapping(domain, dev)) {
2306 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2307 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2308 return NULL;
2309 }
2310
b718cd3d 2311 return domain;
745f2586
JL		 2312}
2313
579305f7
AW		 2314static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2315{
2316 *(u16 *)opaque = alias;
2317 return 0;
2318}
2319
ba395927 2320/* domain is initialized */
146922ec 2321static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2322{
cc4e2575 2323 struct device_domain_info *info = NULL;
579305f7
AW		 2324 struct dmar_domain *domain, *tmp;
2325 struct intel_iommu *iommu;
08a7f456 2326 u16 req_id, dma_alias;
ba395927 2327 unsigned long flags;
aa4d066a 2328 u8 bus, devfn;
ba395927 2329
146922ec 2330 domain = find_domain(dev);
ba395927
KA		 2331 if (domain)
2332 return domain;
2333
579305f7
AW		 2334 iommu = device_to_iommu(dev, &bus, &devfn);
2335 if (!iommu)
2336 return NULL;
2337
08a7f456
JR		 2338 req_id = ((u16)bus << 8) | devfn;
2339
146922ec
DW		 2340 if (dev_is_pci(dev)) {
2341 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2342
579305f7
AW		 2343 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2344
2345 spin_lock_irqsave(&device_domain_lock, flags);
2346 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2347 PCI_BUS_NUM(dma_alias),
2348 dma_alias & 0xff);
2349 if (info) {
2350 iommu = info->iommu;
2351 domain = info->domain;
5a8f40e8 2352 }
579305f7 2353 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2354
579305f7
AW		 2355 /* DMA alias already has a domain, uses it */
2356 if (info)
2357 goto found_domain;
2358 }
ba395927 2359
146922ec 2360 /* Allocate and initialize new domain for the device */
ab8dfe25 2361 domain = alloc_domain(0);
745f2586 2362 if (!domain)
579305f7 2363 return NULL;
dc534b25 2364 if (domain_init(domain, iommu, gaw)) {
579305f7
AW		 2365 domain_exit(domain);
2366 return NULL;
2c2e2c38 2367 }
ba395927 2368
579305f7 2369 /* register PCI DMA alias device */
08a7f456 2370 if (req_id != dma_alias && dev_is_pci(dev)) {
5db31569
JR		 2371 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2372 dma_alias & 0xff, NULL, domain);
579305f7
AW		 2373
2374 if (!tmp || tmp != domain) {
2375 domain_exit(domain);
2376 domain = tmp;
2377 }
2378
b718cd3d 2379 if (!domain)
579305f7 2380 return NULL;
ba395927
KA		 2381 }
2382
2383found_domain:
5db31569 2384 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
579305f7
AW		 2385
2386 if (!tmp || tmp != domain) {
2387 domain_exit(domain);
2388 domain = tmp;
2389 }
b718cd3d
DW		 2390
2391 return domain;
ba395927
KA		 2392}
2393
2c2e2c38 2394static int iommu_identity_mapping;
e0fc7e0b
DW		 2395#define IDENTMAP_ALL 1
2396#define IDENTMAP_GFX 2
2397#define IDENTMAP_AZALIA 4
2c2e2c38 2398
b213203e
DW		 2399static int iommu_domain_identity_map(struct dmar_domain *domain,
2400 unsigned long long start,
2401 unsigned long long end)
ba395927 2402{
c5395d5c
DW		 2403 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2404 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2405
2406 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2407 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2408 pr_err("Reserving iova failed\n");
b213203e 2409 return -ENOMEM;
ba395927
KA		 2410 }
2411
af1089ce 2412 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2413 /*
2414 * RMRR range might have overlap with physical memory range,
2415 * clear it first
2416 */
c5395d5c 2417 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2418
c5395d5c
DW		 2419 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2420 last_vpfn - first_vpfn + 1,
61df7443 2421 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2422}
2423
0b9d9753 2424static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2425 unsigned long long start,
2426 unsigned long long end)
2427{
2428 struct dmar_domain *domain;
2429 int ret;
2430
0b9d9753 2431 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2432 if (!domain)
2433 return -ENOMEM;
2434
19943b0e
DW		 2435 /* For _hardware_ passthrough, don't bother. But for software
2436 passthrough, we do it anyway -- it may indicate a memory
2437 range which is reserved in E820, so which didn't get set
2438 up to start with in si_domain */
2439 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2440 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2441 dev_name(dev), start, end);
19943b0e
DW		 2442 return 0;
2443 }
2444
9f10e5bf
JR		 2445 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2446 dev_name(dev), start, end);
2447
5595b528
DW		 2448 if (end < start) {
2449 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2450 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2451 dmi_get_system_info(DMI_BIOS_VENDOR),
2452 dmi_get_system_info(DMI_BIOS_VERSION),
2453 dmi_get_system_info(DMI_PRODUCT_VERSION));
2454 ret = -EIO;
2455 goto error;
2456 }
2457
2ff729f5
DW		 2458 if (end >> agaw_to_width(domain->agaw)) {
2459 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2460 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2461 agaw_to_width(domain->agaw),
2462 dmi_get_system_info(DMI_BIOS_VENDOR),
2463 dmi_get_system_info(DMI_BIOS_VERSION),
2464 dmi_get_system_info(DMI_PRODUCT_VERSION));
2465 ret = -EIO;
2466 goto error;
2467 }
19943b0e 2468
b213203e 2469 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2470 if (ret)
2471 goto error;
2472
b213203e
DW		 2473 return 0;
2474
2475 error:
ba395927
KA		 2476 domain_exit(domain);
2477 return ret;
ba395927
KA		 2478}
2479
2480static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2481 struct device *dev)
ba395927 2482{
0b9d9753 2483 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2484 return 0;
0b9d9753
DW		 2485 return iommu_prepare_identity_map(dev, rmrr->base_address,
2486 rmrr->end_address);
ba395927
KA		 2487}
2488
d3f13810 2489#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2490static inline void iommu_prepare_isa(void)
2491{
2492 struct pci_dev *pdev;
2493 int ret;
2494
2495 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2496 if (!pdev)
2497 return;
2498
9f10e5bf 2499 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2500 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2501
2502 if (ret)
9f10e5bf 2503 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2504
9b27e82d 2505 pci_dev_put(pdev);
49a0429e
KA		 2506}
2507#else
2508static inline void iommu_prepare_isa(void)
2509{
2510 return;
2511}
d3f13810 2512#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2513
2c2e2c38 2514static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2515
071e1374 2516static int __init si_domain_init(int hw)
2c2e2c38 2517{
c7ab48d2 2518 int nid, ret = 0;
2c2e2c38 2519
ab8dfe25 2520 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2521 if (!si_domain)
2522 return -EFAULT;
2523
2c2e2c38
FY		 2524 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2525 domain_exit(si_domain);
2526 return -EFAULT;
2527 }
2528
0dc79715 2529 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2530
19943b0e
DW		 2531 if (hw)
2532 return 0;
2533
c7ab48d2 2534 for_each_online_node(nid) {
5dfe8660
TH		 2535 unsigned long start_pfn, end_pfn;
2536 int i;
2537
2538 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2539 ret = iommu_domain_identity_map(si_domain,
2540 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2541 if (ret)
2542 return ret;
2543 }
c7ab48d2
DW		 2544 }
2545
2c2e2c38
FY		 2546 return 0;
2547}
2548
9b226624 2549static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2550{
2551 struct device_domain_info *info;
2552
2553 if (likely(!iommu_identity_mapping))
2554 return 0;
2555
9b226624 2556 info = dev->archdata.iommu;
cb452a40
MT		 2557 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2558 return (info->domain == si_domain);
2c2e2c38 2559
2c2e2c38
FY		 2560 return 0;
2561}
2562
28ccce0d 2563static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2564{
0ac72664 2565 struct dmar_domain *ndomain;
5a8f40e8 2566 struct intel_iommu *iommu;
156baca8 2567 u8 bus, devfn;
2c2e2c38 2568
5913c9bf 2569 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2570 if (!iommu)
2571 return -ENODEV;
2572
5db31569 2573 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2574 if (ndomain != domain)
2575 return -EBUSY;
2c2e2c38
FY		 2576
2577 return 0;
2578}
2579
0b9d9753 2580static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2581{
2582 struct dmar_rmrr_unit *rmrr;
832bd858 2583 struct device *tmp;
ea2447f7
TM		 2584 int i;
2585
0e242612 2586 rcu_read_lock();
ea2447f7 2587 for_each_rmrr_units(rmrr) {
b683b230
JL		 2588 /*
2589 * Return TRUE if this RMRR contains the device that
2590 * is passed in.
2591 */
2592 for_each_active_dev_scope(rmrr->devices,
2593 rmrr->devices_cnt, i, tmp)
0b9d9753 2594 if (tmp == dev) {
0e242612 2595 rcu_read_unlock();
ea2447f7 2596 return true;
b683b230 2597 }
ea2447f7 2598 }
0e242612 2599 rcu_read_unlock();
ea2447f7
TM		 2600 return false;
2601}
2602
c875d2c1
AW		 2603/*
2604 * There are a couple cases where we need to restrict the functionality of
2605 * devices associated with RMRRs. The first is when evaluating a device for
2606 * identity mapping because problems exist when devices are moved in and out
2607 * of domains and their respective RMRR information is lost. This means that
2608 * a device with associated RMRRs will never be in a "passthrough" domain.
2609 * The second is use of the device through the IOMMU API. This interface
2610 * expects to have full control of the IOVA space for the device. We cannot
2611 * satisfy both the requirement that RMRR access is maintained and have an
2612 * unencumbered IOVA space. We also have no ability to quiesce the device's
2613 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2614 * We therefore prevent devices associated with an RMRR from participating in
2615 * the IOMMU API, which eliminates them from device assignment.
2616 *
2617 * In both cases we assume that PCI USB devices with RMRRs have them largely
2618 * for historical reasons and that the RMRR space is not actively used post
2619 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2620 *
2621 * The same exception is made for graphics devices, with the requirement that
2622 * any use of the RMRR regions will be torn down before assigning the device
2623 * to a guest.
c875d2c1
AW		 2624 */
2625static bool device_is_rmrr_locked(struct device *dev)
2626{
2627 if (!device_has_rmrr(dev))
2628 return false;
2629
2630 if (dev_is_pci(dev)) {
2631 struct pci_dev *pdev = to_pci_dev(dev);
2632
18436afd 2633 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2634 return false;
2635 }
2636
2637 return true;
2638}
2639
3bdb2591 2640static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2641{
ea2447f7 2642
3bdb2591
DW		 2643 if (dev_is_pci(dev)) {
2644 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2645
c875d2c1 2646 if (device_is_rmrr_locked(dev))
3bdb2591 2647 return 0;
e0fc7e0b 2648
3bdb2591
DW		 2649 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2650 return 1;
e0fc7e0b 2651
3bdb2591
DW		 2652 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2653 return 1;
6941af28 2654
3bdb2591 2655 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2656 return 0;
3bdb2591
DW		 2657
2658 /*
2659 * We want to start off with all devices in the 1:1 domain, and
2660 * take them out later if we find they can't access all of memory.
2661 *
2662 * However, we can't do this for PCI devices behind bridges,
2663 * because all PCI devices behind the same bridge will end up
2664 * with the same source-id on their transactions.
2665 *
2666 * Practically speaking, we can't change things around for these
2667 * devices at run-time, because we can't be sure there'll be no
2668 * DMA transactions in flight for any of their siblings.
2669 *
2670 * So PCI devices (unless they're on the root bus) as well as
2671 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2672 * the 1:1 domain, just in _case_ one of their siblings turns out
2673 * not to be able to map all of memory.
2674 */
2675 if (!pci_is_pcie(pdev)) {
2676 if (!pci_is_root_bus(pdev->bus))
2677 return 0;
2678 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2679 return 0;
2680 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2681 return 0;
3bdb2591
DW		 2682 } else {
2683 if (device_has_rmrr(dev))
2684 return 0;
2685 }
3dfc813d 2686
3bdb2591 2687 /*
3dfc813d 2688 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2689 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2690 * take them out of the 1:1 domain later.
2691 */
8fcc5372
CW		 2692 if (!startup) {
2693 /*
2694 * If the device's dma_mask is less than the system's memory
2695 * size then this is not a candidate for identity mapping.
2696 */
3bdb2591 2697 u64 dma_mask = *dev->dma_mask;
8fcc5372 2698
3bdb2591
DW		 2699 if (dev->coherent_dma_mask &&
2700 dev->coherent_dma_mask < dma_mask)
2701 dma_mask = dev->coherent_dma_mask;
8fcc5372 2702
3bdb2591 2703 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2704 }
6941af28
DW		 2705
2706 return 1;
2707}
2708
cf04eee8
DW		 2709static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2710{
2711 int ret;
2712
2713 if (!iommu_should_identity_map(dev, 1))
2714 return 0;
2715
28ccce0d 2716 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2717 if (!ret)
9f10e5bf
JR		 2718 pr_info("%s identity mapping for device %s\n",
2719 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2720 else if (ret == -ENODEV)
2721 /* device not associated with an iommu */
2722 ret = 0;
2723
2724 return ret;
2725}
2726
2727
071e1374 2728static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2729{
2c2e2c38 2730 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2731 struct dmar_drhd_unit *drhd;
2732 struct intel_iommu *iommu;
2733 struct device *dev;
2734 int i;
2735 int ret = 0;
2c2e2c38 2736
2c2e2c38 2737 for_each_pci_dev(pdev) {
cf04eee8
DW		 2738 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2739 if (ret)
2740 return ret;
2741 }
2742
2743 for_each_active_iommu(iommu, drhd)
2744 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2745 struct acpi_device_physical_node *pn;
2746 struct acpi_device *adev;
2747
2748 if (dev->bus != &acpi_bus_type)
2749 continue;
86080ccc 2750
cf04eee8
DW		 2751 adev= to_acpi_device(dev);
2752 mutex_lock(&adev->physical_node_lock);
2753 list_for_each_entry(pn, &adev->physical_node_list, node) {
2754 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2755 if (ret)
2756 break;
eae460b6 2757 }
cf04eee8
DW		 2758 mutex_unlock(&adev->physical_node_lock);
2759 if (ret)
2760 return ret;
62edf5dc 2761 }
2c2e2c38
FY		 2762
2763 return 0;
2764}
2765
ffebeb46
JL		 2766static void intel_iommu_init_qi(struct intel_iommu *iommu)
2767{
2768 /*
2769 * Start from the sane iommu hardware state.
2770 * If the queued invalidation is already initialized by us
2771 * (for example, while enabling interrupt-remapping) then
2772 * we got the things already rolling from a sane state.
2773 */
2774 if (!iommu->qi) {
2775 /*
2776 * Clear any previous faults.
2777 */
2778 dmar_fault(-1, iommu);
2779 /*
2780 * Disable queued invalidation if supported and already enabled
2781 * before OS handover.
2782 */
2783 dmar_disable_qi(iommu);
2784 }
2785
2786 if (dmar_enable_qi(iommu)) {
2787 /*
2788 * Queued Invalidate not enabled, use Register Based Invalidate
2789 */
2790 iommu->flush.flush_context = __iommu_flush_context;
2791 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2792 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2793 iommu->name);
2794 } else {
2795 iommu->flush.flush_context = qi_flush_context;
2796 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2797 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2798 }
2799}
2800
091d42e4
JR		 2801static int copy_context_table(struct intel_iommu *iommu,
2802 struct root_entry *old_re,
2803 struct context_entry **tbl,
2804 int bus, bool ext)
2805{
2806 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2807 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2808 phys_addr_t old_ce_phys;
2809
2810 tbl_idx = ext ? bus * 2 : bus;
2811
2812 for (devfn = 0; devfn < 256; devfn++) {
2813 /* First calculate the correct index */
2814 idx = (ext ? devfn * 2 : devfn) % 256;
2815
2816 if (idx == 0) {
2817 /* First save what we may have and clean up */
2818 if (new_ce) {
2819 tbl[tbl_idx] = new_ce;
2820 __iommu_flush_cache(iommu, new_ce,
2821 VTD_PAGE_SIZE);
2822 pos = 1;
2823 }
2824
2825 if (old_ce)
2826 iounmap(old_ce);
2827
2828 ret = 0;
2829 if (devfn < 0x80)
2830 old_ce_phys = root_entry_lctp(old_re);
2831 else
2832 old_ce_phys = root_entry_uctp(old_re);
2833
2834 if (!old_ce_phys) {
2835 if (ext && devfn == 0) {
2836 /* No LCTP, try UCTP */
2837 devfn = 0x7f;
2838 continue;
2839 } else {
2840 goto out;
2841 }
2842 }
2843
2844 ret = -ENOMEM;
2845 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2846 if (!old_ce)
2847 goto out;
2848
2849 new_ce = alloc_pgtable_page(iommu->node);
2850 if (!new_ce)
2851 goto out_unmap;
2852
2853 ret = 0;
2854 }
2855
2856 /* Now copy the context entry */
2857 ce = old_ce[idx];
2858
cf484d0e 2859 if (!__context_present(&ce))
091d42e4
JR		 2860 continue;
2861
dbcd861f
JR		 2862 did = context_domain_id(&ce);
2863 if (did >= 0 && did < cap_ndoms(iommu->cap))
2864 set_bit(did, iommu->domain_ids);
2865
cf484d0e
JR		 2866 /*
2867 * We need a marker for copied context entries. This
2868 * marker needs to work for the old format as well as
2869 * for extended context entries.
2870 *
2871 * Bit 67 of the context entry is used. In the old
2872 * format this bit is available to software, in the
2873 * extended format it is the PGE bit, but PGE is ignored
2874 * by HW if PASIDs are disabled (and thus still
2875 * available).
2876 *
2877 * So disable PASIDs first and then mark the entry
2878 * copied. This means that we don't copy PASID
2879 * translations from the old kernel, but this is fine as
2880 * faults there are not fatal.
2881 */
2882 context_clear_pasid_enable(&ce);
2883 context_set_copied(&ce);
2884
091d42e4
JR		 2885 new_ce[idx] = ce;
2886 }
2887
2888 tbl[tbl_idx + pos] = new_ce;
2889
2890 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2891
2892out_unmap:
2893 iounmap(old_ce);
2894
2895out:
2896 return ret;
2897}
2898
2899static int copy_translation_tables(struct intel_iommu *iommu)
2900{
2901 struct context_entry **ctxt_tbls;
2902 struct root_entry *old_rt;
2903 phys_addr_t old_rt_phys;
2904 int ctxt_table_entries;
2905 unsigned long flags;
2906 u64 rtaddr_reg;
2907 int bus, ret;
c3361f2f 2908 bool new_ext, ext;
091d42e4
JR		 2909
2910 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2911 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2912 new_ext = !!ecap_ecs(iommu->ecap);
2913
2914 /*
2915 * The RTT bit can only be changed when translation is disabled,
2916 * but disabling translation means to open a window for data
2917 * corruption. So bail out and don't copy anything if we would
2918 * have to change the bit.
2919 */
2920 if (new_ext != ext)
2921 return -EINVAL;
091d42e4
JR		 2922
2923 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2924 if (!old_rt_phys)
2925 return -EINVAL;
2926
2927 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2928 if (!old_rt)
2929 return -ENOMEM;
2930
2931 /* This is too big for the stack - allocate it from slab */
2932 ctxt_table_entries = ext ? 512 : 256;
2933 ret = -ENOMEM;
2934 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2935 if (!ctxt_tbls)
2936 goto out_unmap;
2937
2938 for (bus = 0; bus < 256; bus++) {
2939 ret = copy_context_table(iommu, &old_rt[bus],
2940 ctxt_tbls, bus, ext);
2941 if (ret) {
2942 pr_err("%s: Failed to copy context table for bus %d\n",
2943 iommu->name, bus);
2944 continue;
2945 }
2946 }
2947
2948 spin_lock_irqsave(&iommu->lock, flags);
2949
2950 /* Context tables are copied, now write them to the root_entry table */
2951 for (bus = 0; bus < 256; bus++) {
2952 int idx = ext ? bus * 2 : bus;
2953 u64 val;
2954
2955 if (ctxt_tbls[idx]) {
2956 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2957 iommu->root_entry[bus].lo = val;
2958 }
2959
2960 if (!ext || !ctxt_tbls[idx + 1])
2961 continue;
2962
2963 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2964 iommu->root_entry[bus].hi = val;
2965 }
2966
2967 spin_unlock_irqrestore(&iommu->lock, flags);
2968
2969 kfree(ctxt_tbls);
2970
2971 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2972
2973 ret = 0;
2974
2975out_unmap:
2976 iounmap(old_rt);
2977
2978 return ret;
2979}
2980
b779260b 2981static int __init init_dmars(void)
ba395927
KA		 2982{
2983 struct dmar_drhd_unit *drhd;
2984 struct dmar_rmrr_unit *rmrr;
a87f4918 2985 bool copied_tables = false;
832bd858 2986 struct device *dev;
ba395927 2987 struct intel_iommu *iommu;
9d783ba0 2988 int i, ret;
2c2e2c38 2989
ba395927
KA		 2990 /*
2991 * for each drhd
2992 * allocate root
2993 * initialize and program root entry to not present
2994 * endfor
2995 */
2996 for_each_drhd_unit(drhd) {
5e0d2a6f 2997 /*
2998 * lock not needed as this is only incremented in the single
2999 * threaded kernel __init code path all other access are read
3000 * only
3001 */
78d8e704 3002 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3003 g_num_of_iommus++;
3004 continue;
3005 }
9f10e5bf 3006 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3007 }
3008
ffebeb46
JL		 3009 /* Preallocate enough resources for IOMMU hot-addition */
3010 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3011 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3012
d9630fe9
WH		 3013 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3014 GFP_KERNEL);
3015 if (!g_iommus) {
9f10e5bf 3016 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3017 ret = -ENOMEM;
3018 goto error;
3019 }
3020
80b20dd8 3021 deferred_flush = kzalloc(g_num_of_iommus *
3022 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3023 if (!deferred_flush) {
5e0d2a6f 3024 ret = -ENOMEM;
989d51fc 3025 goto free_g_iommus;
5e0d2a6f 3026 }
3027
7c919779 3028 for_each_active_iommu(iommu, drhd) {
d9630fe9 3029 g_iommus[iommu->seq_id] = iommu;
ba395927 3030
b63d80d1
JR		 3031 intel_iommu_init_qi(iommu);
3032
e61d98d8
SS		 3033 ret = iommu_init_domains(iommu);
3034 if (ret)
989d51fc 3035 goto free_iommu;
e61d98d8 3036
4158c2ec
JR		 3037 init_translation_status(iommu);
3038
091d42e4
JR		 3039 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3040 iommu_disable_translation(iommu);
3041 clear_translation_pre_enabled(iommu);
3042 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3043 iommu->name);
3044 }
4158c2ec 3045
ba395927
KA		 3046 /*
3047 * TBD:
3048 * we could share the same root & context tables
25985edc 3049 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3050 */
3051 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3052 if (ret)
989d51fc 3053 goto free_iommu;
5f0a7f76 3054
091d42e4
JR		 3055 if (translation_pre_enabled(iommu)) {
3056 pr_info("Translation already enabled - trying to copy translation structures\n");
3057
3058 ret = copy_translation_tables(iommu);
3059 if (ret) {
3060 /*
3061 * We found the IOMMU with translation
3062 * enabled - but failed to copy over the
3063 * old root-entry table. Try to proceed
3064 * by disabling translation now and
3065 * allocating a clean root-entry table.
3066 * This might cause DMAR faults, but
3067 * probably the dump will still succeed.
3068 */
3069 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3070 iommu->name);
3071 iommu_disable_translation(iommu);
3072 clear_translation_pre_enabled(iommu);
3073 } else {
3074 pr_info("Copied translation tables from previous kernel for %s\n",
3075 iommu->name);
a87f4918 3076 copied_tables = true;
091d42e4
JR		 3077 }
3078 }
3079
5f0a7f76
JR		 3080 iommu_flush_write_buffer(iommu);
3081 iommu_set_root_entry(iommu);
3082 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3083 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3084
4ed0d3e6 3085 if (!ecap_pass_through(iommu->ecap))
19943b0e 3086 hw_pass_through = 0;
ba395927
KA		 3087 }
3088
19943b0e 3089 if (iommu_pass_through)
e0fc7e0b
DW		 3090 iommu_identity_mapping |= IDENTMAP_ALL;
3091
d3f13810 3092#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3093 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3094#endif
e0fc7e0b 3095
86080ccc
JR		 3096 if (iommu_identity_mapping) {
3097 ret = si_domain_init(hw_pass_through);
3098 if (ret)
3099 goto free_iommu;
3100 }
3101
e0fc7e0b
DW		 3102 check_tylersburg_isoch();
3103
a87f4918
JR		 3104 /*
3105 * If we copied translations from a previous kernel in the kdump
3106 * case, we can not assign the devices to domains now, as that
3107 * would eliminate the old mappings. So skip this part and defer
3108 * the assignment to device driver initialization time.
3109 */
3110 if (copied_tables)
3111 goto domains_done;
3112
ba395927 3113 /*
19943b0e
DW		 3114 * If pass through is not set or not enabled, setup context entries for
3115 * identity mappings for rmrr, gfx, and isa and may fall back to static
3116 * identity mapping if iommu_identity_mapping is set.
ba395927 3117 */
19943b0e
DW		 3118 if (iommu_identity_mapping) {
3119 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3120 if (ret) {
9f10e5bf 3121 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3122 goto free_iommu;
ba395927
KA		 3123 }
3124 }
ba395927 3125 /*
19943b0e
DW		 3126 * For each rmrr
3127 * for each dev attached to rmrr
3128 * do
3129 * locate drhd for dev, alloc domain for dev
3130 * allocate free domain
3131 * allocate page table entries for rmrr
3132 * if context not allocated for bus
3133 * allocate and init context
3134 * set present in root table for this bus
3135 * init context with domain, translation etc
3136 * endfor
3137 * endfor
ba395927 3138 */
9f10e5bf 3139 pr_info("Setting RMRR:\n");
19943b0e 3140 for_each_rmrr_units(rmrr) {
b683b230
JL		 3141 /* some BIOS lists non-exist devices in DMAR table. */
3142 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3143 i, dev) {
0b9d9753 3144 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3145 if (ret)
9f10e5bf 3146 pr_err("Mapping reserved region failed\n");
ba395927 3147 }
4ed0d3e6 3148 }
49a0429e 3149
19943b0e
DW		 3150 iommu_prepare_isa();
3151
a87f4918
JR		 3152domains_done:
3153
ba395927
KA		 3154 /*
3155 * for each drhd
3156 * enable fault log
3157 * global invalidate context cache
3158 * global invalidate iotlb
3159 * enable translation
3160 */
7c919779 3161 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3162 if (drhd->ignored) {
3163 /*
3164 * we always have to disable PMRs or DMA may fail on
3165 * this device
3166 */
3167 if (force_on)
7c919779 3168 iommu_disable_protect_mem_regions(iommu);
ba395927 3169 continue;
51a63e67 3170 }
ba395927
KA		 3171
3172 iommu_flush_write_buffer(iommu);
3173
3460a6d9
KA		 3174 ret = dmar_set_interrupt(iommu);
3175 if (ret)
989d51fc 3176 goto free_iommu;
3460a6d9 3177
8939ddf6
JR		 3178 if (!translation_pre_enabled(iommu))
3179 iommu_enable_translation(iommu);
3180
b94996c9 3181 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3182 }
3183
3184 return 0;
989d51fc
JL		 3185
3186free_iommu:
ffebeb46
JL		 3187 for_each_active_iommu(iommu, drhd) {
3188 disable_dmar_iommu(iommu);
a868e6b7 3189 free_dmar_iommu(iommu);
ffebeb46 3190 }
9bdc531e 3191 kfree(deferred_flush);
989d51fc 3192free_g_iommus:
d9630fe9 3193 kfree(g_iommus);
989d51fc 3194error:
ba395927
KA		 3195 return ret;
3196}
3197
5a5e02a6 3198/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3199static struct iova *intel_alloc_iova(struct device *dev,
3200 struct dmar_domain *domain,
3201 unsigned long nrpages, uint64_t dma_mask)
ba395927 3202{
ba395927 3203 struct iova *iova = NULL;
ba395927 3204
875764de
DW		 3205 /* Restrict dma_mask to the width that the iommu can handle */
3206 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3207
3208 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3209 /*
3210 * First try to allocate an io virtual address in
284901a9 3211 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3212 * from higher range
ba395927 3213 */
875764de
DW		 3214 iova = alloc_iova(&domain->iovad, nrpages,
3215 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3216 if (iova)
3217 return iova;
3218 }
3219 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3220 if (unlikely(!iova)) {
9f10e5bf 3221 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3222 nrpages, dev_name(dev));
f76aec76
KA		 3223 return NULL;
3224 }
3225
3226 return iova;
3227}
3228
d4b709f4 3229static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3230{
3231 struct dmar_domain *domain;
f76aec76 3232
d4b709f4 3233 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3234 if (!domain) {
9f10e5bf 3235 pr_err("Allocating domain for %s failed\n",
d4b709f4 3236 dev_name(dev));
4fe05bbc 3237 return NULL;
ba395927
KA		 3238 }
3239
f76aec76
KA		 3240 return domain;
3241}
3242
d4b709f4 3243static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3244{
3245 struct device_domain_info *info;
3246
3247 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3248 info = dev->archdata.iommu;
147202aa
DW		 3249 if (likely(info))
3250 return info->domain;
3251
3252 return __get_valid_domain_for_dev(dev);
3253}
3254
ecb509ec 3255/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3256static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3257{
3258 int found;
3259
3d89194a 3260 if (iommu_dummy(dev))
1e4c64c4
DW		 3261 return 1;
3262
2c2e2c38 3263 if (!iommu_identity_mapping)
1e4c64c4 3264 return 0;
2c2e2c38 3265
9b226624 3266 found = identity_mapping(dev);
2c2e2c38 3267 if (found) {
ecb509ec 3268 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3269 return 1;
3270 else {
3271 /*
3272 * 32 bit DMA is removed from si_domain and fall back
3273 * to non-identity mapping.
3274 */
e6de0f8d 3275 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3276 pr_info("32bit %s uses non-identity mapping\n",
3277 dev_name(dev));
2c2e2c38
FY		 3278 return 0;
3279 }
3280 } else {
3281 /*
3282 * In case of a detached 64 bit DMA device from vm, the device
3283 * is put into si_domain for identity mapping.
3284 */
ecb509ec 3285 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3286 int ret;
28ccce0d 3287 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3288 if (!ret) {
9f10e5bf
JR		 3289 pr_info("64bit %s uses identity mapping\n",
3290 dev_name(dev));
2c2e2c38
FY		 3291 return 1;
3292 }
3293 }
3294 }
3295
1e4c64c4 3296 return 0;
2c2e2c38
FY		 3297}
3298
5040a918 3299static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3300 size_t size, int dir, u64 dma_mask)
f76aec76 3301{
f76aec76 3302 struct dmar_domain *domain;
5b6985ce 3303 phys_addr_t start_paddr;
f76aec76
KA		 3304 struct iova *iova;
3305 int prot = 0;
6865f0d1 3306 int ret;
8c11e798 3307 struct intel_iommu *iommu;
33041ec0 3308 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3309
3310 BUG_ON(dir == DMA_NONE);
2c2e2c38 3311
5040a918 3312 if (iommu_no_mapping(dev))
6865f0d1 3313 return paddr;
f76aec76 3314
5040a918 3315 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3316 if (!domain)
3317 return 0;
3318
8c11e798 3319 iommu = domain_get_iommu(domain);
88cb6a74 3320 size = aligned_nrpages(paddr, size);
f76aec76 3321
5040a918 3322 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3323 if (!iova)
3324 goto error;
3325
ba395927
KA		 3326 /*
3327 * Check if DMAR supports zero-length reads on write only
3328 * mappings..
3329 */
3330 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3331 !cap_zlr(iommu->cap))
ba395927
KA		 3332 prot |= DMA_PTE_READ;
3333 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3334 prot |= DMA_PTE_WRITE;
3335 /*
6865f0d1 3336 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3337 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3338 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3339 * is not a big problem
3340 */
0ab36de2 3341 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3342 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3343 if (ret)
3344 goto error;
3345
1f0ef2aa
DW		 3346 /* it's a non-present to present mapping. Only flush if caching mode */
3347 if (cap_caching_mode(iommu->cap))
a1ddcbe9
JR		 3348 iommu_flush_iotlb_psi(iommu, domain,
3349 mm_to_dma_pfn(iova->pfn_lo),
3350 size, 0, 1);
1f0ef2aa 3351 else
8c11e798 3352 iommu_flush_write_buffer(iommu);
f76aec76 3353
03d6a246
DW		 3354 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3355 start_paddr += paddr & ~PAGE_MASK;
3356 return start_paddr;
ba395927 3357
ba395927 3358error:
f76aec76
KA		 3359 if (iova)
3360 __free_iova(&domain->iovad, iova);
9f10e5bf 3361 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3362 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3363 return 0;
3364}
3365
ffbbef5c
FT		 3366static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3367 unsigned long offset, size_t size,
3368 enum dma_data_direction dir,
3369 struct dma_attrs *attrs)
bb9e6d65 3370{
ffbbef5c 3371 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3372 dir, *dev->dma_mask);
bb9e6d65
FT		 3373}
3374
5e0d2a6f 3375static void flush_unmaps(void)
3376{
80b20dd8 3377 int i, j;
5e0d2a6f 3378
5e0d2a6f 3379 timer_on = 0;
3380
3381 /* just flush them all */
3382 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3383 struct intel_iommu *iommu = g_iommus[i];
3384 if (!iommu)
3385 continue;
c42d9f32 3386
9dd2fe89
YZ		 3387 if (!deferred_flush[i].next)
3388 continue;
3389
78d5f0f5
NA		 3390 /* In caching mode, global flushes turn emulation expensive */
3391 if (!cap_caching_mode(iommu->cap))
3392 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3393 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3394 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3395 unsigned long mask;
3396 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3397 struct dmar_domain *domain = deferred_flush[i].domain[j];
3398
3399 /* On real hardware multiple invalidations are expensive */
3400 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3401 iommu_flush_iotlb_psi(iommu, domain,
a156ef99 3402 iova->pfn_lo, iova_size(iova),
ea8ea460 3403 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3404 else {
a156ef99 3405 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3406 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3407 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3408 }
93a23a72 3409 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3410 if (deferred_flush[i].freelist[j])
3411 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3412 }
9dd2fe89 3413 deferred_flush[i].next = 0;
5e0d2a6f 3414 }
3415
5e0d2a6f 3416 list_size = 0;
5e0d2a6f 3417}
3418
3419static void flush_unmaps_timeout(unsigned long data)
3420{
80b20dd8 3421 unsigned long flags;
3422
3423 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3424 flush_unmaps();
80b20dd8 3425 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3426}
3427
ea8ea460 3428static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3429{
3430 unsigned long flags;
80b20dd8 3431 int next, iommu_id;
8c11e798 3432 struct intel_iommu *iommu;
5e0d2a6f 3433
3434 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3435 if (list_size == HIGH_WATER_MARK)
3436 flush_unmaps();
3437
8c11e798
WH		 3438 iommu = domain_get_iommu(dom);
3439 iommu_id = iommu->seq_id;
c42d9f32 3440
80b20dd8 3441 next = deferred_flush[iommu_id].next;
3442 deferred_flush[iommu_id].domain[next] = dom;
3443 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3444 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3445 deferred_flush[iommu_id].next++;
5e0d2a6f 3446
3447 if (!timer_on) {
3448 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3449 timer_on = 1;
3450 }
3451 list_size++;
3452 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3453}
3454
d41a4adb 3455static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3456{
f76aec76 3457 struct dmar_domain *domain;
d794dc9b 3458 unsigned long start_pfn, last_pfn;
ba395927 3459 struct iova *iova;
8c11e798 3460 struct intel_iommu *iommu;
ea8ea460 3461 struct page *freelist;
ba395927 3462
73676832 3463 if (iommu_no_mapping(dev))
f76aec76 3464 return;
2c2e2c38 3465
1525a29a 3466 domain = find_domain(dev);
ba395927
KA		 3467 BUG_ON(!domain);
3468
8c11e798
WH		 3469 iommu = domain_get_iommu(domain);
3470
ba395927 3471 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3472 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3473 (unsigned long long)dev_addr))
ba395927 3474 return;
ba395927 3475
d794dc9b
DW		 3476 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3477 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3478
d794dc9b 3479 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3480 dev_name(dev), start_pfn, last_pfn);
ba395927 3481
ea8ea460 3482 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3483
5e0d2a6f 3484 if (intel_iommu_strict) {
a1ddcbe9 3485 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
ea8ea460 3486 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3487 /* free iova */
3488 __free_iova(&domain->iovad, iova);
ea8ea460 3489 dma_free_pagelist(freelist);
5e0d2a6f 3490 } else {
ea8ea460 3491 add_unmap(domain, iova, freelist);
5e0d2a6f 3492 /*
3493 * queue up the release of the unmap to save the 1/6th of the
3494 * cpu used up by the iotlb flush operation...
3495 */
5e0d2a6f 3496 }
ba395927
KA		 3497}
3498
d41a4adb
JL		 3499static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3500 size_t size, enum dma_data_direction dir,
3501 struct dma_attrs *attrs)
3502{
3503 intel_unmap(dev, dev_addr);
3504}
3505
5040a918 3506static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3507 dma_addr_t *dma_handle, gfp_t flags,
3508 struct dma_attrs *attrs)
ba395927 3509{
36746436 3510 struct page *page = NULL;
ba395927
KA		 3511 int order;
3512
5b6985ce 3513 size = PAGE_ALIGN(size);
ba395927 3514 order = get_order(size);
e8bb910d 3515
5040a918 3516 if (!iommu_no_mapping(dev))
e8bb910d 3517 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3518 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3519 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3520 flags |= GFP_DMA;
3521 else
3522 flags |= GFP_DMA32;
3523 }
ba395927 3524
36746436
AM		 3525 if (flags & __GFP_WAIT) {
3526 unsigned int count = size >> PAGE_SHIFT;
3527
3528 page = dma_alloc_from_contiguous(dev, count, order);
3529 if (page && iommu_no_mapping(dev) &&
3530 page_to_phys(page) + size > dev->coherent_dma_mask) {
3531 dma_release_from_contiguous(dev, page, count);
3532 page = NULL;
3533 }
3534 }
3535
3536 if (!page)
3537 page = alloc_pages(flags, order);
3538 if (!page)
ba395927 3539 return NULL;
36746436 3540 memset(page_address(page), 0, size);
ba395927 3541
36746436 3542 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3543 DMA_BIDIRECTIONAL,
5040a918 3544 dev->coherent_dma_mask);
ba395927 3545 if (*dma_handle)
36746436
AM		 3546 return page_address(page);
3547 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3548 __free_pages(page, order);
3549
ba395927
KA		 3550 return NULL;
3551}
3552
5040a918 3553static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3554 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3555{
3556 int order;
36746436 3557 struct page *page = virt_to_page(vaddr);
ba395927 3558
5b6985ce 3559 size = PAGE_ALIGN(size);
ba395927
KA		 3560 order = get_order(size);
3561
d41a4adb 3562 intel_unmap(dev, dma_handle);
36746436
AM		 3563 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3564 __free_pages(page, order);
ba395927
KA		 3565}
3566
5040a918 3567static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3568 int nelems, enum dma_data_direction dir,
3569 struct dma_attrs *attrs)
ba395927 3570{
d41a4adb 3571 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3572}
3573
ba395927 3574static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3575 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3576{
3577 int i;
c03ab37c 3578 struct scatterlist *sg;
ba395927 3579
c03ab37c 3580 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3581 BUG_ON(!sg_page(sg));
4cf2e75d 3582 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3583 sg->dma_length = sg->length;
ba395927
KA		 3584 }
3585 return nelems;
3586}
3587
5040a918 3588static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3589 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3590{
ba395927 3591 int i;
ba395927 3592 struct dmar_domain *domain;
f76aec76
KA		 3593 size_t size = 0;
3594 int prot = 0;
f76aec76
KA		 3595 struct iova *iova = NULL;
3596 int ret;
c03ab37c 3597 struct scatterlist *sg;
b536d24d 3598 unsigned long start_vpfn;
8c11e798 3599 struct intel_iommu *iommu;
ba395927
KA		 3600
3601 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3602 if (iommu_no_mapping(dev))
3603 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3604
5040a918 3605 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3606 if (!domain)
3607 return 0;
3608
8c11e798
WH		 3609 iommu = domain_get_iommu(domain);
3610
b536d24d 3611 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3612 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3613
5040a918
DW		 3614 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3615 *dev->dma_mask);
f76aec76 3616 if (!iova) {
c03ab37c 3617 sglist->dma_length = 0;
f76aec76
KA		 3618 return 0;
3619 }
3620
3621 /*
3622 * Check if DMAR supports zero-length reads on write only
3623 * mappings..
3624 */
3625 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3626 !cap_zlr(iommu->cap))
f76aec76
KA		 3627 prot |= DMA_PTE_READ;
3628 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3629 prot |= DMA_PTE_WRITE;
3630
b536d24d 3631 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3632
f532959b 3633 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3634 if (unlikely(ret)) {
e1605495
DW		 3635 dma_pte_free_pagetable(domain, start_vpfn,
3636 start_vpfn + size - 1);
e1605495
DW		 3637 __free_iova(&domain->iovad, iova);
3638 return 0;
ba395927
KA		 3639 }
3640
1f0ef2aa
DW		 3641 /* it's a non-present to present mapping. Only flush if caching mode */
3642 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3643 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3644 else
8c11e798 3645 iommu_flush_write_buffer(iommu);
1f0ef2aa 3646
ba395927
KA		 3647 return nelems;
3648}
3649
dfb805e8
FT		 3650static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3651{
3652 return !dma_addr;
3653}
3654
160c1d8e 3655struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3656 .alloc = intel_alloc_coherent,
3657 .free = intel_free_coherent,
ba395927
KA		 3658 .map_sg = intel_map_sg,
3659 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3660 .map_page = intel_map_page,
3661 .unmap_page = intel_unmap_page,
dfb805e8 3662 .mapping_error = intel_mapping_error,
ba395927
KA		 3663};
3664
3665static inline int iommu_domain_cache_init(void)
3666{
3667 int ret = 0;
3668
3669 iommu_domain_cache = kmem_cache_create("iommu_domain",
3670 sizeof(struct dmar_domain),
3671 0,
3672 SLAB_HWCACHE_ALIGN,
3673
3674 NULL);
3675 if (!iommu_domain_cache) {
9f10e5bf 3676 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3677 ret = -ENOMEM;
3678 }
3679
3680 return ret;
3681}
3682
3683static inline int iommu_devinfo_cache_init(void)
3684{
3685 int ret = 0;
3686
3687 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3688 sizeof(struct device_domain_info),
3689 0,
3690 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3691 NULL);
3692 if (!iommu_devinfo_cache) {
9f10e5bf 3693 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3694 ret = -ENOMEM;
3695 }
3696
3697 return ret;
3698}
3699
ba395927
KA		 3700static int __init iommu_init_mempool(void)
3701{
3702 int ret;
3703 ret = iommu_iova_cache_init();
3704 if (ret)
3705 return ret;
3706
3707 ret = iommu_domain_cache_init();
3708 if (ret)
3709 goto domain_error;
3710
3711 ret = iommu_devinfo_cache_init();
3712 if (!ret)
3713 return ret;
3714
3715 kmem_cache_destroy(iommu_domain_cache);
3716domain_error:
85b45456 3717 iommu_iova_cache_destroy();
ba395927
KA		 3718
3719 return -ENOMEM;
3720}
3721
3722static void __init iommu_exit_mempool(void)
3723{
3724 kmem_cache_destroy(iommu_devinfo_cache);
3725 kmem_cache_destroy(iommu_domain_cache);
85b45456 3726 iommu_iova_cache_destroy();
ba395927
KA		 3727}
3728
556ab45f
DW		 3729static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3730{
3731 struct dmar_drhd_unit *drhd;
3732 u32 vtbar;
3733 int rc;
3734
3735 /* We know that this device on this chipset has its own IOMMU.
3736 * If we find it under a different IOMMU, then the BIOS is lying
3737 * to us. Hope that the IOMMU for this device is actually
3738 * disabled, and it needs no translation...
3739 */
3740 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3741 if (rc) {
3742 /* "can't" happen */
3743 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3744 return;
3745 }
3746 vtbar &= 0xffff0000;
3747
3748 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3749 drhd = dmar_find_matched_drhd_unit(pdev);
3750 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3751 TAINT_FIRMWARE_WORKAROUND,
3752 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3753 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3754}
3755DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3756
ba395927
KA		 3757static void __init init_no_remapping_devices(void)
3758{
3759 struct dmar_drhd_unit *drhd;
832bd858 3760 struct device *dev;
b683b230 3761 int i;
ba395927
KA		 3762
3763 for_each_drhd_unit(drhd) {
3764 if (!drhd->include_all) {
b683b230
JL		 3765 for_each_active_dev_scope(drhd->devices,
3766 drhd->devices_cnt, i, dev)
3767 break;
832bd858 3768 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3769 if (i == drhd->devices_cnt)
3770 drhd->ignored = 1;
3771 }
3772 }
3773
7c919779 3774 for_each_active_drhd_unit(drhd) {
7c919779 3775 if (drhd->include_all)
ba395927
KA		 3776 continue;
3777
b683b230
JL		 3778 for_each_active_dev_scope(drhd->devices,
3779 drhd->devices_cnt, i, dev)
832bd858 3780 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3781 break;
ba395927
KA		 3782 if (i < drhd->devices_cnt)
3783 continue;
3784
c0771df8
DW		 3785 /* This IOMMU has *only* gfx devices. Either bypass it or
3786 set the gfx_mapped flag, as appropriate */
3787 if (dmar_map_gfx) {
3788 intel_iommu_gfx_mapped = 1;
3789 } else {
3790 drhd->ignored = 1;
b683b230
JL		 3791 for_each_active_dev_scope(drhd->devices,
3792 drhd->devices_cnt, i, dev)
832bd858 3793 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3794 }
3795 }
3796}
3797
f59c7b69
FY		 3798#ifdef CONFIG_SUSPEND
3799static int init_iommu_hw(void)
3800{
3801 struct dmar_drhd_unit *drhd;
3802 struct intel_iommu *iommu = NULL;
3803
3804 for_each_active_iommu(iommu, drhd)
3805 if (iommu->qi)
3806 dmar_reenable_qi(iommu);
3807
b779260b
JC		 3808 for_each_iommu(iommu, drhd) {
3809 if (drhd->ignored) {
3810 /*
3811 * we always have to disable PMRs or DMA may fail on
3812 * this device
3813 */
3814 if (force_on)
3815 iommu_disable_protect_mem_regions(iommu);
3816 continue;
3817 }
3818
f59c7b69
FY		 3819 iommu_flush_write_buffer(iommu);
3820
3821 iommu_set_root_entry(iommu);
3822
3823 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3824 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3825 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3826 iommu_enable_translation(iommu);
b94996c9 3827 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3828 }
3829
3830 return 0;
3831}
3832
3833static void iommu_flush_all(void)
3834{
3835 struct dmar_drhd_unit *drhd;
3836 struct intel_iommu *iommu;
3837
3838 for_each_active_iommu(iommu, drhd) {
3839 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3840 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3841 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3842 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3843 }
3844}
3845
134fac3f 3846static int iommu_suspend(void)
f59c7b69
FY		 3847{
3848 struct dmar_drhd_unit *drhd;
3849 struct intel_iommu *iommu = NULL;
3850 unsigned long flag;
3851
3852 for_each_active_iommu(iommu, drhd) {
3853 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3854 GFP_ATOMIC);
3855 if (!iommu->iommu_state)
3856 goto nomem;
3857 }
3858
3859 iommu_flush_all();
3860
3861 for_each_active_iommu(iommu, drhd) {
3862 iommu_disable_translation(iommu);
3863
1f5b3c3f 3864 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3865
3866 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3867 readl(iommu->reg + DMAR_FECTL_REG);
3868 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3869 readl(iommu->reg + DMAR_FEDATA_REG);
3870 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3871 readl(iommu->reg + DMAR_FEADDR_REG);
3872 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3873 readl(iommu->reg + DMAR_FEUADDR_REG);
3874
1f5b3c3f 3875 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3876 }
3877 return 0;
3878
3879nomem:
3880 for_each_active_iommu(iommu, drhd)
3881 kfree(iommu->iommu_state);
3882
3883 return -ENOMEM;
3884}
3885
134fac3f 3886static void iommu_resume(void)
f59c7b69
FY		 3887{
3888 struct dmar_drhd_unit *drhd;
3889 struct intel_iommu *iommu = NULL;
3890 unsigned long flag;
3891
3892 if (init_iommu_hw()) {
b779260b
JC		 3893 if (force_on)
3894 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3895 else
3896 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3897 return;
f59c7b69
FY		 3898 }
3899
3900 for_each_active_iommu(iommu, drhd) {
3901
1f5b3c3f 3902 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3903
3904 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3905 iommu->reg + DMAR_FECTL_REG);
3906 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3907 iommu->reg + DMAR_FEDATA_REG);
3908 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3909 iommu->reg + DMAR_FEADDR_REG);
3910 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3911 iommu->reg + DMAR_FEUADDR_REG);
3912
1f5b3c3f 3913 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3914 }
3915
3916 for_each_active_iommu(iommu, drhd)
3917 kfree(iommu->iommu_state);
f59c7b69
FY		 3918}
3919
134fac3f 3920static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3921 .resume = iommu_resume,
3922 .suspend = iommu_suspend,
3923};
3924
134fac3f 3925static void __init init_iommu_pm_ops(void)
f59c7b69 3926{
134fac3f 3927 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3928}
3929
3930#else
99592ba4 3931static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3932#endif /* CONFIG_PM */
3933
318fe7df 3934
c2a0b538 3935int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3936{
3937 struct acpi_dmar_reserved_memory *rmrr;
3938 struct dmar_rmrr_unit *rmrru;
3939
3940 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3941 if (!rmrru)
3942 return -ENOMEM;
3943
3944 rmrru->hdr = header;
3945 rmrr = (struct acpi_dmar_reserved_memory *)header;
3946 rmrru->base_address = rmrr->base_address;
3947 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3948 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3949 ((void *)rmrr) + rmrr->header.length,
3950 &rmrru->devices_cnt);
3951 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3952 kfree(rmrru);
3953 return -ENOMEM;
3954 }
318fe7df 3955
2e455289 3956 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3957
2e455289 3958 return 0;
318fe7df
SS		 3959}
3960
6b197249
JL		 3961static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3962{
3963 struct dmar_atsr_unit *atsru;
3964 struct acpi_dmar_atsr *tmp;
3965
3966 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3967 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3968 if (atsr->segment != tmp->segment)
3969 continue;
3970 if (atsr->header.length != tmp->header.length)
3971 continue;
3972 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3973 return atsru;
3974 }
3975
3976 return NULL;
3977}
3978
3979int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 3980{
3981 struct acpi_dmar_atsr *atsr;
3982 struct dmar_atsr_unit *atsru;
3983
6b197249
JL		 3984 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3985 return 0;
3986
318fe7df 3987 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 3988 atsru = dmar_find_atsr(atsr);
3989 if (atsru)
3990 return 0;
3991
3992 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 3993 if (!atsru)
3994 return -ENOMEM;
3995
6b197249
JL		 3996 /*
3997 * If memory is allocated from slab by ACPI _DSM method, we need to
3998 * copy the memory content because the memory buffer will be freed
3999 * on return.
4000 */
4001 atsru->hdr = (void *)(atsru + 1);
4002 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4003 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4004 if (!atsru->include_all) {
4005 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4006 (void *)atsr + atsr->header.length,
4007 &atsru->devices_cnt);
4008 if (atsru->devices_cnt && atsru->devices == NULL) {
4009 kfree(atsru);
4010 return -ENOMEM;
4011 }
4012 }
318fe7df 4013
0e242612 4014 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4015
4016 return 0;
4017}
4018
9bdc531e
JL		 4019static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4020{
4021 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4022 kfree(atsru);
4023}
4024
6b197249
JL		 4025int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4026{
4027 struct acpi_dmar_atsr *atsr;
4028 struct dmar_atsr_unit *atsru;
4029
4030 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4031 atsru = dmar_find_atsr(atsr);
4032 if (atsru) {
4033 list_del_rcu(&atsru->list);
4034 synchronize_rcu();
4035 intel_iommu_free_atsr(atsru);
4036 }
4037
4038 return 0;
4039}
4040
4041int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4042{
4043 int i;
4044 struct device *dev;
4045 struct acpi_dmar_atsr *atsr;
4046 struct dmar_atsr_unit *atsru;
4047
4048 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4049 atsru = dmar_find_atsr(atsr);
4050 if (!atsru)
4051 return 0;
4052
4053 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4054 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4055 i, dev)
4056 return -EBUSY;
4057
4058 return 0;
4059}
4060
ffebeb46
JL		 4061static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4062{
4063 int sp, ret = 0;
4064 struct intel_iommu *iommu = dmaru->iommu;
4065
4066 if (g_iommus[iommu->seq_id])
4067 return 0;
4068
4069 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4070 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4071 iommu->name);
4072 return -ENXIO;
4073 }
4074 if (!ecap_sc_support(iommu->ecap) &&
4075 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4076 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4077 iommu->name);
4078 return -ENXIO;
4079 }
4080 sp = domain_update_iommu_superpage(iommu) - 1;
4081 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4082 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4083 iommu->name);
4084 return -ENXIO;
4085 }
4086
4087 /*
4088 * Disable translation if already enabled prior to OS handover.
4089 */
4090 if (iommu->gcmd & DMA_GCMD_TE)
4091 iommu_disable_translation(iommu);
4092
4093 g_iommus[iommu->seq_id] = iommu;
4094 ret = iommu_init_domains(iommu);
4095 if (ret == 0)
4096 ret = iommu_alloc_root_entry(iommu);
4097 if (ret)
4098 goto out;
4099
4100 if (dmaru->ignored) {
4101 /*
4102 * we always have to disable PMRs or DMA may fail on this device
4103 */
4104 if (force_on)
4105 iommu_disable_protect_mem_regions(iommu);
4106 return 0;
4107 }
4108
4109 intel_iommu_init_qi(iommu);
4110 iommu_flush_write_buffer(iommu);
4111 ret = dmar_set_interrupt(iommu);
4112 if (ret)
4113 goto disable_iommu;
4114
4115 iommu_set_root_entry(iommu);
4116 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4117 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4118 iommu_enable_translation(iommu);
4119
ffebeb46
JL		 4120 iommu_disable_protect_mem_regions(iommu);
4121 return 0;
4122
4123disable_iommu:
4124 disable_dmar_iommu(iommu);
4125out:
4126 free_dmar_iommu(iommu);
4127 return ret;
4128}
4129
6b197249
JL		 4130int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4131{
ffebeb46
JL		 4132 int ret = 0;
4133 struct intel_iommu *iommu = dmaru->iommu;
4134
4135 if (!intel_iommu_enabled)
4136 return 0;
4137 if (iommu == NULL)
4138 return -EINVAL;
4139
4140 if (insert) {
4141 ret = intel_iommu_add(dmaru);
4142 } else {
4143 disable_dmar_iommu(iommu);
4144 free_dmar_iommu(iommu);
4145 }
4146
4147 return ret;
6b197249
JL		 4148}
4149
9bdc531e
JL		 4150static void intel_iommu_free_dmars(void)
4151{
4152 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4153 struct dmar_atsr_unit *atsru, *atsr_n;
4154
4155 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4156 list_del(&rmrru->list);
4157 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4158 kfree(rmrru);
318fe7df
SS		 4159 }
4160
9bdc531e
JL		 4161 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4162 list_del(&atsru->list);
4163 intel_iommu_free_atsr(atsru);
4164 }
318fe7df
SS		 4165}
4166
4167int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4168{
b683b230 4169 int i, ret = 1;
318fe7df 4170 struct pci_bus *bus;
832bd858
DW		 4171 struct pci_dev *bridge = NULL;
4172 struct device *tmp;
318fe7df
SS		 4173 struct acpi_dmar_atsr *atsr;
4174 struct dmar_atsr_unit *atsru;
4175
4176 dev = pci_physfn(dev);
318fe7df 4177 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4178 bridge = bus->self;
318fe7df 4179 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4180 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4181 return 0;
b5f82ddf 4182 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4183 break;
318fe7df 4184 }
b5f82ddf
JL		 4185 if (!bridge)
4186 return 0;
318fe7df 4187
0e242612 4188 rcu_read_lock();
b5f82ddf
JL		 4189 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4190 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4191 if (atsr->segment != pci_domain_nr(dev->bus))
4192 continue;
4193
b683b230 4194 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4195 if (tmp == &bridge->dev)
b683b230 4196 goto out;
b5f82ddf
JL		 4197
4198 if (atsru->include_all)
b683b230 4199 goto out;
b5f82ddf 4200 }
b683b230
JL		 4201 ret = 0;
4202out:
0e242612 4203 rcu_read_unlock();
318fe7df 4204
b683b230 4205 return ret;
318fe7df
SS		 4206}
4207
59ce0515
JL		 4208int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4209{
4210 int ret = 0;
4211 struct dmar_rmrr_unit *rmrru;
4212 struct dmar_atsr_unit *atsru;
4213 struct acpi_dmar_atsr *atsr;
4214 struct acpi_dmar_reserved_memory *rmrr;
4215
4216 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4217 return 0;
4218
4219 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4220 rmrr = container_of(rmrru->hdr,
4221 struct acpi_dmar_reserved_memory, header);
4222 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4223 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4224 ((void *)rmrr) + rmrr->header.length,
4225 rmrr->segment, rmrru->devices,
4226 rmrru->devices_cnt);
27e24950 4227 if(ret < 0)
59ce0515
JL		 4228 return ret;
4229 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4230 dmar_remove_dev_scope(info, rmrr->segment,
4231 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4232 }
4233 }
4234
4235 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4236 if (atsru->include_all)
4237 continue;
4238
4239 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4240 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4241 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4242 (void *)atsr + atsr->header.length,
4243 atsr->segment, atsru->devices,
4244 atsru->devices_cnt);
4245 if (ret > 0)
4246 break;
4247 else if(ret < 0)
4248 return ret;
4249 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4250 if (dmar_remove_dev_scope(info, atsr->segment,
4251 atsru->devices, atsru->devices_cnt))
4252 break;
4253 }
4254 }
4255
4256 return 0;
4257}
4258
99dcaded
FY		 4259/*
4260 * Here we only respond to action of unbound device from driver.
4261 *
4262 * Added device is not attached to its DMAR domain here yet. That will happen
4263 * when mapping the device to iova.
4264 */
4265static int device_notifier(struct notifier_block *nb,
4266 unsigned long action, void *data)
4267{
4268 struct device *dev = data;
99dcaded
FY		 4269 struct dmar_domain *domain;
4270
3d89194a 4271 if (iommu_dummy(dev))
44cd613c
DW		 4272 return 0;
4273
1196c2fb 4274 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4275 return 0;
4276
1525a29a 4277 domain = find_domain(dev);
99dcaded
FY		 4278 if (!domain)
4279 return 0;
4280
e6de0f8d 4281 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4282 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4283 domain_exit(domain);
a97590e5 4284
99dcaded
FY		 4285 return 0;
4286}
4287
4288static struct notifier_block device_nb = {
4289 .notifier_call = device_notifier,
4290};
4291
75f05569
JL		 4292static int intel_iommu_memory_notifier(struct notifier_block *nb,
4293 unsigned long val, void *v)
4294{
4295 struct memory_notify *mhp = v;
4296 unsigned long long start, end;
4297 unsigned long start_vpfn, last_vpfn;
4298
4299 switch (val) {
4300 case MEM_GOING_ONLINE:
4301 start = mhp->start_pfn << PAGE_SHIFT;
4302 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4303 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4304 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4305 start, end);
4306 return NOTIFY_BAD;
4307 }
4308 break;
4309
4310 case MEM_OFFLINE:
4311 case MEM_CANCEL_ONLINE:
4312 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4313 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4314 while (start_vpfn <= last_vpfn) {
4315 struct iova *iova;
4316 struct dmar_drhd_unit *drhd;
4317 struct intel_iommu *iommu;
ea8ea460 4318 struct page *freelist;
75f05569
JL		 4319
4320 iova = find_iova(&si_domain->iovad, start_vpfn);
4321 if (iova == NULL) {
9f10e5bf 4322 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4323 start_vpfn);
4324 break;
4325 }
4326
4327 iova = split_and_remove_iova(&si_domain->iovad, iova,
4328 start_vpfn, last_vpfn);
4329 if (iova == NULL) {
9f10e5bf 4330 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4331 start_vpfn, last_vpfn);
4332 return NOTIFY_BAD;
4333 }
4334
ea8ea460
DW		 4335 freelist = domain_unmap(si_domain, iova->pfn_lo,
4336 iova->pfn_hi);
4337
75f05569
JL		 4338 rcu_read_lock();
4339 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4340 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4341 iova->pfn_lo, iova_size(iova),
ea8ea460 4342 !freelist, 0);
75f05569 4343 rcu_read_unlock();
ea8ea460 4344 dma_free_pagelist(freelist);
75f05569
JL		 4345
4346 start_vpfn = iova->pfn_hi + 1;
4347 free_iova_mem(iova);
4348 }
4349 break;
4350 }
4351
4352 return NOTIFY_OK;
4353}
4354
4355static struct notifier_block intel_iommu_memory_nb = {
4356 .notifier_call = intel_iommu_memory_notifier,
4357 .priority = 0
4358};
4359
a5459cfe
AW		 4360
4361static ssize_t intel_iommu_show_version(struct device *dev,
4362 struct device_attribute *attr,
4363 char *buf)
4364{
4365 struct intel_iommu *iommu = dev_get_drvdata(dev);
4366 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4367 return sprintf(buf, "%d:%d\n",
4368 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4369}
4370static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4371
4372static ssize_t intel_iommu_show_address(struct device *dev,
4373 struct device_attribute *attr,
4374 char *buf)
4375{
4376 struct intel_iommu *iommu = dev_get_drvdata(dev);
4377 return sprintf(buf, "%llx\n", iommu->reg_phys);
4378}
4379static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4380
4381static ssize_t intel_iommu_show_cap(struct device *dev,
4382 struct device_attribute *attr,
4383 char *buf)
4384{
4385 struct intel_iommu *iommu = dev_get_drvdata(dev);
4386 return sprintf(buf, "%llx\n", iommu->cap);
4387}
4388static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4389
4390static ssize_t intel_iommu_show_ecap(struct device *dev,
4391 struct device_attribute *attr,
4392 char *buf)
4393{
4394 struct intel_iommu *iommu = dev_get_drvdata(dev);
4395 return sprintf(buf, "%llx\n", iommu->ecap);
4396}
4397static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4398
2238c082
AW		 4399static ssize_t intel_iommu_show_ndoms(struct device *dev,
4400 struct device_attribute *attr,
4401 char *buf)
4402{
4403 struct intel_iommu *iommu = dev_get_drvdata(dev);
4404 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4405}
4406static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4407
4408static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4409 struct device_attribute *attr,
4410 char *buf)
4411{
4412 struct intel_iommu *iommu = dev_get_drvdata(dev);
4413 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4414 cap_ndoms(iommu->cap)));
4415}
4416static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4417
a5459cfe
AW		 4418static struct attribute *intel_iommu_attrs[] = {
4419 &dev_attr_version.attr,
4420 &dev_attr_address.attr,
4421 &dev_attr_cap.attr,
4422 &dev_attr_ecap.attr,
2238c082
AW		 4423 &dev_attr_domains_supported.attr,
4424 &dev_attr_domains_used.attr,
a5459cfe
AW		 4425 NULL,
4426};
4427
4428static struct attribute_group intel_iommu_group = {
4429 .name = "intel-iommu",
4430 .attrs = intel_iommu_attrs,
4431};
4432
4433const struct attribute_group *intel_iommu_groups[] = {
4434 &intel_iommu_group,
4435 NULL,
4436};
4437
ba395927
KA		 4438int __init intel_iommu_init(void)
4439{
9bdc531e 4440 int ret = -ENODEV;
3a93c841 4441 struct dmar_drhd_unit *drhd;
7c919779 4442 struct intel_iommu *iommu;
ba395927 4443
a59b50e9
JC		 4444 /* VT-d is required for a TXT/tboot launch, so enforce that */
4445 force_on = tboot_force_iommu();
4446
3a5670e8
JL		 4447 if (iommu_init_mempool()) {
4448 if (force_on)
4449 panic("tboot: Failed to initialize iommu memory\n");
4450 return -ENOMEM;
4451 }
4452
4453 down_write(&dmar_global_lock);
a59b50e9
JC		 4454 if (dmar_table_init()) {
4455 if (force_on)
4456 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4457 goto out_free_dmar;
a59b50e9 4458 }
ba395927 4459
c2c7286a 4460 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4461 if (force_on)
4462 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4463 goto out_free_dmar;
a59b50e9 4464 }
1886e8a9 4465
75f1cdf1 4466 if (no_iommu || dmar_disabled)
9bdc531e 4467 goto out_free_dmar;
2ae21010 4468
318fe7df 4469 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4470 pr_info("No RMRR found\n");
318fe7df
SS		 4471
4472 if (list_empty(&dmar_atsr_units))
9f10e5bf 4473 pr_info("No ATSR found\n");
318fe7df 4474
51a63e67
JC		 4475 if (dmar_init_reserved_ranges()) {
4476 if (force_on)
4477 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4478 goto out_free_reserved_range;
51a63e67 4479 }
ba395927
KA		 4480
4481 init_no_remapping_devices();
4482
b779260b 4483 ret = init_dmars();
ba395927 4484 if (ret) {
a59b50e9
JC		 4485 if (force_on)
4486 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4487 pr_err("Initialization failed\n");
9bdc531e 4488 goto out_free_reserved_range;
ba395927 4489 }
3a5670e8 4490 up_write(&dmar_global_lock);
9f10e5bf 4491 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4492
5e0d2a6f 4493 init_timer(&unmap_timer);
75f1cdf1
FT		 4494#ifdef CONFIG_SWIOTLB
4495 swiotlb = 0;
4496#endif
19943b0e 4497 dma_ops = &intel_dma_ops;
4ed0d3e6 4498
134fac3f 4499 init_iommu_pm_ops();
a8bcbb0d 4500
a5459cfe
AW		 4501 for_each_active_iommu(iommu, drhd)
4502 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4503 intel_iommu_groups,
2439d4aa 4504 "%s", iommu->name);
a5459cfe 4505
4236d97d 4506 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4507 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4508 if (si_domain && !hw_pass_through)
4509 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4510
8bc1f85c
ED		 4511 intel_iommu_enabled = 1;
4512
ba395927 4513 return 0;
9bdc531e
JL		 4514
4515out_free_reserved_range:
4516 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4517out_free_dmar:
4518 intel_iommu_free_dmars();
3a5670e8
JL		 4519 up_write(&dmar_global_lock);
4520 iommu_exit_mempool();
9bdc531e 4521 return ret;
ba395927 4522}
e820482c 4523
2452d9db 4524static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
579305f7
AW		 4525{
4526 struct intel_iommu *iommu = opaque;
4527
2452d9db 4528 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 4529 return 0;
4530}
4531
4532/*
4533 * NB - intel-iommu lacks any sort of reference counting for the users of
4534 * dependent devices. If multiple endpoints have intersecting dependent
4535 * devices, unbinding the driver from any one of them will possibly leave
4536 * the others unable to operate.
4537 */
2452d9db 4538static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
3199aa6b 4539{
0bcb3e28 4540 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4541 return;
4542
2452d9db 4543 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
3199aa6b
HW		 4544}
4545
127c7615 4546static void __dmar_remove_one_dev_info(struct device_domain_info *info)
c7151a8d 4547{
c7151a8d
WH		 4548 struct intel_iommu *iommu;
4549 unsigned long flags;
c7151a8d 4550
55d94043
JR		 4551 assert_spin_locked(&device_domain_lock);
4552
127c7615 4553 if (WARN_ON(!info))
c7151a8d
WH		 4554 return;
4555
127c7615 4556 iommu = info->iommu;
c7151a8d 4557
127c7615
JR		 4558 if (info->dev) {
4559 iommu_disable_dev_iotlb(info);
4560 domain_context_clear(iommu, info->dev);
4561 }
c7151a8d 4562
b608ac3b 4563 unlink_domain_info(info);
3e7abe25 4564
d160aca5 4565 spin_lock_irqsave(&iommu->lock, flags);
127c7615 4566 domain_detach_iommu(info->domain, iommu);
d160aca5 4567 spin_unlock_irqrestore(&iommu->lock, flags);
127c7615
JR		 4568
4569 free_devinfo_mem(info);
c7151a8d
WH		 4570}
4571
55d94043
JR		 4572static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4573 struct device *dev)
4574{
127c7615 4575 struct device_domain_info *info;
55d94043
JR		 4576 unsigned long flags;
4577
4578 spin_lock_irqsave(&device_domain_lock, flags);
127c7615
JR		 4579 info = dev->archdata.iommu;
4580 __dmar_remove_one_dev_info(info);
55d94043
JR		 4581 spin_unlock_irqrestore(&device_domain_lock, flags);
4582}
4583
2c2e2c38 4584static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4585{
4586 int adjust_width;
4587
0fb5fe87
RM		 4588 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4589 DMA_32BIT_PFN);
5e98c4b1
WH		 4590 domain_reserve_special_ranges(domain);
4591
4592 /* calculate AGAW */
4593 domain->gaw = guest_width;
4594 adjust_width = guestwidth_to_adjustwidth(guest_width);
4595 domain->agaw = width_to_agaw(adjust_width);
4596
5e98c4b1 4597 domain->iommu_coherency = 0;
c5b15255 4598 domain->iommu_snooping = 0;
6dd9a7c7 4599 domain->iommu_superpage = 0;
fe40f1e0 4600 domain->max_addr = 0;
5e98c4b1
WH		 4601
4602 /* always allocate the top pgd */
4c923d47 4603 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4604 if (!domain->pgd)
4605 return -ENOMEM;
4606 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4607 return 0;
4608}
4609
00a77deb 4610static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4611{
5d450806 4612 struct dmar_domain *dmar_domain;
00a77deb
JR		 4613 struct iommu_domain *domain;
4614
4615 if (type != IOMMU_DOMAIN_UNMANAGED)
4616 return NULL;
38717946 4617
ab8dfe25 4618 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4619 if (!dmar_domain) {
9f10e5bf 4620 pr_err("Can't allocate dmar_domain\n");
00a77deb 4621 return NULL;
38717946 4622 }
2c2e2c38 4623 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4624 pr_err("Domain initialization failed\n");
92d03cc8 4625 domain_exit(dmar_domain);
00a77deb 4626 return NULL;
38717946 4627 }
8140a95d 4628 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4629
00a77deb 4630 domain = &dmar_domain->domain;
8a0e715b
JR		 4631 domain->geometry.aperture_start = 0;
4632 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4633 domain->geometry.force_aperture = true;
4634
00a77deb 4635 return domain;
38717946 4636}
38717946 4637
00a77deb 4638static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4639{
00a77deb 4640 domain_exit(to_dmar_domain(domain));
38717946 4641}
38717946 4642
4c5478c9
JR		 4643static int intel_iommu_attach_device(struct iommu_domain *domain,
4644 struct device *dev)
38717946 4645{
00a77deb 4646 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4647 struct intel_iommu *iommu;
4648 int addr_width;
156baca8 4649 u8 bus, devfn;
faa3d6f5 4650
c875d2c1
AW		 4651 if (device_is_rmrr_locked(dev)) {
4652 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4653 return -EPERM;
4654 }
4655
7207d8f9
DW		 4656 /* normally dev is not mapped */
4657 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4658 struct dmar_domain *old_domain;
4659
1525a29a 4660 old_domain = find_domain(dev);
faa3d6f5 4661 if (old_domain) {
d160aca5 4662 rcu_read_lock();
de7e8886 4663 dmar_remove_one_dev_info(old_domain, dev);
d160aca5 4664 rcu_read_unlock();
62c22167
JR		 4665
4666 if (!domain_type_is_vm_or_si(old_domain) &&
4667 list_empty(&old_domain->devices))
4668 domain_exit(old_domain);
faa3d6f5
WH		 4669 }
4670 }
4671
156baca8 4672 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4673 if (!iommu)
4674 return -ENODEV;
4675
4676 /* check if this iommu agaw is sufficient for max mapped address */
4677 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4678 if (addr_width > cap_mgaw(iommu->cap))
4679 addr_width = cap_mgaw(iommu->cap);
4680
4681 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4682 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4683 "sufficient for the mapped address (%llx)\n",
a99c47a2 4684 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4685 return -EFAULT;
4686 }
a99c47a2
TL		 4687 dmar_domain->gaw = addr_width;
4688
4689 /*
4690 * Knock out extra levels of page tables if necessary
4691 */
4692 while (iommu->agaw < dmar_domain->agaw) {
4693 struct dma_pte *pte;
4694
4695 pte = dmar_domain->pgd;
4696 if (dma_pte_present(pte)) {
25cbff16
SY		 4697 dmar_domain->pgd = (struct dma_pte *)
4698 phys_to_virt(dma_pte_addr(pte));
7a661013 4699 free_pgtable_page(pte);
a99c47a2
TL		 4700 }
4701 dmar_domain->agaw--;
4702 }
fe40f1e0 4703
28ccce0d 4704 return domain_add_dev_info(dmar_domain, dev);
38717946 4705}
38717946 4706
4c5478c9
JR		 4707static void intel_iommu_detach_device(struct iommu_domain *domain,
4708 struct device *dev)
38717946 4709{
e6de0f8d 4710 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4711}
c7151a8d 4712
b146a1c9
JR		 4713static int intel_iommu_map(struct iommu_domain *domain,
4714 unsigned long iova, phys_addr_t hpa,
5009065d 4715 size_t size, int iommu_prot)
faa3d6f5 4716{
00a77deb 4717 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4718 u64 max_addr;
dde57a21 4719 int prot = 0;
faa3d6f5 4720 int ret;
fe40f1e0 4721
dde57a21
JR		 4722 if (iommu_prot & IOMMU_READ)
4723 prot |= DMA_PTE_READ;
4724 if (iommu_prot & IOMMU_WRITE)
4725 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4726 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4727 prot |= DMA_PTE_SNP;
dde57a21 4728
163cc52c 4729 max_addr = iova + size;
dde57a21 4730 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4731 u64 end;
4732
4733 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4734 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4735 if (end < max_addr) {
9f10e5bf 4736 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4737 "sufficient for the mapped address (%llx)\n",
8954da1f 4738 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4739 return -EFAULT;
4740 }
dde57a21 4741 dmar_domain->max_addr = max_addr;
fe40f1e0 4742 }
ad051221
DW		 4743 /* Round up size to next multiple of PAGE_SIZE, if it and
4744 the low bits of hpa would take us onto the next page */
88cb6a74 4745 size = aligned_nrpages(hpa, size);
ad051221
DW		 4746 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4747 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4748 return ret;
38717946 4749}
38717946 4750
5009065d 4751static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4752 unsigned long iova, size_t size)
38717946 4753{
00a77deb 4754 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4755 struct page *freelist = NULL;
4756 struct intel_iommu *iommu;
4757 unsigned long start_pfn, last_pfn;
4758 unsigned int npages;
42e8c186 4759 int iommu_id, level = 0;
5cf0a76f
DW		 4760
4761 /* Cope with horrid API which requires us to unmap more than the
4762 size argument if it happens to be a large-page mapping. */
4763 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4764 BUG();
4765
4766 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4767 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4768
ea8ea460
DW		 4769 start_pfn = iova >> VTD_PAGE_SHIFT;
4770 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4771
4772 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4773
4774 npages = last_pfn - start_pfn + 1;
4775
29a27719 4776 for_each_domain_iommu(iommu_id, dmar_domain) {
a1ddcbe9
JR		 4777 iommu = g_iommus[iommu_id];
4778
42e8c186
JR		 4779 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4780 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 4781 }
4782
4783 dma_free_pagelist(freelist);
fe40f1e0 4784
163cc52c
DW		 4785 if (dmar_domain->max_addr == iova + size)
4786 dmar_domain->max_addr = iova;
b146a1c9 4787
5cf0a76f 4788 return size;
38717946 4789}
38717946 4790
d14d6577 4791static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4792 dma_addr_t iova)
38717946 4793{
00a77deb 4794 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4795 struct dma_pte *pte;
5cf0a76f 4796 int level = 0;
faa3d6f5 4797 u64 phys = 0;
38717946 4798
5cf0a76f 4799 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4800 if (pte)
faa3d6f5 4801 phys = dma_pte_addr(pte);
38717946 4802
faa3d6f5 4803 return phys;
38717946 4804}
a8bcbb0d 4805
5d587b8d 4806static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4807{
dbb9fd86 4808 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4809 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4810 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4811 return irq_remapping_enabled == 1;
dbb9fd86 4812
5d587b8d 4813 return false;
dbb9fd86
SY		 4814}
4815
abdfdde2
AW		 4816static int intel_iommu_add_device(struct device *dev)
4817{
a5459cfe 4818 struct intel_iommu *iommu;
abdfdde2 4819 struct iommu_group *group;
156baca8 4820 u8 bus, devfn;
70ae6f0d 4821
a5459cfe
AW		 4822 iommu = device_to_iommu(dev, &bus, &devfn);
4823 if (!iommu)
70ae6f0d
AW		 4824 return -ENODEV;
4825
a5459cfe 4826 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4827
e17f9ff4 4828 group = iommu_group_get_for_dev(dev);
783f157b 4829
e17f9ff4
AW		 4830 if (IS_ERR(group))
4831 return PTR_ERR(group);
bcb71abe 4832
abdfdde2 4833 iommu_group_put(group);
e17f9ff4 4834 return 0;
abdfdde2 4835}
70ae6f0d 4836
abdfdde2
AW		 4837static void intel_iommu_remove_device(struct device *dev)
4838{
a5459cfe
AW		 4839 struct intel_iommu *iommu;
4840 u8 bus, devfn;
4841
4842 iommu = device_to_iommu(dev, &bus, &devfn);
4843 if (!iommu)
4844 return;
4845
abdfdde2 4846 iommu_group_remove_device(dev);
a5459cfe
AW		 4847
4848 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4849}
4850
b22f6434 4851static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4852 .capable = intel_iommu_capable,
00a77deb
JR		 4853 .domain_alloc = intel_iommu_domain_alloc,
4854 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4855 .attach_dev = intel_iommu_attach_device,
4856 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4857 .map = intel_iommu_map,
4858 .unmap = intel_iommu_unmap,
315786eb 4859 .map_sg = default_iommu_map_sg,
a8bcbb0d 4860 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4861 .add_device = intel_iommu_add_device,
4862 .remove_device = intel_iommu_remove_device,
6d1c56a9 4863 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4864};
9af88143 4865
9452618e
DV		 4866static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4867{
4868 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4869 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4870 dmar_map_gfx = 0;
4871}
4872
4873DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4874DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4875DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4876DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4877DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4878DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4879DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4880
d34d6517 4881static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4882{
4883 /*
4884 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4885 * but needs it. Same seems to hold for the desktop versions.
9af88143 4886 */
9f10e5bf 4887 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4888 rwbf_quirk = 1;
4889}
4890
4891DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4892DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4893DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4894DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4895DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4896DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4897DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4898
eecfd57f
AJ		 4899#define GGC 0x52
4900#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4901#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4902#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4903#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4904#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4905#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4906#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4907#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4908
d34d6517 4909static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4910{
4911 unsigned short ggc;
4912
eecfd57f 4913 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4914 return;
4915
eecfd57f 4916 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4917 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4918 dmar_map_gfx = 0;
6fbcfb3e
DW		 4919 } else if (dmar_map_gfx) {
4920 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4921 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4922 intel_iommu_strict = 1;
4923 }
9eecabcb
DW		 4924}
4925DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4926DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4927DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4928DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4929
e0fc7e0b
DW		 4930/* On Tylersburg chipsets, some BIOSes have been known to enable the
4931 ISOCH DMAR unit for the Azalia sound device, but not give it any
4932 TLB entries, which causes it to deadlock. Check for that. We do
4933 this in a function called from init_dmars(), instead of in a PCI
4934 quirk, because we don't want to print the obnoxious "BIOS broken"
4935 message if VT-d is actually disabled.
4936*/
4937static void __init check_tylersburg_isoch(void)
4938{
4939 struct pci_dev *pdev;
4940 uint32_t vtisochctrl;
4941
4942 /* If there's no Azalia in the system anyway, forget it. */
4943 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4944 if (!pdev)
4945 return;
4946 pci_dev_put(pdev);
4947
4948 /* System Management Registers. Might be hidden, in which case
4949 we can't do the sanity check. But that's OK, because the
4950 known-broken BIOSes _don't_ actually hide it, so far. */
4951 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4952 if (!pdev)
4953 return;
4954
4955 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4956 pci_dev_put(pdev);
4957 return;
4958 }
4959
4960 pci_dev_put(pdev);
4961
4962 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4963 if (vtisochctrl & 1)
4964 return;
4965
4966 /* Drop all bits other than the number of TLB entries */
4967 vtisochctrl &= 0x1c;
4968
4969 /* If we have the recommended number of TLB entries (16), fine. */
4970 if (vtisochctrl == 0x10)
4971 return;
4972
4973 /* Zero TLB entries? You get to ride the short bus to school. */
4974 if (!vtisochctrl) {
4975 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4976 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4977 dmi_get_system_info(DMI_BIOS_VENDOR),
4978 dmi_get_system_info(DMI_BIOS_VERSION),
4979 dmi_get_system_info(DMI_PRODUCT_VERSION));
4980 iommu_identity_mapping |= IDENTMAP_AZALIA;
4981 return;
4982 }
9f10e5bf
JR		 4983
4984 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 4985 vtisochctrl);
4986}
Linux 6.x block layer and io_uring tree(s)