projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 4.2-rc3
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
1ce28feb
WH		 367/* domain represents a virtual machine, more than one devices
368 * across iommus may be owned in one domain, e.g. kvm guest.
369 */
ab8dfe25 370#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 371
2c2e2c38 372/* si_domain contains mulitple devices */
ab8dfe25 373#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 374
99126f7c
MM		 375struct dmar_domain {
376 int id; /* domain id */
4c923d47 377 int nid; /* node id */
78d8e704 378 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 379 /* bitmap of iommus this domain uses*/
99126f7c 380
00a77deb 381 struct list_head devices; /* all devices' list */
99126f7c
MM		 382 struct iova_domain iovad; /* iova's that belong to this domain */
383
384 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 385 int gaw; /* max guest address width */
386
387 /* adjusted guest address width, 0 is level 2 30-bit */
388 int agaw;
389
3b5410e7 390 int flags; /* flags to find out type of domain */
8e604097
WH		 391
392 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 393 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 394 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 395 int iommu_superpage;/* Level of superpages supported:
396 0 == 4KiB (no superpages), 1 == 2MiB,
397 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 398 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 399 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 400
401 struct iommu_domain domain; /* generic domain data structure for
402 iommu core */
99126f7c
MM		 403};
404
a647dacb
MM		 405/* PCI domain-device relationship */
406struct device_domain_info {
407 struct list_head link; /* link to domain siblings */
408 struct list_head global; /* link to global list */
276dbf99 409 u8 bus; /* PCI bus number */
a647dacb 410 u8 devfn; /* PCI devfn number */
0bcb3e28 411 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 412 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 413 struct dmar_domain *domain; /* pointer to domain */
414};
415
b94e4117
JL		 416struct dmar_rmrr_unit {
417 struct list_head list; /* list of rmrr units */
418 struct acpi_dmar_header *hdr; /* ACPI header */
419 u64 base_address; /* reserved base address*/
420 u64 end_address; /* reserved end address */
832bd858 421 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 422 int devices_cnt; /* target device count */
423};
424
425struct dmar_atsr_unit {
426 struct list_head list; /* list of ATSR units */
427 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 428 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 429 int devices_cnt; /* target device count */
430 u8 include_all:1; /* include all ports */
431};
432
433static LIST_HEAD(dmar_atsr_units);
434static LIST_HEAD(dmar_rmrr_units);
435
436#define for_each_rmrr_units(rmrr) \
437 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
438
5e0d2a6f 439static void flush_unmaps_timeout(unsigned long data);
440
b707cb02 441static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 442
80b20dd8 443#define HIGH_WATER_MARK 250
444struct deferred_flush_tables {
445 int next;
446 struct iova *iova[HIGH_WATER_MARK];
447 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 448 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 449};
450
451static struct deferred_flush_tables *deferred_flush;
452
5e0d2a6f 453/* bitmap for indexing intel_iommus */
5e0d2a6f 454static int g_num_of_iommus;
455
456static DEFINE_SPINLOCK(async_umap_flush_lock);
457static LIST_HEAD(unmaps_to_do);
458
459static int timer_on;
460static long list_size;
5e0d2a6f 461
92d03cc8 462static void domain_exit(struct dmar_domain *domain);
ba395927 463static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 464static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 465 struct device *dev);
92d03cc8 466static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 467 struct device *dev);
2a46ddf7
JL		 468static int domain_detach_iommu(struct dmar_domain *domain,
469 struct intel_iommu *iommu);
ba395927 470
d3f13810 471#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 472int dmar_disabled = 0;
473#else
474int dmar_disabled = 1;
d3f13810 475#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 476
8bc1f85c
ED		 477int intel_iommu_enabled = 0;
478EXPORT_SYMBOL_GPL(intel_iommu_enabled);
479
2d9e667e 480static int dmar_map_gfx = 1;
7d3b03ce 481static int dmar_forcedac;
5e0d2a6f 482static int intel_iommu_strict;
6dd9a7c7 483static int intel_iommu_superpage = 1;
c83b2f20
DW		 484static int intel_iommu_ecs = 1;
485
486/* We only actually use ECS when PASID support (on the new bit 40)
487 * is also advertised. Some early implementations — the ones with
488 * PASID support on bit 28 — have issues even when we *only* use
489 * extended root/context tables. */
490#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
491 ecap_pasid(iommu->ecap))
ba395927 492
c0771df8
DW		 493int intel_iommu_gfx_mapped;
494EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
495
ba395927
KA		 496#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
497static DEFINE_SPINLOCK(device_domain_lock);
498static LIST_HEAD(device_domain_list);
499
b22f6434 500static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 501
4158c2ec
JR		 502static bool translation_pre_enabled(struct intel_iommu *iommu)
503{
504 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
505}
506
091d42e4
JR		 507static void clear_translation_pre_enabled(struct intel_iommu *iommu)
508{
509 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
510}
511
4158c2ec
JR		 512static void init_translation_status(struct intel_iommu *iommu)
513{
514 u32 gsts;
515
516 gsts = readl(iommu->reg + DMAR_GSTS_REG);
517 if (gsts & DMA_GSTS_TES)
518 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
519}
520
00a77deb
JR		 521/* Convert generic 'struct iommu_domain to private struct dmar_domain */
522static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
523{
524 return container_of(dom, struct dmar_domain, domain);
525}
526
ba395927
KA		 527static int __init intel_iommu_setup(char *str)
528{
529 if (!str)
530 return -EINVAL;
531 while (*str) {
0cd5c3c8
KM		 532 if (!strncmp(str, "on", 2)) {
533 dmar_disabled = 0;
9f10e5bf 534 pr_info("IOMMU enabled\n");
0cd5c3c8 535 } else if (!strncmp(str, "off", 3)) {
ba395927 536 dmar_disabled = 1;
9f10e5bf 537 pr_info("IOMMU disabled\n");
ba395927
KA		 538 } else if (!strncmp(str, "igfx_off", 8)) {
539 dmar_map_gfx = 0;
9f10e5bf 540 pr_info("Disable GFX device mapping\n");
7d3b03ce 541 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 542 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 543 dmar_forcedac = 1;
5e0d2a6f 544 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 545 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 546 intel_iommu_strict = 1;
6dd9a7c7 547 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 548 pr_info("Disable supported super page\n");
6dd9a7c7 549 intel_iommu_superpage = 0;
c83b2f20
DW		 550 } else if (!strncmp(str, "ecs_off", 7)) {
551 printk(KERN_INFO
552 "Intel-IOMMU: disable extended context table support\n");
553 intel_iommu_ecs = 0;
ba395927
KA		 554 }
555
556 str += strcspn(str, ",");
557 while (*str == ',')
558 str++;
559 }
560 return 0;
561}
562__setup("intel_iommu=", intel_iommu_setup);
563
564static struct kmem_cache *iommu_domain_cache;
565static struct kmem_cache *iommu_devinfo_cache;
ba395927 566
4c923d47 567static inline void *alloc_pgtable_page(int node)
eb3fa7cb 568{
4c923d47
SS		 569 struct page *page;
570 void *vaddr = NULL;
eb3fa7cb 571
4c923d47
SS		 572 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
573 if (page)
574 vaddr = page_address(page);
eb3fa7cb 575 return vaddr;
ba395927
KA		 576}
577
578static inline void free_pgtable_page(void *vaddr)
579{
580 free_page((unsigned long)vaddr);
581}
582
583static inline void *alloc_domain_mem(void)
584{
354bb65e 585 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 586}
587
38717946 588static void free_domain_mem(void *vaddr)
ba395927
KA		 589{
590 kmem_cache_free(iommu_domain_cache, vaddr);
591}
592
593static inline void * alloc_devinfo_mem(void)
594{
354bb65e 595 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 596}
597
598static inline void free_devinfo_mem(void *vaddr)
599{
600 kmem_cache_free(iommu_devinfo_cache, vaddr);
601}
602
ab8dfe25
JL		 603static inline int domain_type_is_vm(struct dmar_domain *domain)
604{
605 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
606}
607
608static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
609{
610 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
611 DOMAIN_FLAG_STATIC_IDENTITY);
612}
1b573683 613
162d1b10
JL		 614static inline int domain_pfn_supported(struct dmar_domain *domain,
615 unsigned long pfn)
616{
617 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
618
619 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
620}
621
4ed0d3e6 622static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 623{
624 unsigned long sagaw;
625 int agaw = -1;
626
627 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 628 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 629 agaw >= 0; agaw--) {
630 if (test_bit(agaw, &sagaw))
631 break;
632 }
633
634 return agaw;
635}
636
4ed0d3e6
FY		 637/*
638 * Calculate max SAGAW for each iommu.
639 */
640int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
641{
642 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
643}
644
645/*
646 * calculate agaw for each iommu.
647 * "SAGAW" may be different across iommus, use a default agaw, and
648 * get a supported less agaw for iommus that don't support the default agaw.
649 */
650int iommu_calculate_agaw(struct intel_iommu *iommu)
651{
652 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
653}
654
2c2e2c38 655/* This functionin only returns single iommu in a domain */
8c11e798
WH		 656static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
657{
658 int iommu_id;
659
2c2e2c38 660 /* si_domain and vm domain should not get here. */
ab8dfe25 661 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 662 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 663 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
664 return NULL;
665
666 return g_iommus[iommu_id];
667}
668
8e604097
WH		 669static void domain_update_iommu_coherency(struct dmar_domain *domain)
670{
d0501960
DW		 671 struct dmar_drhd_unit *drhd;
672 struct intel_iommu *iommu;
2f119c78
QL		 673 bool found = false;
674 int i;
2e12bc29 675
d0501960 676 domain->iommu_coherency = 1;
8e604097 677
1b198bb0 678 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 679 found = true;
8e604097
WH		 680 if (!ecap_coherent(g_iommus[i]->ecap)) {
681 domain->iommu_coherency = 0;
682 break;
683 }
8e604097 684 }
d0501960
DW		 685 if (found)
686 return;
687
688 /* No hardware attached; use lowest common denominator */
689 rcu_read_lock();
690 for_each_active_iommu(iommu, drhd) {
691 if (!ecap_coherent(iommu->ecap)) {
692 domain->iommu_coherency = 0;
693 break;
694 }
695 }
696 rcu_read_unlock();
8e604097
WH		 697}
698
161f6934 699static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 700{
161f6934
JL		 701 struct dmar_drhd_unit *drhd;
702 struct intel_iommu *iommu;
703 int ret = 1;
58c610bd 704
161f6934
JL		 705 rcu_read_lock();
706 for_each_active_iommu(iommu, drhd) {
707 if (iommu != skip) {
708 if (!ecap_sc_support(iommu->ecap)) {
709 ret = 0;
710 break;
711 }
58c610bd 712 }
58c610bd 713 }
161f6934
JL		 714 rcu_read_unlock();
715
716 return ret;
58c610bd
SY		 717}
718
161f6934 719static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 720{
8140a95d 721 struct dmar_drhd_unit *drhd;
161f6934 722 struct intel_iommu *iommu;
8140a95d 723 int mask = 0xf;
6dd9a7c7
YS		 724
725 if (!intel_iommu_superpage) {
161f6934 726 return 0;
6dd9a7c7
YS		 727 }
728
8140a95d 729 /* set iommu_superpage to the smallest common denominator */
0e242612 730 rcu_read_lock();
8140a95d 731 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 732 if (iommu != skip) {
733 mask &= cap_super_page_val(iommu->cap);
734 if (!mask)
735 break;
6dd9a7c7
YS		 736 }
737 }
0e242612
JL		 738 rcu_read_unlock();
739
161f6934 740 return fls(mask);
6dd9a7c7
YS		 741}
742
58c610bd
SY		 743/* Some capabilities may be different across iommus */
744static void domain_update_iommu_cap(struct dmar_domain *domain)
745{
746 domain_update_iommu_coherency(domain);
161f6934
JL		 747 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
748 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 749}
750
03ecc32c
DW		 751static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
752 u8 bus, u8 devfn, int alloc)
753{
754 struct root_entry *root = &iommu->root_entry[bus];
755 struct context_entry *context;
756 u64 *entry;
757
c83b2f20 758 if (ecs_enabled(iommu)) {
03ecc32c
DW		 759 if (devfn >= 0x80) {
760 devfn -= 0x80;
761 entry = &root->hi;
762 }
763 devfn *= 2;
764 }
765 entry = &root->lo;
766 if (*entry & 1)
767 context = phys_to_virt(*entry & VTD_PAGE_MASK);
768 else {
769 unsigned long phy_addr;
770 if (!alloc)
771 return NULL;
772
773 context = alloc_pgtable_page(iommu->node);
774 if (!context)
775 return NULL;
776
777 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
778 phy_addr = virt_to_phys((void *)context);
779 *entry = phy_addr | 1;
780 __iommu_flush_cache(iommu, entry, sizeof(*entry));
781 }
782 return &context[devfn];
783}
784
4ed6a540
DW		 785static int iommu_dummy(struct device *dev)
786{
787 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
788}
789
156baca8 790static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 791{
792 struct dmar_drhd_unit *drhd = NULL;
b683b230 793 struct intel_iommu *iommu;
156baca8
DW		 794 struct device *tmp;
795 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 796 u16 segment = 0;
c7151a8d
WH		 797 int i;
798
4ed6a540
DW		 799 if (iommu_dummy(dev))
800 return NULL;
801
156baca8
DW		 802 if (dev_is_pci(dev)) {
803 pdev = to_pci_dev(dev);
804 segment = pci_domain_nr(pdev->bus);
ca5b74d2 805 } else if (has_acpi_companion(dev))
156baca8
DW		 806 dev = &ACPI_COMPANION(dev)->dev;
807
0e242612 808 rcu_read_lock();
b683b230 809 for_each_active_iommu(iommu, drhd) {
156baca8 810 if (pdev && segment != drhd->segment)
276dbf99 811 continue;
c7151a8d 812
b683b230 813 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 814 drhd->devices_cnt, i, tmp) {
815 if (tmp == dev) {
816 *bus = drhd->devices[i].bus;
817 *devfn = drhd->devices[i].devfn;
b683b230 818 goto out;
156baca8
DW		 819 }
820
821 if (!pdev || !dev_is_pci(tmp))
822 continue;
823
824 ptmp = to_pci_dev(tmp);
825 if (ptmp->subordinate &&
826 ptmp->subordinate->number <= pdev->bus->number &&
827 ptmp->subordinate->busn_res.end >= pdev->bus->number)
828 goto got_pdev;
924b6231 829 }
c7151a8d 830
156baca8
DW		 831 if (pdev && drhd->include_all) {
832 got_pdev:
833 *bus = pdev->bus->number;
834 *devfn = pdev->devfn;
b683b230 835 goto out;
156baca8 836 }
c7151a8d 837 }
b683b230 838 iommu = NULL;
156baca8 839 out:
0e242612 840 rcu_read_unlock();
c7151a8d 841
b683b230 842 return iommu;
c7151a8d
WH		 843}
844
5331fe6f
WH		 845static void domain_flush_cache(struct dmar_domain *domain,
846 void *addr, int size)
847{
848 if (!domain->iommu_coherency)
849 clflush_cache_range(addr, size);
850}
851
ba395927
KA		 852static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
853{
ba395927 854 struct context_entry *context;
03ecc32c 855 int ret = 0;
ba395927
KA		 856 unsigned long flags;
857
858 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 859 context = iommu_context_addr(iommu, bus, devfn, 0);
860 if (context)
861 ret = context_present(context);
ba395927
KA		 862 spin_unlock_irqrestore(&iommu->lock, flags);
863 return ret;
864}
865
866static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
867{
ba395927
KA		 868 struct context_entry *context;
869 unsigned long flags;
870
871 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 872 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 873 if (context) {
03ecc32c
DW		 874 context_clear_entry(context);
875 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 876 }
877 spin_unlock_irqrestore(&iommu->lock, flags);
878}
879
880static void free_context_table(struct intel_iommu *iommu)
881{
ba395927
KA		 882 int i;
883 unsigned long flags;
884 struct context_entry *context;
885
886 spin_lock_irqsave(&iommu->lock, flags);
887 if (!iommu->root_entry) {
888 goto out;
889 }
890 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 891 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 892 if (context)
893 free_pgtable_page(context);
03ecc32c 894
c83b2f20 895 if (!ecs_enabled(iommu))
03ecc32c
DW		 896 continue;
897
898 context = iommu_context_addr(iommu, i, 0x80, 0);
899 if (context)
900 free_pgtable_page(context);
901
ba395927
KA		 902 }
903 free_pgtable_page(iommu->root_entry);
904 iommu->root_entry = NULL;
905out:
906 spin_unlock_irqrestore(&iommu->lock, flags);
907}
908
b026fd28 909static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 910 unsigned long pfn, int *target_level)
ba395927 911{
ba395927
KA		 912 struct dma_pte *parent, *pte = NULL;
913 int level = agaw_to_level(domain->agaw);
4399c8bf 914 int offset;
ba395927
KA		 915
916 BUG_ON(!domain->pgd);
f9423606 917
162d1b10 918 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 919 /* Address beyond IOMMU's addressing capabilities. */
920 return NULL;
921
ba395927
KA		 922 parent = domain->pgd;
923
5cf0a76f 924 while (1) {
ba395927
KA		 925 void *tmp_page;
926
b026fd28 927 offset = pfn_level_offset(pfn, level);
ba395927 928 pte = &parent[offset];
5cf0a76f 929 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 930 break;
5cf0a76f 931 if (level == *target_level)
ba395927
KA		 932 break;
933
19c239ce 934 if (!dma_pte_present(pte)) {
c85994e4
DW		 935 uint64_t pteval;
936
4c923d47 937 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 938
206a73c1 939 if (!tmp_page)
ba395927 940 return NULL;
206a73c1 941
c85994e4 942 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 943 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 944 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 945 /* Someone else set it while we were thinking; use theirs. */
946 free_pgtable_page(tmp_page);
effad4b5 947 else
c85994e4 948 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 949 }
5cf0a76f
DW		 950 if (level == 1)
951 break;
952
19c239ce 953 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 954 level--;
955 }
956
5cf0a76f
DW		 957 if (!*target_level)
958 *target_level = level;
959
ba395927
KA		 960 return pte;
961}
962
6dd9a7c7 963
ba395927 964/* return address's pte at specific level */
90dcfb5e
DW		 965static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
966 unsigned long pfn,
6dd9a7c7 967 int level, int *large_page)
ba395927
KA		 968{
969 struct dma_pte *parent, *pte = NULL;
970 int total = agaw_to_level(domain->agaw);
971 int offset;
972
973 parent = domain->pgd;
974 while (level <= total) {
90dcfb5e 975 offset = pfn_level_offset(pfn, total);
ba395927
KA		 976 pte = &parent[offset];
977 if (level == total)
978 return pte;
979
6dd9a7c7
YS		 980 if (!dma_pte_present(pte)) {
981 *large_page = total;
ba395927 982 break;
6dd9a7c7
YS		 983 }
984
e16922af 985 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 986 *large_page = total;
987 return pte;
988 }
989
19c239ce 990 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 991 total--;
992 }
993 return NULL;
994}
995
ba395927 996/* clear last level pte, a tlb flush should be followed */
5cf0a76f 997static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 998 unsigned long start_pfn,
999 unsigned long last_pfn)
ba395927 1000{
6dd9a7c7 1001 unsigned int large_page = 1;
310a5ab9 1002 struct dma_pte *first_pte, *pte;
66eae846 1003
162d1b10
JL		 1004 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1005 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1006 BUG_ON(start_pfn > last_pfn);
ba395927 1007
04b18e65 1008 /* we don't need lock here; nobody else touches the iova range */
59c36286 1009 do {
6dd9a7c7
YS		 1010 large_page = 1;
1011 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1012 if (!pte) {
6dd9a7c7 1013 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1014 continue;
1015 }
6dd9a7c7 1016 do {
310a5ab9 1017 dma_clear_pte(pte);
6dd9a7c7 1018 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1019 pte++;
75e6bf96
DW		 1020 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1021
310a5ab9
DW		 1022 domain_flush_cache(domain, first_pte,
1023 (void *)pte - (void *)first_pte);
59c36286
DW		 1024
1025 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1026}
1027
3269ee0b
AW		 1028static void dma_pte_free_level(struct dmar_domain *domain, int level,
1029 struct dma_pte *pte, unsigned long pfn,
1030 unsigned long start_pfn, unsigned long last_pfn)
1031{
1032 pfn = max(start_pfn, pfn);
1033 pte = &pte[pfn_level_offset(pfn, level)];
1034
1035 do {
1036 unsigned long level_pfn;
1037 struct dma_pte *level_pte;
1038
1039 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1040 goto next;
1041
1042 level_pfn = pfn & level_mask(level - 1);
1043 level_pte = phys_to_virt(dma_pte_addr(pte));
1044
1045 if (level > 2)
1046 dma_pte_free_level(domain, level - 1, level_pte,
1047 level_pfn, start_pfn, last_pfn);
1048
1049 /* If range covers entire pagetable, free it */
1050 if (!(start_pfn > level_pfn ||
08336fd2 1051 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1052 dma_clear_pte(pte);
1053 domain_flush_cache(domain, pte, sizeof(*pte));
1054 free_pgtable_page(level_pte);
1055 }
1056next:
1057 pfn += level_size(level);
1058 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1059}
1060
ba395927
KA		 1061/* free page table pages. last level pte should already be cleared */
1062static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1063 unsigned long start_pfn,
1064 unsigned long last_pfn)
ba395927 1065{
162d1b10
JL		 1066 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1067 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1068 BUG_ON(start_pfn > last_pfn);
ba395927 1069
d41a4adb
JL		 1070 dma_pte_clear_range(domain, start_pfn, last_pfn);
1071
f3a0a52f 1072 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1073 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1074 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1075
ba395927 1076 /* free pgd */
d794dc9b 1077 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1078 free_pgtable_page(domain->pgd);
1079 domain->pgd = NULL;
1080 }
1081}
1082
ea8ea460
DW		 1083/* When a page at a given level is being unlinked from its parent, we don't
1084 need to *modify* it at all. All we need to do is make a list of all the
1085 pages which can be freed just as soon as we've flushed the IOTLB and we
1086 know the hardware page-walk will no longer touch them.
1087 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1088 be freed. */
1089static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1090 int level, struct dma_pte *pte,
1091 struct page *freelist)
1092{
1093 struct page *pg;
1094
1095 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1096 pg->freelist = freelist;
1097 freelist = pg;
1098
1099 if (level == 1)
1100 return freelist;
1101
adeb2590
JL		 1102 pte = page_address(pg);
1103 do {
ea8ea460
DW		 1104 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1105 freelist = dma_pte_list_pagetables(domain, level - 1,
1106 pte, freelist);
adeb2590
JL		 1107 pte++;
1108 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1109
1110 return freelist;
1111}
1112
1113static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1114 struct dma_pte *pte, unsigned long pfn,
1115 unsigned long start_pfn,
1116 unsigned long last_pfn,
1117 struct page *freelist)
1118{
1119 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1120
1121 pfn = max(start_pfn, pfn);
1122 pte = &pte[pfn_level_offset(pfn, level)];
1123
1124 do {
1125 unsigned long level_pfn;
1126
1127 if (!dma_pte_present(pte))
1128 goto next;
1129
1130 level_pfn = pfn & level_mask(level);
1131
1132 /* If range covers entire pagetable, free it */
1133 if (start_pfn <= level_pfn &&
1134 last_pfn >= level_pfn + level_size(level) - 1) {
1135 /* These suborbinate page tables are going away entirely. Don't
1136 bother to clear them; we're just going to *free* them. */
1137 if (level > 1 && !dma_pte_superpage(pte))
1138 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1139
1140 dma_clear_pte(pte);
1141 if (!first_pte)
1142 first_pte = pte;
1143 last_pte = pte;
1144 } else if (level > 1) {
1145 /* Recurse down into a level that isn't *entirely* obsolete */
1146 freelist = dma_pte_clear_level(domain, level - 1,
1147 phys_to_virt(dma_pte_addr(pte)),
1148 level_pfn, start_pfn, last_pfn,
1149 freelist);
1150 }
1151next:
1152 pfn += level_size(level);
1153 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1154
1155 if (first_pte)
1156 domain_flush_cache(domain, first_pte,
1157 (void *)++last_pte - (void *)first_pte);
1158
1159 return freelist;
1160}
1161
1162/* We can't just free the pages because the IOMMU may still be walking
1163 the page tables, and may have cached the intermediate levels. The
1164 pages can only be freed after the IOTLB flush has been done. */
1165struct page *domain_unmap(struct dmar_domain *domain,
1166 unsigned long start_pfn,
1167 unsigned long last_pfn)
1168{
ea8ea460
DW		 1169 struct page *freelist = NULL;
1170
162d1b10
JL		 1171 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1173 BUG_ON(start_pfn > last_pfn);
1174
1175 /* we don't need lock here; nobody else touches the iova range */
1176 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1177 domain->pgd, 0, start_pfn, last_pfn, NULL);
1178
1179 /* free pgd */
1180 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1181 struct page *pgd_page = virt_to_page(domain->pgd);
1182 pgd_page->freelist = freelist;
1183 freelist = pgd_page;
1184
1185 domain->pgd = NULL;
1186 }
1187
1188 return freelist;
1189}
1190
1191void dma_free_pagelist(struct page *freelist)
1192{
1193 struct page *pg;
1194
1195 while ((pg = freelist)) {
1196 freelist = pg->freelist;
1197 free_pgtable_page(page_address(pg));
1198 }
1199}
1200
ba395927
KA		 1201/* iommu handling */
1202static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1203{
1204 struct root_entry *root;
1205 unsigned long flags;
1206
4c923d47 1207 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1208 if (!root) {
9f10e5bf 1209 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1210 iommu->name);
ba395927 1211 return -ENOMEM;
ffebeb46 1212 }
ba395927 1213
5b6985ce 1214 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1215
1216 spin_lock_irqsave(&iommu->lock, flags);
1217 iommu->root_entry = root;
1218 spin_unlock_irqrestore(&iommu->lock, flags);
1219
1220 return 0;
1221}
1222
ba395927
KA		 1223static void iommu_set_root_entry(struct intel_iommu *iommu)
1224{
03ecc32c 1225 u64 addr;
c416daa9 1226 u32 sts;
ba395927
KA		 1227 unsigned long flag;
1228
03ecc32c 1229 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1230 if (ecs_enabled(iommu))
03ecc32c 1231 addr |= DMA_RTADDR_RTT;
ba395927 1232
1f5b3c3f 1233 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1234 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1235
c416daa9 1236 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1240 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1241
1f5b3c3f 1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1243}
1244
1245static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1246{
1247 u32 val;
1248 unsigned long flag;
1249
9af88143 1250 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1251 return;
ba395927 1252
1f5b3c3f 1253 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1254 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1255
1256 /* Make sure hardware complete it */
1257 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1258 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1259
1f5b3c3f 1260 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1261}
1262
1263/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1264static void __iommu_flush_context(struct intel_iommu *iommu,
1265 u16 did, u16 source_id, u8 function_mask,
1266 u64 type)
ba395927
KA		 1267{
1268 u64 val = 0;
1269 unsigned long flag;
1270
ba395927
KA		 1271 switch (type) {
1272 case DMA_CCMD_GLOBAL_INVL:
1273 val = DMA_CCMD_GLOBAL_INVL;
1274 break;
1275 case DMA_CCMD_DOMAIN_INVL:
1276 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1277 break;
1278 case DMA_CCMD_DEVICE_INVL:
1279 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1280 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1281 break;
1282 default:
1283 BUG();
1284 }
1285 val |= DMA_CCMD_ICC;
1286
1f5b3c3f 1287 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1288 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1289
1290 /* Make sure hardware complete it */
1291 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1292 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1293
1f5b3c3f 1294 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1295}
1296
ba395927 1297/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1298static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1299 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1300{
1301 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1302 u64 val = 0, val_iva = 0;
1303 unsigned long flag;
1304
ba395927
KA		 1305 switch (type) {
1306 case DMA_TLB_GLOBAL_FLUSH:
1307 /* global flush doesn't need set IVA_REG */
1308 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1309 break;
1310 case DMA_TLB_DSI_FLUSH:
1311 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1312 break;
1313 case DMA_TLB_PSI_FLUSH:
1314 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1315 /* IH bit is passed in as part of address */
ba395927
KA		 1316 val_iva = size_order | addr;
1317 break;
1318 default:
1319 BUG();
1320 }
1321 /* Note: set drain read/write */
1322#if 0
1323 /*
1324 * This is probably to be super secure.. Looks like we can
1325 * ignore it without any impact.
1326 */
1327 if (cap_read_drain(iommu->cap))
1328 val |= DMA_TLB_READ_DRAIN;
1329#endif
1330 if (cap_write_drain(iommu->cap))
1331 val |= DMA_TLB_WRITE_DRAIN;
1332
1f5b3c3f 1333 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1334 /* Note: Only uses first TLB reg currently */
1335 if (val_iva)
1336 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1337 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1338
1339 /* Make sure hardware complete it */
1340 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1341 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1342
1f5b3c3f 1343 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1344
1345 /* check IOTLB invalidation granularity */
1346 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1347 pr_err("Flush IOTLB failed\n");
ba395927 1348 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1349 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1350 (unsigned long long)DMA_TLB_IIRG(type),
1351 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1352}
1353
64ae892b
DW		 1354static struct device_domain_info *
1355iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1356 u8 bus, u8 devfn)
93a23a72 1357{
2f119c78 1358 bool found = false;
93a23a72
YZ		 1359 unsigned long flags;
1360 struct device_domain_info *info;
0bcb3e28 1361 struct pci_dev *pdev;
93a23a72
YZ		 1362
1363 if (!ecap_dev_iotlb_support(iommu->ecap))
1364 return NULL;
1365
1366 if (!iommu->qi)
1367 return NULL;
1368
1369 spin_lock_irqsave(&device_domain_lock, flags);
1370 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1371 if (info->iommu == iommu && info->bus == bus &&
1372 info->devfn == devfn) {
2f119c78 1373 found = true;
93a23a72
YZ		 1374 break;
1375 }
1376 spin_unlock_irqrestore(&device_domain_lock, flags);
1377
0bcb3e28 1378 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1379 return NULL;
1380
0bcb3e28
DW		 1381 pdev = to_pci_dev(info->dev);
1382
1383 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1384 return NULL;
1385
0bcb3e28 1386 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1387 return NULL;
1388
93a23a72
YZ		 1389 return info;
1390}
1391
1392static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1393{
0bcb3e28 1394 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1395 return;
1396
0bcb3e28 1397 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1398}
1399
1400static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1401{
0bcb3e28
DW		 1402 if (!info->dev || !dev_is_pci(info->dev) ||
1403 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1404 return;
1405
0bcb3e28 1406 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1407}
1408
1409static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1410 u64 addr, unsigned mask)
1411{
1412 u16 sid, qdep;
1413 unsigned long flags;
1414 struct device_domain_info *info;
1415
1416 spin_lock_irqsave(&device_domain_lock, flags);
1417 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1418 struct pci_dev *pdev;
1419 if (!info->dev || !dev_is_pci(info->dev))
1420 continue;
1421
1422 pdev = to_pci_dev(info->dev);
1423 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1424 continue;
1425
1426 sid = info->bus << 8 | info->devfn;
0bcb3e28 1427 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1428 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1429 }
1430 spin_unlock_irqrestore(&device_domain_lock, flags);
1431}
1432
1f0ef2aa 1433static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1434 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1435{
9dd2fe89 1436 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1437 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1438
ba395927
KA		 1439 BUG_ON(pages == 0);
1440
ea8ea460
DW		 1441 if (ih)
1442 ih = 1 << 6;
ba395927 1443 /*
9dd2fe89
YZ		 1444 * Fallback to domain selective flush if no PSI support or the size is
1445 * too big.
ba395927
KA		 1446 * PSI requires page size to be 2 ^ x, and the base address is naturally
1447 * aligned to the size
1448 */
9dd2fe89
YZ		 1449 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1450 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1451 DMA_TLB_DSI_FLUSH);
9dd2fe89 1452 else
ea8ea460 1453 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1454 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1455
1456 /*
82653633
NA		 1457 * In caching mode, changes of pages from non-present to present require
1458 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1459 */
82653633 1460 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1461 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1462}
1463
f8bab735 1464static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1465{
1466 u32 pmen;
1467 unsigned long flags;
1468
1f5b3c3f 1469 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1470 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1471 pmen &= ~DMA_PMEN_EPM;
1472 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1473
1474 /* wait for the protected region status bit to clear */
1475 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1476 readl, !(pmen & DMA_PMEN_PRS), pmen);
1477
1f5b3c3f 1478 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1479}
1480
2a41ccee 1481static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1482{
1483 u32 sts;
1484 unsigned long flags;
1485
1f5b3c3f 1486 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1487 iommu->gcmd |= DMA_GCMD_TE;
1488 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1489
1490 /* Make sure hardware complete it */
1491 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1492 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1493
1f5b3c3f 1494 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1495}
1496
2a41ccee 1497static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1498{
1499 u32 sts;
1500 unsigned long flag;
1501
1f5b3c3f 1502 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1503 iommu->gcmd &= ~DMA_GCMD_TE;
1504 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1505
1506 /* Make sure hardware complete it */
1507 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1508 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1509
1f5b3c3f 1510 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1511}
1512
3460a6d9 1513
ba395927
KA		 1514static int iommu_init_domains(struct intel_iommu *iommu)
1515{
1516 unsigned long ndomains;
1517 unsigned long nlongs;
1518
1519 ndomains = cap_ndoms(iommu->cap);
9f10e5bf
JR		 1520 pr_debug("%s: Number of Domains supported <%ld>\n",
1521 iommu->name, ndomains);
ba395927
KA		 1522 nlongs = BITS_TO_LONGS(ndomains);
1523
94a91b50
DD		 1524 spin_lock_init(&iommu->lock);
1525
ba395927
KA		 1526 /* TBD: there might be 64K domains,
1527 * consider other allocation for future chip
1528 */
1529 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1530 if (!iommu->domain_ids) {
9f10e5bf
JR		 1531 pr_err("%s: Allocating domain id array failed\n",
1532 iommu->name);
ba395927
KA		 1533 return -ENOMEM;
1534 }
1535 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1536 GFP_KERNEL);
1537 if (!iommu->domains) {
9f10e5bf
JR		 1538 pr_err("%s: Allocating domain array failed\n",
1539 iommu->name);
852bdb04
JL		 1540 kfree(iommu->domain_ids);
1541 iommu->domain_ids = NULL;
ba395927
KA		 1542 return -ENOMEM;
1543 }
1544
1545 /*
1546 * if Caching mode is set, then invalid translations are tagged
1547 * with domainid 0. Hence we need to pre-allocate it.
1548 */
1549 if (cap_caching_mode(iommu->cap))
1550 set_bit(0, iommu->domain_ids);
1551 return 0;
1552}
ba395927 1553
ffebeb46 1554static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1555{
1556 struct dmar_domain *domain;
2a46ddf7 1557 int i;
ba395927 1558
94a91b50 1559 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1560 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1561 /*
1562 * Domain id 0 is reserved for invalid translation
1563 * if hardware supports caching mode.
1564 */
1565 if (cap_caching_mode(iommu->cap) && i == 0)
1566 continue;
1567
94a91b50
DD		 1568 domain = iommu->domains[i];
1569 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1570 if (domain_detach_iommu(domain, iommu) == 0 &&
1571 !domain_type_is_vm(domain))
92d03cc8 1572 domain_exit(domain);
5e98c4b1 1573 }
ba395927
KA		 1574 }
1575
1576 if (iommu->gcmd & DMA_GCMD_TE)
1577 iommu_disable_translation(iommu);
ffebeb46 1578}
ba395927 1579
ffebeb46
JL		 1580static void free_dmar_iommu(struct intel_iommu *iommu)
1581{
1582 if ((iommu->domains) && (iommu->domain_ids)) {
1583 kfree(iommu->domains);
1584 kfree(iommu->domain_ids);
1585 iommu->domains = NULL;
1586 iommu->domain_ids = NULL;
1587 }
ba395927 1588
d9630fe9
WH		 1589 g_iommus[iommu->seq_id] = NULL;
1590
ba395927
KA		 1591 /* free context mapping */
1592 free_context_table(iommu);
ba395927
KA		 1593}
1594
ab8dfe25 1595static struct dmar_domain *alloc_domain(int flags)
ba395927 1596{
92d03cc8
JL		 1597 /* domain id for virtual machine, it won't be set in context */
1598 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1599 struct dmar_domain *domain;
ba395927
KA		 1600
1601 domain = alloc_domain_mem();
1602 if (!domain)
1603 return NULL;
1604
ab8dfe25 1605 memset(domain, 0, sizeof(*domain));
4c923d47 1606 domain->nid = -1;
ab8dfe25 1607 domain->flags = flags;
92d03cc8
JL		 1608 spin_lock_init(&domain->iommu_lock);
1609 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1610 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1611 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1612
1613 return domain;
1614}
1615
fb170fb4
JL		 1616static int __iommu_attach_domain(struct dmar_domain *domain,
1617 struct intel_iommu *iommu)
2c2e2c38
FY		 1618{
1619 int num;
1620 unsigned long ndomains;
2c2e2c38 1621
ba395927 1622 ndomains = cap_ndoms(iommu->cap);
ba395927 1623 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1624 if (num < ndomains) {
1625 set_bit(num, iommu->domain_ids);
1626 iommu->domains[num] = domain;
1627 } else {
1628 num = -ENOSPC;
ba395927
KA		 1629 }
1630
fb170fb4
JL		 1631 return num;
1632}
1633
1634static int iommu_attach_domain(struct dmar_domain *domain,
1635 struct intel_iommu *iommu)
1636{
1637 int num;
1638 unsigned long flags;
1639
1640 spin_lock_irqsave(&iommu->lock, flags);
1641 num = __iommu_attach_domain(domain, iommu);
44bde614 1642 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1643 if (num < 0)
9f10e5bf 1644 pr_err("%s: No free domain ids\n", iommu->name);
ba395927 1645
fb170fb4 1646 return num;
ba395927
KA		 1647}
1648
44bde614
JL		 1649static int iommu_attach_vm_domain(struct dmar_domain *domain,
1650 struct intel_iommu *iommu)
1651{
1652 int num;
1653 unsigned long ndomains;
1654
1655 ndomains = cap_ndoms(iommu->cap);
1656 for_each_set_bit(num, iommu->domain_ids, ndomains)
1657 if (iommu->domains[num] == domain)
1658 return num;
1659
1660 return __iommu_attach_domain(domain, iommu);
1661}
1662
2c2e2c38
FY		 1663static void iommu_detach_domain(struct dmar_domain *domain,
1664 struct intel_iommu *iommu)
ba395927
KA		 1665{
1666 unsigned long flags;
2c2e2c38 1667 int num, ndomains;
ba395927 1668
8c11e798 1669 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1670 if (domain_type_is_vm_or_si(domain)) {
1671 ndomains = cap_ndoms(iommu->cap);
1672 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1673 if (iommu->domains[num] == domain) {
1674 clear_bit(num, iommu->domain_ids);
1675 iommu->domains[num] = NULL;
1676 break;
1677 }
2c2e2c38 1678 }
fb170fb4
JL		 1679 } else {
1680 clear_bit(domain->id, iommu->domain_ids);
1681 iommu->domains[domain->id] = NULL;
2c2e2c38 1682 }
8c11e798 1683 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1684}
1685
fb170fb4
JL		 1686static void domain_attach_iommu(struct dmar_domain *domain,
1687 struct intel_iommu *iommu)
1688{
1689 unsigned long flags;
1690
1691 spin_lock_irqsave(&domain->iommu_lock, flags);
1692 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1693 domain->iommu_count++;
1694 if (domain->iommu_count == 1)
1695 domain->nid = iommu->node;
1696 domain_update_iommu_cap(domain);
1697 }
1698 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1699}
1700
1701static int domain_detach_iommu(struct dmar_domain *domain,
1702 struct intel_iommu *iommu)
1703{
1704 unsigned long flags;
1705 int count = INT_MAX;
1706
1707 spin_lock_irqsave(&domain->iommu_lock, flags);
1708 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1709 count = --domain->iommu_count;
1710 domain_update_iommu_cap(domain);
1711 }
1712 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1713
1714 return count;
1715}
1716
ba395927 1717static struct iova_domain reserved_iova_list;
8a443df4 1718static struct lock_class_key reserved_rbtree_key;
ba395927 1719
51a63e67 1720static int dmar_init_reserved_ranges(void)
ba395927
KA		 1721{
1722 struct pci_dev *pdev = NULL;
1723 struct iova *iova;
1724 int i;
ba395927 1725
0fb5fe87
RM		 1726 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1727 DMA_32BIT_PFN);
ba395927 1728
8a443df4
MG		 1729 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1730 &reserved_rbtree_key);
1731
ba395927
KA		 1732 /* IOAPIC ranges shouldn't be accessed by DMA */
1733 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1734 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1735 if (!iova) {
9f10e5bf 1736 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1737 return -ENODEV;
1738 }
ba395927
KA		 1739
1740 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1741 for_each_pci_dev(pdev) {
1742 struct resource *r;
1743
1744 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1745 r = &pdev->resource[i];
1746 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1747 continue;
1a4a4551
DW		 1748 iova = reserve_iova(&reserved_iova_list,
1749 IOVA_PFN(r->start),
1750 IOVA_PFN(r->end));
51a63e67 1751 if (!iova) {
9f10e5bf 1752 pr_err("Reserve iova failed\n");
51a63e67
JC		 1753 return -ENODEV;
1754 }
ba395927
KA		 1755 }
1756 }
51a63e67 1757 return 0;
ba395927
KA		 1758}
1759
1760static void domain_reserve_special_ranges(struct dmar_domain *domain)
1761{
1762 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1763}
1764
1765static inline int guestwidth_to_adjustwidth(int gaw)
1766{
1767 int agaw;
1768 int r = (gaw - 12) % 9;
1769
1770 if (r == 0)
1771 agaw = gaw;
1772 else
1773 agaw = gaw + 9 - r;
1774 if (agaw > 64)
1775 agaw = 64;
1776 return agaw;
1777}
1778
1779static int domain_init(struct dmar_domain *domain, int guest_width)
1780{
1781 struct intel_iommu *iommu;
1782 int adjust_width, agaw;
1783 unsigned long sagaw;
1784
0fb5fe87
RM		 1785 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1786 DMA_32BIT_PFN);
ba395927
KA		 1787 domain_reserve_special_ranges(domain);
1788
1789 /* calculate AGAW */
8c11e798 1790 iommu = domain_get_iommu(domain);
ba395927
KA		 1791 if (guest_width > cap_mgaw(iommu->cap))
1792 guest_width = cap_mgaw(iommu->cap);
1793 domain->gaw = guest_width;
1794 adjust_width = guestwidth_to_adjustwidth(guest_width);
1795 agaw = width_to_agaw(adjust_width);
1796 sagaw = cap_sagaw(iommu->cap);
1797 if (!test_bit(agaw, &sagaw)) {
1798 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1799 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1800 agaw = find_next_bit(&sagaw, 5, agaw);
1801 if (agaw >= 5)
1802 return -ENODEV;
1803 }
1804 domain->agaw = agaw;
ba395927 1805
8e604097
WH		 1806 if (ecap_coherent(iommu->ecap))
1807 domain->iommu_coherency = 1;
1808 else
1809 domain->iommu_coherency = 0;
1810
58c610bd
SY		 1811 if (ecap_sc_support(iommu->ecap))
1812 domain->iommu_snooping = 1;
1813 else
1814 domain->iommu_snooping = 0;
1815
214e39aa
DW		 1816 if (intel_iommu_superpage)
1817 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1818 else
1819 domain->iommu_superpage = 0;
1820
4c923d47 1821 domain->nid = iommu->node;
c7151a8d 1822
ba395927 1823 /* always allocate the top pgd */
4c923d47 1824 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1825 if (!domain->pgd)
1826 return -ENOMEM;
5b6985ce 1827 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1828 return 0;
1829}
1830
1831static void domain_exit(struct dmar_domain *domain)
1832{
ea8ea460 1833 struct page *freelist = NULL;
71684406 1834 int i;
ba395927
KA		 1835
1836 /* Domain 0 is reserved, so dont process it */
1837 if (!domain)
1838 return;
1839
7b668357
AW		 1840 /* Flush any lazy unmaps that may reference this domain */
1841 if (!intel_iommu_strict)
1842 flush_unmaps_timeout(0);
1843
92d03cc8 1844 /* remove associated devices */
ba395927 1845 domain_remove_dev_info(domain);
92d03cc8 1846
ba395927
KA		 1847 /* destroy iovas */
1848 put_iova_domain(&domain->iovad);
ba395927 1849
ea8ea460 1850 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1851
92d03cc8 1852 /* clear attached or cached domains */
0e242612 1853 rcu_read_lock();
71684406
AW		 1854 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1855 iommu_detach_domain(domain, g_iommus[i]);
0e242612 1856 rcu_read_unlock();
2c2e2c38 1857
ea8ea460
DW		 1858 dma_free_pagelist(freelist);
1859
ba395927
KA		 1860 free_domain_mem(domain);
1861}
1862
64ae892b
DW		 1863static int domain_context_mapping_one(struct dmar_domain *domain,
1864 struct intel_iommu *iommu,
1865 u8 bus, u8 devfn, int translation)
ba395927
KA		 1866{
1867 struct context_entry *context;
ba395927 1868 unsigned long flags;
ea6606b0 1869 struct dma_pte *pgd;
ea6606b0
WH		 1870 int id;
1871 int agaw;
93a23a72 1872 struct device_domain_info *info = NULL;
ba395927
KA		 1873
1874 pr_debug("Set context mapping for %02x:%02x.%d\n",
1875 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1876
ba395927 1877 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1878 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1879 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1880
03ecc32c
DW		 1881 spin_lock_irqsave(&iommu->lock, flags);
1882 context = iommu_context_addr(iommu, bus, devfn, 1);
1883 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1884 if (!context)
1885 return -ENOMEM;
1886 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1887 if (context_present(context)) {
ba395927
KA		 1888 spin_unlock_irqrestore(&iommu->lock, flags);
1889 return 0;
1890 }
1891
cf484d0e
JR		 1892 context_clear_entry(context);
1893
ea6606b0
WH		 1894 id = domain->id;
1895 pgd = domain->pgd;
1896
ab8dfe25 1897 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1898 if (domain_type_is_vm(domain)) {
1899 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1900 if (id < 0) {
ea6606b0 1901 spin_unlock_irqrestore(&iommu->lock, flags);
9f10e5bf 1902 pr_err("%s: No free domain ids\n", iommu->name);
ea6606b0
WH		 1903 return -EFAULT;
1904 }
ea6606b0
WH		 1905 }
1906
1907 /* Skip top levels of page tables for
1908 * iommu which has less agaw than default.
1672af11 1909 * Unnecessary for PT mode.
ea6606b0 1910 */
1672af11
CW		 1911 if (translation != CONTEXT_TT_PASS_THROUGH) {
1912 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1913 pgd = phys_to_virt(dma_pte_addr(pgd));
1914 if (!dma_pte_present(pgd)) {
1915 spin_unlock_irqrestore(&iommu->lock, flags);
1916 return -ENOMEM;
1917 }
ea6606b0
WH		 1918 }
1919 }
1920 }
1921
1922 context_set_domain_id(context, id);
4ed0d3e6 1923
93a23a72 1924 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1925 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1926 translation = info ? CONTEXT_TT_DEV_IOTLB :
1927 CONTEXT_TT_MULTI_LEVEL;
1928 }
4ed0d3e6
FY		 1929 /*
1930 * In pass through mode, AW must be programmed to indicate the largest
1931 * AGAW value supported by hardware. And ASR is ignored by hardware.
1932 */
93a23a72 1933 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1934 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1935 else {
1936 context_set_address_root(context, virt_to_phys(pgd));
1937 context_set_address_width(context, iommu->agaw);
1938 }
4ed0d3e6
FY		 1939
1940 context_set_translation_type(context, translation);
c07e7d21
MM		 1941 context_set_fault_enable(context);
1942 context_set_present(context);
5331fe6f 1943 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1944
4c25a2c1
DW		 1945 /*
1946 * It's a non-present to present mapping. If hardware doesn't cache
1947 * non-present entry we only need to flush the write-buffer. If the
1948 * _does_ cache non-present entries, then it does so in the special
1949 * domain #0, which we have to flush:
1950 */
1951 if (cap_caching_mode(iommu->cap)) {
1952 iommu->flush.flush_context(iommu, 0,
1953 (((u16)bus) << 8) | devfn,
1954 DMA_CCMD_MASK_NOBIT,
1955 DMA_CCMD_DEVICE_INVL);
18fd779a 1956 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1957 } else {
ba395927 1958 iommu_flush_write_buffer(iommu);
4c25a2c1 1959 }
93a23a72 1960 iommu_enable_dev_iotlb(info);
ba395927 1961 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1962
fb170fb4
JL		 1963 domain_attach_iommu(domain, iommu);
1964
ba395927
KA		 1965 return 0;
1966}
1967
579305f7
AW		 1968struct domain_context_mapping_data {
1969 struct dmar_domain *domain;
1970 struct intel_iommu *iommu;
1971 int translation;
1972};
1973
1974static int domain_context_mapping_cb(struct pci_dev *pdev,
1975 u16 alias, void *opaque)
1976{
1977 struct domain_context_mapping_data *data = opaque;
1978
1979 return domain_context_mapping_one(data->domain, data->iommu,
1980 PCI_BUS_NUM(alias), alias & 0xff,
1981 data->translation);
1982}
1983
ba395927 1984static int
e1f167f3
DW		 1985domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1986 int translation)
ba395927 1987{
64ae892b 1988 struct intel_iommu *iommu;
156baca8 1989 u8 bus, devfn;
579305f7 1990 struct domain_context_mapping_data data;
64ae892b 1991
e1f167f3 1992 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1993 if (!iommu)
1994 return -ENODEV;
ba395927 1995
579305f7
AW		 1996 if (!dev_is_pci(dev))
1997 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1998 translation);
579305f7
AW		 1999
2000 data.domain = domain;
2001 data.iommu = iommu;
2002 data.translation = translation;
2003
2004 return pci_for_each_dma_alias(to_pci_dev(dev),
2005 &domain_context_mapping_cb, &data);
2006}
2007
2008static int domain_context_mapped_cb(struct pci_dev *pdev,
2009 u16 alias, void *opaque)
2010{
2011 struct intel_iommu *iommu = opaque;
2012
2013 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2014}
2015
e1f167f3 2016static int domain_context_mapped(struct device *dev)
ba395927 2017{
5331fe6f 2018 struct intel_iommu *iommu;
156baca8 2019 u8 bus, devfn;
5331fe6f 2020
e1f167f3 2021 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2022 if (!iommu)
2023 return -ENODEV;
ba395927 2024
579305f7
AW		 2025 if (!dev_is_pci(dev))
2026 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2027
579305f7
AW		 2028 return !pci_for_each_dma_alias(to_pci_dev(dev),
2029 domain_context_mapped_cb, iommu);
ba395927
KA		 2030}
2031
f532959b
FY		 2032/* Returns a number of VTD pages, but aligned to MM page size */
2033static inline unsigned long aligned_nrpages(unsigned long host_addr,
2034 size_t size)
2035{
2036 host_addr &= ~PAGE_MASK;
2037 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2038}
2039
6dd9a7c7
YS		 2040/* Return largest possible superpage level for a given mapping */
2041static inline int hardware_largepage_caps(struct dmar_domain *domain,
2042 unsigned long iov_pfn,
2043 unsigned long phy_pfn,
2044 unsigned long pages)
2045{
2046 int support, level = 1;
2047 unsigned long pfnmerge;
2048
2049 support = domain->iommu_superpage;
2050
2051 /* To use a large page, the virtual *and* physical addresses
2052 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2053 of them will mean we have to use smaller pages. So just
2054 merge them and check both at once. */
2055 pfnmerge = iov_pfn | phy_pfn;
2056
2057 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2058 pages >>= VTD_STRIDE_SHIFT;
2059 if (!pages)
2060 break;
2061 pfnmerge >>= VTD_STRIDE_SHIFT;
2062 level++;
2063 support--;
2064 }
2065 return level;
2066}
2067
9051aa02
DW		 2068static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2069 struct scatterlist *sg, unsigned long phys_pfn,
2070 unsigned long nr_pages, int prot)
e1605495
DW		 2071{
2072 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2073 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2074 unsigned long sg_res = 0;
6dd9a7c7
YS		 2075 unsigned int largepage_lvl = 0;
2076 unsigned long lvl_pages = 0;
e1605495 2077
162d1b10 2078 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2079
2080 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2081 return -EINVAL;
2082
2083 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2084
cc4f14aa
JL		 2085 if (!sg) {
2086 sg_res = nr_pages;
9051aa02
DW		 2087 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2088 }
2089
6dd9a7c7 2090 while (nr_pages > 0) {
c85994e4
DW		 2091 uint64_t tmp;
2092
e1605495 2093 if (!sg_res) {
f532959b 2094 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2095 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2096 sg->dma_length = sg->length;
2097 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2098 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2099 }
6dd9a7c7 2100
e1605495 2101 if (!pte) {
6dd9a7c7
YS		 2102 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2103
5cf0a76f 2104 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2105 if (!pte)
2106 return -ENOMEM;
6dd9a7c7 2107 /* It is large page*/
6491d4d0 2108 if (largepage_lvl > 1) {
6dd9a7c7 2109 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2110 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2111 /*
2112 * Ensure that old small page tables are
2113 * removed to make room for superpage,
2114 * if they exist.
2115 */
6491d4d0 2116 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2117 iov_pfn + lvl_pages - 1);
6491d4d0 2118 } else {
6dd9a7c7 2119 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2120 }
6dd9a7c7 2121
e1605495
DW		 2122 }
2123 /* We don't need lock here, nobody else
2124 * touches the iova range
2125 */
7766a3fb 2126 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2127 if (tmp) {
1bf20f0d 2128 static int dumps = 5;
9f10e5bf
JR		 2129 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2130 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2131 if (dumps) {
2132 dumps--;
2133 debug_dma_dump_mappings(NULL);
2134 }
2135 WARN_ON(1);
2136 }
6dd9a7c7
YS		 2137
2138 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2139
2140 BUG_ON(nr_pages < lvl_pages);
2141 BUG_ON(sg_res < lvl_pages);
2142
2143 nr_pages -= lvl_pages;
2144 iov_pfn += lvl_pages;
2145 phys_pfn += lvl_pages;
2146 pteval += lvl_pages * VTD_PAGE_SIZE;
2147 sg_res -= lvl_pages;
2148
2149 /* If the next PTE would be the first in a new page, then we
2150 need to flush the cache on the entries we've just written.
2151 And then we'll need to recalculate 'pte', so clear it and
2152 let it get set again in the if (!pte) block above.
2153
2154 If we're done (!nr_pages) we need to flush the cache too.
2155
2156 Also if we've been setting superpages, we may need to
2157 recalculate 'pte' and switch back to smaller pages for the
2158 end of the mapping, if the trailing size is not enough to
2159 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2160 pte++;
6dd9a7c7
YS		 2161 if (!nr_pages || first_pte_in_page(pte) ||
2162 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2163 domain_flush_cache(domain, first_pte,
2164 (void *)pte - (void *)first_pte);
2165 pte = NULL;
2166 }
6dd9a7c7
YS		 2167
2168 if (!sg_res && nr_pages)
e1605495
DW		 2169 sg = sg_next(sg);
2170 }
2171 return 0;
2172}
2173
9051aa02
DW		 2174static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175 struct scatterlist *sg, unsigned long nr_pages,
2176 int prot)
ba395927 2177{
9051aa02
DW		 2178 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2179}
6f6a00e4 2180
9051aa02
DW		 2181static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2182 unsigned long phys_pfn, unsigned long nr_pages,
2183 int prot)
2184{
2185 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2186}
2187
c7151a8d 2188static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2189{
c7151a8d
WH		 2190 if (!iommu)
2191 return;
8c11e798
WH		 2192
2193 clear_context_table(iommu, bus, devfn);
2194 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2195 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2196 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2197}
2198
109b9b04
DW		 2199static inline void unlink_domain_info(struct device_domain_info *info)
2200{
2201 assert_spin_locked(&device_domain_lock);
2202 list_del(&info->link);
2203 list_del(&info->global);
2204 if (info->dev)
0bcb3e28 2205 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2206}
2207
ba395927
KA		 2208static void domain_remove_dev_info(struct dmar_domain *domain)
2209{
3a74ca01 2210 struct device_domain_info *info, *tmp;
fb170fb4 2211 unsigned long flags;
ba395927
KA		 2212
2213 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2214 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2215 unlink_domain_info(info);
ba395927
KA		 2216 spin_unlock_irqrestore(&device_domain_lock, flags);
2217
93a23a72 2218 iommu_disable_dev_iotlb(info);
7c7faa11 2219 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2220
ab8dfe25 2221 if (domain_type_is_vm(domain)) {
7c7faa11 2222 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2223 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2224 }
2225
2226 free_devinfo_mem(info);
ba395927
KA		 2227 spin_lock_irqsave(&device_domain_lock, flags);
2228 }
2229 spin_unlock_irqrestore(&device_domain_lock, flags);
2230}
2231
2232/*
2233 * find_domain
1525a29a 2234 * Note: we use struct device->archdata.iommu stores the info
ba395927 2235 */
1525a29a 2236static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2237{
2238 struct device_domain_info *info;
2239
2240 /* No lock here, assumes no domain exit in normal case */
1525a29a 2241 info = dev->archdata.iommu;
ba395927
KA		 2242 if (info)
2243 return info->domain;
2244 return NULL;
2245}
2246
5a8f40e8 2247static inline struct device_domain_info *
745f2586
JL		 2248dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2249{
2250 struct device_domain_info *info;
2251
2252 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2253 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2254 info->devfn == devfn)
5a8f40e8 2255 return info;
745f2586
JL		 2256
2257 return NULL;
2258}
2259
5a8f40e8 2260static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2261 int bus, int devfn,
b718cd3d
DW		 2262 struct device *dev,
2263 struct dmar_domain *domain)
745f2586 2264{
5a8f40e8 2265 struct dmar_domain *found = NULL;
745f2586
JL		 2266 struct device_domain_info *info;
2267 unsigned long flags;
2268
2269 info = alloc_devinfo_mem();
2270 if (!info)
b718cd3d 2271 return NULL;
745f2586 2272
745f2586
JL		 2273 info->bus = bus;
2274 info->devfn = devfn;
2275 info->dev = dev;
2276 info->domain = domain;
5a8f40e8 2277 info->iommu = iommu;
745f2586
JL		 2278
2279 spin_lock_irqsave(&device_domain_lock, flags);
2280 if (dev)
0bcb3e28 2281 found = find_domain(dev);
5a8f40e8
DW		 2282 else {
2283 struct device_domain_info *info2;
41e80dca 2284 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2285 if (info2)
2286 found = info2->domain;
2287 }
745f2586
JL		 2288 if (found) {
2289 spin_unlock_irqrestore(&device_domain_lock, flags);
2290 free_devinfo_mem(info);
b718cd3d
DW		 2291 /* Caller must free the original domain */
2292 return found;
745f2586
JL		 2293 }
2294
b718cd3d
DW		 2295 list_add(&info->link, &domain->devices);
2296 list_add(&info->global, &device_domain_list);
2297 if (dev)
2298 dev->archdata.iommu = info;
2299 spin_unlock_irqrestore(&device_domain_lock, flags);
2300
2301 return domain;
745f2586
JL		 2302}
2303
579305f7
AW		 2304static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2305{
2306 *(u16 *)opaque = alias;
2307 return 0;
2308}
2309
ba395927 2310/* domain is initialized */
146922ec 2311static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2312{
579305f7
AW		 2313 struct dmar_domain *domain, *tmp;
2314 struct intel_iommu *iommu;
5a8f40e8 2315 struct device_domain_info *info;
579305f7 2316 u16 dma_alias;
ba395927 2317 unsigned long flags;
aa4d066a 2318 u8 bus, devfn;
ba395927 2319
146922ec 2320 domain = find_domain(dev);
ba395927
KA		 2321 if (domain)
2322 return domain;
2323
579305f7
AW		 2324 iommu = device_to_iommu(dev, &bus, &devfn);
2325 if (!iommu)
2326 return NULL;
2327
146922ec
DW		 2328 if (dev_is_pci(dev)) {
2329 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2330
579305f7
AW		 2331 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2332
2333 spin_lock_irqsave(&device_domain_lock, flags);
2334 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2335 PCI_BUS_NUM(dma_alias),
2336 dma_alias & 0xff);
2337 if (info) {
2338 iommu = info->iommu;
2339 domain = info->domain;
5a8f40e8 2340 }
579305f7 2341 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2342
579305f7
AW		 2343 /* DMA alias already has a domain, uses it */
2344 if (info)
2345 goto found_domain;
2346 }
ba395927 2347
146922ec 2348 /* Allocate and initialize new domain for the device */
ab8dfe25 2349 domain = alloc_domain(0);
745f2586 2350 if (!domain)
579305f7 2351 return NULL;
44bde614
JL		 2352 domain->id = iommu_attach_domain(domain, iommu);
2353 if (domain->id < 0) {
2fe9723d 2354 free_domain_mem(domain);
579305f7 2355 return NULL;
2c2e2c38 2356 }
fb170fb4 2357 domain_attach_iommu(domain, iommu);
579305f7
AW		 2358 if (domain_init(domain, gaw)) {
2359 domain_exit(domain);
2360 return NULL;
2c2e2c38 2361 }
ba395927 2362
579305f7
AW		 2363 /* register PCI DMA alias device */
2364 if (dev_is_pci(dev)) {
2365 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2366 dma_alias & 0xff, NULL, domain);
2367
2368 if (!tmp || tmp != domain) {
2369 domain_exit(domain);
2370 domain = tmp;
2371 }
2372
b718cd3d 2373 if (!domain)
579305f7 2374 return NULL;
ba395927
KA		 2375 }
2376
2377found_domain:
579305f7
AW		 2378 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2379
2380 if (!tmp || tmp != domain) {
2381 domain_exit(domain);
2382 domain = tmp;
2383 }
b718cd3d
DW		 2384
2385 return domain;
ba395927
KA		 2386}
2387
2c2e2c38 2388static int iommu_identity_mapping;
e0fc7e0b
DW		 2389#define IDENTMAP_ALL 1
2390#define IDENTMAP_GFX 2
2391#define IDENTMAP_AZALIA 4
2c2e2c38 2392
b213203e
DW		 2393static int iommu_domain_identity_map(struct dmar_domain *domain,
2394 unsigned long long start,
2395 unsigned long long end)
ba395927 2396{
c5395d5c
DW		 2397 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2398 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2399
2400 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2401 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2402 pr_err("Reserving iova failed\n");
b213203e 2403 return -ENOMEM;
ba395927
KA		 2404 }
2405
c5395d5c
DW		 2406 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2407 start, end, domain->id);
ba395927
KA		 2408 /*
2409 * RMRR range might have overlap with physical memory range,
2410 * clear it first
2411 */
c5395d5c 2412 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2413
c5395d5c
DW		 2414 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2415 last_vpfn - first_vpfn + 1,
61df7443 2416 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2417}
2418
0b9d9753 2419static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2420 unsigned long long start,
2421 unsigned long long end)
2422{
2423 struct dmar_domain *domain;
2424 int ret;
2425
0b9d9753 2426 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2427 if (!domain)
2428 return -ENOMEM;
2429
19943b0e
DW		 2430 /* For _hardware_ passthrough, don't bother. But for software
2431 passthrough, we do it anyway -- it may indicate a memory
2432 range which is reserved in E820, so which didn't get set
2433 up to start with in si_domain */
2434 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2435 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2436 dev_name(dev), start, end);
19943b0e
DW		 2437 return 0;
2438 }
2439
9f10e5bf
JR		 2440 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2441 dev_name(dev), start, end);
2442
5595b528
DW		 2443 if (end < start) {
2444 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2445 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2446 dmi_get_system_info(DMI_BIOS_VENDOR),
2447 dmi_get_system_info(DMI_BIOS_VERSION),
2448 dmi_get_system_info(DMI_PRODUCT_VERSION));
2449 ret = -EIO;
2450 goto error;
2451 }
2452
2ff729f5
DW		 2453 if (end >> agaw_to_width(domain->agaw)) {
2454 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2455 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2456 agaw_to_width(domain->agaw),
2457 dmi_get_system_info(DMI_BIOS_VENDOR),
2458 dmi_get_system_info(DMI_BIOS_VERSION),
2459 dmi_get_system_info(DMI_PRODUCT_VERSION));
2460 ret = -EIO;
2461 goto error;
2462 }
19943b0e 2463
b213203e 2464 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2465 if (ret)
2466 goto error;
2467
2468 /* context entry init */
0b9d9753 2469 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2470 if (ret)
2471 goto error;
2472
2473 return 0;
2474
2475 error:
ba395927
KA		 2476 domain_exit(domain);
2477 return ret;
ba395927
KA		 2478}
2479
2480static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2481 struct device *dev)
ba395927 2482{
0b9d9753 2483 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2484 return 0;
0b9d9753
DW		 2485 return iommu_prepare_identity_map(dev, rmrr->base_address,
2486 rmrr->end_address);
ba395927
KA		 2487}
2488
d3f13810 2489#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2490static inline void iommu_prepare_isa(void)
2491{
2492 struct pci_dev *pdev;
2493 int ret;
2494
2495 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2496 if (!pdev)
2497 return;
2498
9f10e5bf 2499 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2500 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2501
2502 if (ret)
9f10e5bf 2503 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2504
9b27e82d 2505 pci_dev_put(pdev);
49a0429e
KA		 2506}
2507#else
2508static inline void iommu_prepare_isa(void)
2509{
2510 return;
2511}
d3f13810 2512#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2513
2c2e2c38 2514static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2515
071e1374 2516static int __init si_domain_init(int hw)
2c2e2c38
FY		 2517{
2518 struct dmar_drhd_unit *drhd;
2519 struct intel_iommu *iommu;
c7ab48d2 2520 int nid, ret = 0;
44bde614 2521 bool first = true;
2c2e2c38 2522
ab8dfe25 2523 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2524 if (!si_domain)
2525 return -EFAULT;
2526
2c2e2c38
FY		 2527 for_each_active_iommu(iommu, drhd) {
2528 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2529 if (ret < 0) {
2c2e2c38
FY		 2530 domain_exit(si_domain);
2531 return -EFAULT;
44bde614
JL		 2532 } else if (first) {
2533 si_domain->id = ret;
2534 first = false;
2535 } else if (si_domain->id != ret) {
2536 domain_exit(si_domain);
2537 return -EFAULT;
2c2e2c38 2538 }
fb170fb4 2539 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2540 }
2541
2542 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2543 domain_exit(si_domain);
2544 return -EFAULT;
2545 }
2546
9f10e5bf 2547 pr_debug("Identity mapping domain is domain %d\n",
9544c003 2548 si_domain->id);
2c2e2c38 2549
19943b0e
DW		 2550 if (hw)
2551 return 0;
2552
c7ab48d2 2553 for_each_online_node(nid) {
5dfe8660
TH		 2554 unsigned long start_pfn, end_pfn;
2555 int i;
2556
2557 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2558 ret = iommu_domain_identity_map(si_domain,
2559 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2560 if (ret)
2561 return ret;
2562 }
c7ab48d2
DW		 2563 }
2564
2c2e2c38
FY		 2565 return 0;
2566}
2567
9b226624 2568static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2569{
2570 struct device_domain_info *info;
2571
2572 if (likely(!iommu_identity_mapping))
2573 return 0;
2574
9b226624 2575 info = dev->archdata.iommu;
cb452a40
MT		 2576 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2577 return (info->domain == si_domain);
2c2e2c38 2578
2c2e2c38
FY		 2579 return 0;
2580}
2581
2582static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2583 struct device *dev, int translation)
2c2e2c38 2584{
0ac72664 2585 struct dmar_domain *ndomain;
5a8f40e8 2586 struct intel_iommu *iommu;
156baca8 2587 u8 bus, devfn;
5fe60f4e 2588 int ret;
2c2e2c38 2589
5913c9bf 2590 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2591 if (!iommu)
2592 return -ENODEV;
2593
5913c9bf 2594 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2595 if (ndomain != domain)
2596 return -EBUSY;
2c2e2c38 2597
5913c9bf 2598 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2599 if (ret) {
5913c9bf 2600 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2601 return ret;
2602 }
2603
2c2e2c38
FY		 2604 return 0;
2605}
2606
0b9d9753 2607static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2608{
2609 struct dmar_rmrr_unit *rmrr;
832bd858 2610 struct device *tmp;
ea2447f7
TM		 2611 int i;
2612
0e242612 2613 rcu_read_lock();
ea2447f7 2614 for_each_rmrr_units(rmrr) {
b683b230
JL		 2615 /*
2616 * Return TRUE if this RMRR contains the device that
2617 * is passed in.
2618 */
2619 for_each_active_dev_scope(rmrr->devices,
2620 rmrr->devices_cnt, i, tmp)
0b9d9753 2621 if (tmp == dev) {
0e242612 2622 rcu_read_unlock();
ea2447f7 2623 return true;
b683b230 2624 }
ea2447f7 2625 }
0e242612 2626 rcu_read_unlock();
ea2447f7
TM		 2627 return false;
2628}
2629
c875d2c1
AW		 2630/*
2631 * There are a couple cases where we need to restrict the functionality of
2632 * devices associated with RMRRs. The first is when evaluating a device for
2633 * identity mapping because problems exist when devices are moved in and out
2634 * of domains and their respective RMRR information is lost. This means that
2635 * a device with associated RMRRs will never be in a "passthrough" domain.
2636 * The second is use of the device through the IOMMU API. This interface
2637 * expects to have full control of the IOVA space for the device. We cannot
2638 * satisfy both the requirement that RMRR access is maintained and have an
2639 * unencumbered IOVA space. We also have no ability to quiesce the device's
2640 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2641 * We therefore prevent devices associated with an RMRR from participating in
2642 * the IOMMU API, which eliminates them from device assignment.
2643 *
2644 * In both cases we assume that PCI USB devices with RMRRs have them largely
2645 * for historical reasons and that the RMRR space is not actively used post
2646 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2647 *
2648 * The same exception is made for graphics devices, with the requirement that
2649 * any use of the RMRR regions will be torn down before assigning the device
2650 * to a guest.
c875d2c1
AW		 2651 */
2652static bool device_is_rmrr_locked(struct device *dev)
2653{
2654 if (!device_has_rmrr(dev))
2655 return false;
2656
2657 if (dev_is_pci(dev)) {
2658 struct pci_dev *pdev = to_pci_dev(dev);
2659
18436afd 2660 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2661 return false;
2662 }
2663
2664 return true;
2665}
2666
3bdb2591 2667static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2668{
ea2447f7 2669
3bdb2591
DW		 2670 if (dev_is_pci(dev)) {
2671 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2672
c875d2c1 2673 if (device_is_rmrr_locked(dev))
3bdb2591 2674 return 0;
e0fc7e0b 2675
3bdb2591
DW		 2676 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2677 return 1;
e0fc7e0b 2678
3bdb2591
DW		 2679 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2680 return 1;
6941af28 2681
3bdb2591 2682 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2683 return 0;
3bdb2591
DW		 2684
2685 /*
2686 * We want to start off with all devices in the 1:1 domain, and
2687 * take them out later if we find they can't access all of memory.
2688 *
2689 * However, we can't do this for PCI devices behind bridges,
2690 * because all PCI devices behind the same bridge will end up
2691 * with the same source-id on their transactions.
2692 *
2693 * Practically speaking, we can't change things around for these
2694 * devices at run-time, because we can't be sure there'll be no
2695 * DMA transactions in flight for any of their siblings.
2696 *
2697 * So PCI devices (unless they're on the root bus) as well as
2698 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2699 * the 1:1 domain, just in _case_ one of their siblings turns out
2700 * not to be able to map all of memory.
2701 */
2702 if (!pci_is_pcie(pdev)) {
2703 if (!pci_is_root_bus(pdev->bus))
2704 return 0;
2705 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2706 return 0;
2707 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2708 return 0;
3bdb2591
DW		 2709 } else {
2710 if (device_has_rmrr(dev))
2711 return 0;
2712 }
3dfc813d 2713
3bdb2591 2714 /*
3dfc813d 2715 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2716 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2717 * take them out of the 1:1 domain later.
2718 */
8fcc5372
CW		 2719 if (!startup) {
2720 /*
2721 * If the device's dma_mask is less than the system's memory
2722 * size then this is not a candidate for identity mapping.
2723 */
3bdb2591 2724 u64 dma_mask = *dev->dma_mask;
8fcc5372 2725
3bdb2591
DW		 2726 if (dev->coherent_dma_mask &&
2727 dev->coherent_dma_mask < dma_mask)
2728 dma_mask = dev->coherent_dma_mask;
8fcc5372 2729
3bdb2591 2730 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2731 }
6941af28
DW		 2732
2733 return 1;
2734}
2735
cf04eee8
DW		 2736static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2737{
2738 int ret;
2739
2740 if (!iommu_should_identity_map(dev, 1))
2741 return 0;
2742
2743 ret = domain_add_dev_info(si_domain, dev,
2744 hw ? CONTEXT_TT_PASS_THROUGH :
2745 CONTEXT_TT_MULTI_LEVEL);
2746 if (!ret)
9f10e5bf
JR		 2747 pr_info("%s identity mapping for device %s\n",
2748 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2749 else if (ret == -ENODEV)
2750 /* device not associated with an iommu */
2751 ret = 0;
2752
2753 return ret;
2754}
2755
2756
071e1374 2757static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2758{
2c2e2c38 2759 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2760 struct dmar_drhd_unit *drhd;
2761 struct intel_iommu *iommu;
2762 struct device *dev;
2763 int i;
2764 int ret = 0;
2c2e2c38 2765
2c2e2c38 2766 for_each_pci_dev(pdev) {
cf04eee8
DW		 2767 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2768 if (ret)
2769 return ret;
2770 }
2771
2772 for_each_active_iommu(iommu, drhd)
2773 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2774 struct acpi_device_physical_node *pn;
2775 struct acpi_device *adev;
2776
2777 if (dev->bus != &acpi_bus_type)
2778 continue;
86080ccc 2779
cf04eee8
DW		 2780 adev= to_acpi_device(dev);
2781 mutex_lock(&adev->physical_node_lock);
2782 list_for_each_entry(pn, &adev->physical_node_list, node) {
2783 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2784 if (ret)
2785 break;
eae460b6 2786 }
cf04eee8
DW		 2787 mutex_unlock(&adev->physical_node_lock);
2788 if (ret)
2789 return ret;
62edf5dc 2790 }
2c2e2c38
FY		 2791
2792 return 0;
2793}
2794
ffebeb46
JL		 2795static void intel_iommu_init_qi(struct intel_iommu *iommu)
2796{
2797 /*
2798 * Start from the sane iommu hardware state.
2799 * If the queued invalidation is already initialized by us
2800 * (for example, while enabling interrupt-remapping) then
2801 * we got the things already rolling from a sane state.
2802 */
2803 if (!iommu->qi) {
2804 /*
2805 * Clear any previous faults.
2806 */
2807 dmar_fault(-1, iommu);
2808 /*
2809 * Disable queued invalidation if supported and already enabled
2810 * before OS handover.
2811 */
2812 dmar_disable_qi(iommu);
2813 }
2814
2815 if (dmar_enable_qi(iommu)) {
2816 /*
2817 * Queued Invalidate not enabled, use Register Based Invalidate
2818 */
2819 iommu->flush.flush_context = __iommu_flush_context;
2820 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2821 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2822 iommu->name);
2823 } else {
2824 iommu->flush.flush_context = qi_flush_context;
2825 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2826 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2827 }
2828}
2829
091d42e4
JR		 2830static int copy_context_table(struct intel_iommu *iommu,
2831 struct root_entry *old_re,
2832 struct context_entry **tbl,
2833 int bus, bool ext)
2834{
2835 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2836 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2837 phys_addr_t old_ce_phys;
2838
2839 tbl_idx = ext ? bus * 2 : bus;
2840
2841 for (devfn = 0; devfn < 256; devfn++) {
2842 /* First calculate the correct index */
2843 idx = (ext ? devfn * 2 : devfn) % 256;
2844
2845 if (idx == 0) {
2846 /* First save what we may have and clean up */
2847 if (new_ce) {
2848 tbl[tbl_idx] = new_ce;
2849 __iommu_flush_cache(iommu, new_ce,
2850 VTD_PAGE_SIZE);
2851 pos = 1;
2852 }
2853
2854 if (old_ce)
2855 iounmap(old_ce);
2856
2857 ret = 0;
2858 if (devfn < 0x80)
2859 old_ce_phys = root_entry_lctp(old_re);
2860 else
2861 old_ce_phys = root_entry_uctp(old_re);
2862
2863 if (!old_ce_phys) {
2864 if (ext && devfn == 0) {
2865 /* No LCTP, try UCTP */
2866 devfn = 0x7f;
2867 continue;
2868 } else {
2869 goto out;
2870 }
2871 }
2872
2873 ret = -ENOMEM;
2874 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2875 if (!old_ce)
2876 goto out;
2877
2878 new_ce = alloc_pgtable_page(iommu->node);
2879 if (!new_ce)
2880 goto out_unmap;
2881
2882 ret = 0;
2883 }
2884
2885 /* Now copy the context entry */
2886 ce = old_ce[idx];
2887
cf484d0e 2888 if (!__context_present(&ce))
091d42e4
JR		 2889 continue;
2890
dbcd861f
JR		 2891 did = context_domain_id(&ce);
2892 if (did >= 0 && did < cap_ndoms(iommu->cap))
2893 set_bit(did, iommu->domain_ids);
2894
cf484d0e
JR		 2895 /*
2896 * We need a marker for copied context entries. This
2897 * marker needs to work for the old format as well as
2898 * for extended context entries.
2899 *
2900 * Bit 67 of the context entry is used. In the old
2901 * format this bit is available to software, in the
2902 * extended format it is the PGE bit, but PGE is ignored
2903 * by HW if PASIDs are disabled (and thus still
2904 * available).
2905 *
2906 * So disable PASIDs first and then mark the entry
2907 * copied. This means that we don't copy PASID
2908 * translations from the old kernel, but this is fine as
2909 * faults there are not fatal.
2910 */
2911 context_clear_pasid_enable(&ce);
2912 context_set_copied(&ce);
2913
091d42e4
JR		 2914 new_ce[idx] = ce;
2915 }
2916
2917 tbl[tbl_idx + pos] = new_ce;
2918
2919 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2920
2921out_unmap:
2922 iounmap(old_ce);
2923
2924out:
2925 return ret;
2926}
2927
2928static int copy_translation_tables(struct intel_iommu *iommu)
2929{
2930 struct context_entry **ctxt_tbls;
2931 struct root_entry *old_rt;
2932 phys_addr_t old_rt_phys;
2933 int ctxt_table_entries;
2934 unsigned long flags;
2935 u64 rtaddr_reg;
2936 int bus, ret;
c3361f2f 2937 bool new_ext, ext;
091d42e4
JR		 2938
2939 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2940 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2941 new_ext = !!ecap_ecs(iommu->ecap);
2942
2943 /*
2944 * The RTT bit can only be changed when translation is disabled,
2945 * but disabling translation means to open a window for data
2946 * corruption. So bail out and don't copy anything if we would
2947 * have to change the bit.
2948 */
2949 if (new_ext != ext)
2950 return -EINVAL;
091d42e4
JR		 2951
2952 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2953 if (!old_rt_phys)
2954 return -EINVAL;
2955
2956 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2957 if (!old_rt)
2958 return -ENOMEM;
2959
2960 /* This is too big for the stack - allocate it from slab */
2961 ctxt_table_entries = ext ? 512 : 256;
2962 ret = -ENOMEM;
2963 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2964 if (!ctxt_tbls)
2965 goto out_unmap;
2966
2967 for (bus = 0; bus < 256; bus++) {
2968 ret = copy_context_table(iommu, &old_rt[bus],
2969 ctxt_tbls, bus, ext);
2970 if (ret) {
2971 pr_err("%s: Failed to copy context table for bus %d\n",
2972 iommu->name, bus);
2973 continue;
2974 }
2975 }
2976
2977 spin_lock_irqsave(&iommu->lock, flags);
2978
2979 /* Context tables are copied, now write them to the root_entry table */
2980 for (bus = 0; bus < 256; bus++) {
2981 int idx = ext ? bus * 2 : bus;
2982 u64 val;
2983
2984 if (ctxt_tbls[idx]) {
2985 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2986 iommu->root_entry[bus].lo = val;
2987 }
2988
2989 if (!ext || !ctxt_tbls[idx + 1])
2990 continue;
2991
2992 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2993 iommu->root_entry[bus].hi = val;
2994 }
2995
2996 spin_unlock_irqrestore(&iommu->lock, flags);
2997
2998 kfree(ctxt_tbls);
2999
3000 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3001
3002 ret = 0;
3003
3004out_unmap:
3005 iounmap(old_rt);
3006
3007 return ret;
3008}
3009
b779260b 3010static int __init init_dmars(void)
ba395927
KA		 3011{
3012 struct dmar_drhd_unit *drhd;
3013 struct dmar_rmrr_unit *rmrr;
a87f4918 3014 bool copied_tables = false;
832bd858 3015 struct device *dev;
ba395927 3016 struct intel_iommu *iommu;
9d783ba0 3017 int i, ret;
2c2e2c38 3018
ba395927
KA		 3019 /*
3020 * for each drhd
3021 * allocate root
3022 * initialize and program root entry to not present
3023 * endfor
3024 */
3025 for_each_drhd_unit(drhd) {
5e0d2a6f 3026 /*
3027 * lock not needed as this is only incremented in the single
3028 * threaded kernel __init code path all other access are read
3029 * only
3030 */
78d8e704 3031 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3032 g_num_of_iommus++;
3033 continue;
3034 }
9f10e5bf 3035 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3036 }
3037
ffebeb46
JL		 3038 /* Preallocate enough resources for IOMMU hot-addition */
3039 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3040 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3041
d9630fe9
WH		 3042 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3043 GFP_KERNEL);
3044 if (!g_iommus) {
9f10e5bf 3045 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3046 ret = -ENOMEM;
3047 goto error;
3048 }
3049
80b20dd8 3050 deferred_flush = kzalloc(g_num_of_iommus *
3051 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3052 if (!deferred_flush) {
5e0d2a6f 3053 ret = -ENOMEM;
989d51fc 3054 goto free_g_iommus;
5e0d2a6f 3055 }
3056
7c919779 3057 for_each_active_iommu(iommu, drhd) {
d9630fe9 3058 g_iommus[iommu->seq_id] = iommu;
ba395927 3059
b63d80d1
JR		 3060 intel_iommu_init_qi(iommu);
3061
e61d98d8
SS		 3062 ret = iommu_init_domains(iommu);
3063 if (ret)
989d51fc 3064 goto free_iommu;
e61d98d8 3065
4158c2ec
JR		 3066 init_translation_status(iommu);
3067
091d42e4
JR		 3068 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3069 iommu_disable_translation(iommu);
3070 clear_translation_pre_enabled(iommu);
3071 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3072 iommu->name);
3073 }
4158c2ec 3074
ba395927
KA		 3075 /*
3076 * TBD:
3077 * we could share the same root & context tables
25985edc 3078 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3079 */
3080 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3081 if (ret)
989d51fc 3082 goto free_iommu;
5f0a7f76 3083
091d42e4
JR		 3084 if (translation_pre_enabled(iommu)) {
3085 pr_info("Translation already enabled - trying to copy translation structures\n");
3086
3087 ret = copy_translation_tables(iommu);
3088 if (ret) {
3089 /*
3090 * We found the IOMMU with translation
3091 * enabled - but failed to copy over the
3092 * old root-entry table. Try to proceed
3093 * by disabling translation now and
3094 * allocating a clean root-entry table.
3095 * This might cause DMAR faults, but
3096 * probably the dump will still succeed.
3097 */
3098 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3099 iommu->name);
3100 iommu_disable_translation(iommu);
3101 clear_translation_pre_enabled(iommu);
3102 } else {
3103 pr_info("Copied translation tables from previous kernel for %s\n",
3104 iommu->name);
a87f4918 3105 copied_tables = true;
091d42e4
JR		 3106 }
3107 }
3108
5f0a7f76
JR		 3109 iommu_flush_write_buffer(iommu);
3110 iommu_set_root_entry(iommu);
3111 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3112 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3113
4ed0d3e6 3114 if (!ecap_pass_through(iommu->ecap))
19943b0e 3115 hw_pass_through = 0;
ba395927
KA		 3116 }
3117
19943b0e 3118 if (iommu_pass_through)
e0fc7e0b
DW		 3119 iommu_identity_mapping |= IDENTMAP_ALL;
3120
d3f13810 3121#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3122 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3123#endif
e0fc7e0b 3124
86080ccc
JR		 3125 if (iommu_identity_mapping) {
3126 ret = si_domain_init(hw_pass_through);
3127 if (ret)
3128 goto free_iommu;
3129 }
3130
e0fc7e0b
DW		 3131 check_tylersburg_isoch();
3132
a87f4918
JR		 3133 /*
3134 * If we copied translations from a previous kernel in the kdump
3135 * case, we can not assign the devices to domains now, as that
3136 * would eliminate the old mappings. So skip this part and defer
3137 * the assignment to device driver initialization time.
3138 */
3139 if (copied_tables)
3140 goto domains_done;
3141
ba395927 3142 /*
19943b0e
DW		 3143 * If pass through is not set or not enabled, setup context entries for
3144 * identity mappings for rmrr, gfx, and isa and may fall back to static
3145 * identity mapping if iommu_identity_mapping is set.
ba395927 3146 */
19943b0e
DW		 3147 if (iommu_identity_mapping) {
3148 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3149 if (ret) {
9f10e5bf 3150 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3151 goto free_iommu;
ba395927
KA		 3152 }
3153 }
ba395927 3154 /*
19943b0e
DW		 3155 * For each rmrr
3156 * for each dev attached to rmrr
3157 * do
3158 * locate drhd for dev, alloc domain for dev
3159 * allocate free domain
3160 * allocate page table entries for rmrr
3161 * if context not allocated for bus
3162 * allocate and init context
3163 * set present in root table for this bus
3164 * init context with domain, translation etc
3165 * endfor
3166 * endfor
ba395927 3167 */
9f10e5bf 3168 pr_info("Setting RMRR:\n");
19943b0e 3169 for_each_rmrr_units(rmrr) {
b683b230
JL		 3170 /* some BIOS lists non-exist devices in DMAR table. */
3171 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3172 i, dev) {
0b9d9753 3173 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3174 if (ret)
9f10e5bf 3175 pr_err("Mapping reserved region failed\n");
ba395927 3176 }
4ed0d3e6 3177 }
49a0429e 3178
19943b0e
DW		 3179 iommu_prepare_isa();
3180
a87f4918
JR		 3181domains_done:
3182
ba395927
KA		 3183 /*
3184 * for each drhd
3185 * enable fault log
3186 * global invalidate context cache
3187 * global invalidate iotlb
3188 * enable translation
3189 */
7c919779 3190 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3191 if (drhd->ignored) {
3192 /*
3193 * we always have to disable PMRs or DMA may fail on
3194 * this device
3195 */
3196 if (force_on)
7c919779 3197 iommu_disable_protect_mem_regions(iommu);
ba395927 3198 continue;
51a63e67 3199 }
ba395927
KA		 3200
3201 iommu_flush_write_buffer(iommu);
3202
3460a6d9
KA		 3203 ret = dmar_set_interrupt(iommu);
3204 if (ret)
989d51fc 3205 goto free_iommu;
3460a6d9 3206
8939ddf6
JR		 3207 if (!translation_pre_enabled(iommu))
3208 iommu_enable_translation(iommu);
3209
b94996c9 3210 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3211 }
3212
3213 return 0;
989d51fc
JL		 3214
3215free_iommu:
ffebeb46
JL		 3216 for_each_active_iommu(iommu, drhd) {
3217 disable_dmar_iommu(iommu);
a868e6b7 3218 free_dmar_iommu(iommu);
ffebeb46 3219 }
9bdc531e 3220 kfree(deferred_flush);
989d51fc 3221free_g_iommus:
d9630fe9 3222 kfree(g_iommus);
989d51fc 3223error:
ba395927
KA		 3224 return ret;
3225}
3226
5a5e02a6 3227/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3228static struct iova *intel_alloc_iova(struct device *dev,
3229 struct dmar_domain *domain,
3230 unsigned long nrpages, uint64_t dma_mask)
ba395927 3231{
ba395927 3232 struct iova *iova = NULL;
ba395927 3233
875764de
DW		 3234 /* Restrict dma_mask to the width that the iommu can handle */
3235 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3236
3237 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3238 /*
3239 * First try to allocate an io virtual address in
284901a9 3240 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3241 * from higher range
ba395927 3242 */
875764de
DW		 3243 iova = alloc_iova(&domain->iovad, nrpages,
3244 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3245 if (iova)
3246 return iova;
3247 }
3248 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3249 if (unlikely(!iova)) {
9f10e5bf 3250 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3251 nrpages, dev_name(dev));
f76aec76
KA		 3252 return NULL;
3253 }
3254
3255 return iova;
3256}
3257
d4b709f4 3258static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3259{
3260 struct dmar_domain *domain;
3261 int ret;
3262
d4b709f4 3263 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3264 if (!domain) {
9f10e5bf 3265 pr_err("Allocating domain for %s failed\n",
d4b709f4 3266 dev_name(dev));
4fe05bbc 3267 return NULL;
ba395927
KA		 3268 }
3269
3270 /* make sure context mapping is ok */
d4b709f4
DW		 3271 if (unlikely(!domain_context_mapped(dev))) {
3272 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 3273 if (ret) {
9f10e5bf 3274 pr_err("Domain context map for %s failed\n",
d4b709f4 3275 dev_name(dev));
4fe05bbc 3276 return NULL;
f76aec76 3277 }
ba395927
KA		 3278 }
3279
f76aec76
KA		 3280 return domain;
3281}
3282
d4b709f4 3283static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3284{
3285 struct device_domain_info *info;
3286
3287 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3288 info = dev->archdata.iommu;
147202aa
DW		 3289 if (likely(info))
3290 return info->domain;
3291
3292 return __get_valid_domain_for_dev(dev);
3293}
3294
ecb509ec 3295/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3296static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3297{
3298 int found;
3299
3d89194a 3300 if (iommu_dummy(dev))
1e4c64c4
DW		 3301 return 1;
3302
2c2e2c38 3303 if (!iommu_identity_mapping)
1e4c64c4 3304 return 0;
2c2e2c38 3305
9b226624 3306 found = identity_mapping(dev);
2c2e2c38 3307 if (found) {
ecb509ec 3308 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3309 return 1;
3310 else {
3311 /*
3312 * 32 bit DMA is removed from si_domain and fall back
3313 * to non-identity mapping.
3314 */
bf9c9eda 3315 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3316 pr_info("32bit %s uses non-identity mapping\n",
3317 dev_name(dev));
2c2e2c38
FY		 3318 return 0;
3319 }
3320 } else {
3321 /*
3322 * In case of a detached 64 bit DMA device from vm, the device
3323 * is put into si_domain for identity mapping.
3324 */
ecb509ec 3325 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3326 int ret;
5913c9bf 3327 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3328 hw_pass_through ?
3329 CONTEXT_TT_PASS_THROUGH :
3330 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38 3331 if (!ret) {
9f10e5bf
JR		 3332 pr_info("64bit %s uses identity mapping\n",
3333 dev_name(dev));
2c2e2c38
FY		 3334 return 1;
3335 }
3336 }
3337 }
3338
1e4c64c4 3339 return 0;
2c2e2c38
FY		 3340}
3341
5040a918 3342static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3343 size_t size, int dir, u64 dma_mask)
f76aec76 3344{
f76aec76 3345 struct dmar_domain *domain;
5b6985ce 3346 phys_addr_t start_paddr;
f76aec76
KA		 3347 struct iova *iova;
3348 int prot = 0;
6865f0d1 3349 int ret;
8c11e798 3350 struct intel_iommu *iommu;
33041ec0 3351 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3352
3353 BUG_ON(dir == DMA_NONE);
2c2e2c38 3354
5040a918 3355 if (iommu_no_mapping(dev))
6865f0d1 3356 return paddr;
f76aec76 3357
5040a918 3358 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3359 if (!domain)
3360 return 0;
3361
8c11e798 3362 iommu = domain_get_iommu(domain);
88cb6a74 3363 size = aligned_nrpages(paddr, size);
f76aec76 3364
5040a918 3365 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3366 if (!iova)
3367 goto error;
3368
ba395927
KA		 3369 /*
3370 * Check if DMAR supports zero-length reads on write only
3371 * mappings..
3372 */
3373 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3374 !cap_zlr(iommu->cap))
ba395927
KA		 3375 prot |= DMA_PTE_READ;
3376 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3377 prot |= DMA_PTE_WRITE;
3378 /*
6865f0d1 3379 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3380 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3381 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3382 * is not a big problem
3383 */
0ab36de2 3384 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3385 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3386 if (ret)
3387 goto error;
3388
1f0ef2aa
DW		 3389 /* it's a non-present to present mapping. Only flush if caching mode */
3390 if (cap_caching_mode(iommu->cap))
ea8ea460 3391 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3392 else
8c11e798 3393 iommu_flush_write_buffer(iommu);
f76aec76 3394
03d6a246
DW		 3395 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3396 start_paddr += paddr & ~PAGE_MASK;
3397 return start_paddr;
ba395927 3398
ba395927 3399error:
f76aec76
KA		 3400 if (iova)
3401 __free_iova(&domain->iovad, iova);
9f10e5bf 3402 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3403 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3404 return 0;
3405}
3406
ffbbef5c
FT		 3407static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3408 unsigned long offset, size_t size,
3409 enum dma_data_direction dir,
3410 struct dma_attrs *attrs)
bb9e6d65 3411{
ffbbef5c 3412 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3413 dir, *dev->dma_mask);
bb9e6d65
FT		 3414}
3415
5e0d2a6f 3416static void flush_unmaps(void)
3417{
80b20dd8 3418 int i, j;
5e0d2a6f 3419
5e0d2a6f 3420 timer_on = 0;
3421
3422 /* just flush them all */
3423 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3424 struct intel_iommu *iommu = g_iommus[i];
3425 if (!iommu)
3426 continue;
c42d9f32 3427
9dd2fe89
YZ		 3428 if (!deferred_flush[i].next)
3429 continue;
3430
78d5f0f5
NA		 3431 /* In caching mode, global flushes turn emulation expensive */
3432 if (!cap_caching_mode(iommu->cap))
3433 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3434 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3435 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3436 unsigned long mask;
3437 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3438 struct dmar_domain *domain = deferred_flush[i].domain[j];
3439
3440 /* On real hardware multiple invalidations are expensive */
3441 if (cap_caching_mode(iommu->cap))
3442 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3443 iova->pfn_lo, iova_size(iova),
ea8ea460 3444 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3445 else {
a156ef99 3446 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3447 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3448 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3449 }
93a23a72 3450 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3451 if (deferred_flush[i].freelist[j])
3452 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3453 }
9dd2fe89 3454 deferred_flush[i].next = 0;
5e0d2a6f 3455 }
3456
5e0d2a6f 3457 list_size = 0;
5e0d2a6f 3458}
3459
3460static void flush_unmaps_timeout(unsigned long data)
3461{
80b20dd8 3462 unsigned long flags;
3463
3464 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3465 flush_unmaps();
80b20dd8 3466 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3467}
3468
ea8ea460 3469static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3470{
3471 unsigned long flags;
80b20dd8 3472 int next, iommu_id;
8c11e798 3473 struct intel_iommu *iommu;
5e0d2a6f 3474
3475 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3476 if (list_size == HIGH_WATER_MARK)
3477 flush_unmaps();
3478
8c11e798
WH		 3479 iommu = domain_get_iommu(dom);
3480 iommu_id = iommu->seq_id;
c42d9f32 3481
80b20dd8 3482 next = deferred_flush[iommu_id].next;
3483 deferred_flush[iommu_id].domain[next] = dom;
3484 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3485 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3486 deferred_flush[iommu_id].next++;
5e0d2a6f 3487
3488 if (!timer_on) {
3489 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3490 timer_on = 1;
3491 }
3492 list_size++;
3493 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3494}
3495
d41a4adb 3496static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3497{
f76aec76 3498 struct dmar_domain *domain;
d794dc9b 3499 unsigned long start_pfn, last_pfn;
ba395927 3500 struct iova *iova;
8c11e798 3501 struct intel_iommu *iommu;
ea8ea460 3502 struct page *freelist;
ba395927 3503
73676832 3504 if (iommu_no_mapping(dev))
f76aec76 3505 return;
2c2e2c38 3506
1525a29a 3507 domain = find_domain(dev);
ba395927
KA		 3508 BUG_ON(!domain);
3509
8c11e798
WH		 3510 iommu = domain_get_iommu(domain);
3511
ba395927 3512 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3513 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3514 (unsigned long long)dev_addr))
ba395927 3515 return;
ba395927 3516
d794dc9b
DW		 3517 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3518 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3519
d794dc9b 3520 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3521 dev_name(dev), start_pfn, last_pfn);
ba395927 3522
ea8ea460 3523 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3524
5e0d2a6f 3525 if (intel_iommu_strict) {
03d6a246 3526 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3527 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3528 /* free iova */
3529 __free_iova(&domain->iovad, iova);
ea8ea460 3530 dma_free_pagelist(freelist);
5e0d2a6f 3531 } else {
ea8ea460 3532 add_unmap(domain, iova, freelist);
5e0d2a6f 3533 /*
3534 * queue up the release of the unmap to save the 1/6th of the
3535 * cpu used up by the iotlb flush operation...
3536 */
5e0d2a6f 3537 }
ba395927
KA		 3538}
3539
d41a4adb
JL		 3540static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3541 size_t size, enum dma_data_direction dir,
3542 struct dma_attrs *attrs)
3543{
3544 intel_unmap(dev, dev_addr);
3545}
3546
5040a918 3547static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3548 dma_addr_t *dma_handle, gfp_t flags,
3549 struct dma_attrs *attrs)
ba395927 3550{
36746436 3551 struct page *page = NULL;
ba395927
KA		 3552 int order;
3553
5b6985ce 3554 size = PAGE_ALIGN(size);
ba395927 3555 order = get_order(size);
e8bb910d 3556
5040a918 3557 if (!iommu_no_mapping(dev))
e8bb910d 3558 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3559 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3560 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3561 flags |= GFP_DMA;
3562 else
3563 flags |= GFP_DMA32;
3564 }
ba395927 3565
36746436
AM		 3566 if (flags & __GFP_WAIT) {
3567 unsigned int count = size >> PAGE_SHIFT;
3568
3569 page = dma_alloc_from_contiguous(dev, count, order);
3570 if (page && iommu_no_mapping(dev) &&
3571 page_to_phys(page) + size > dev->coherent_dma_mask) {
3572 dma_release_from_contiguous(dev, page, count);
3573 page = NULL;
3574 }
3575 }
3576
3577 if (!page)
3578 page = alloc_pages(flags, order);
3579 if (!page)
ba395927 3580 return NULL;
36746436 3581 memset(page_address(page), 0, size);
ba395927 3582
36746436 3583 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3584 DMA_BIDIRECTIONAL,
5040a918 3585 dev->coherent_dma_mask);
ba395927 3586 if (*dma_handle)
36746436
AM		 3587 return page_address(page);
3588 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3589 __free_pages(page, order);
3590
ba395927
KA		 3591 return NULL;
3592}
3593
5040a918 3594static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3595 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3596{
3597 int order;
36746436 3598 struct page *page = virt_to_page(vaddr);
ba395927 3599
5b6985ce 3600 size = PAGE_ALIGN(size);
ba395927
KA		 3601 order = get_order(size);
3602
d41a4adb 3603 intel_unmap(dev, dma_handle);
36746436
AM		 3604 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3605 __free_pages(page, order);
ba395927
KA		 3606}
3607
5040a918 3608static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3609 int nelems, enum dma_data_direction dir,
3610 struct dma_attrs *attrs)
ba395927 3611{
d41a4adb 3612 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3613}
3614
ba395927 3615static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3616 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3617{
3618 int i;
c03ab37c 3619 struct scatterlist *sg;
ba395927 3620
c03ab37c 3621 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3622 BUG_ON(!sg_page(sg));
4cf2e75d 3623 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3624 sg->dma_length = sg->length;
ba395927
KA		 3625 }
3626 return nelems;
3627}
3628
5040a918 3629static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3630 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3631{
ba395927 3632 int i;
ba395927 3633 struct dmar_domain *domain;
f76aec76
KA		 3634 size_t size = 0;
3635 int prot = 0;
f76aec76
KA		 3636 struct iova *iova = NULL;
3637 int ret;
c03ab37c 3638 struct scatterlist *sg;
b536d24d 3639 unsigned long start_vpfn;
8c11e798 3640 struct intel_iommu *iommu;
ba395927
KA		 3641
3642 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3643 if (iommu_no_mapping(dev))
3644 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3645
5040a918 3646 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3647 if (!domain)
3648 return 0;
3649
8c11e798
WH		 3650 iommu = domain_get_iommu(domain);
3651
b536d24d 3652 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3653 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3654
5040a918
DW		 3655 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3656 *dev->dma_mask);
f76aec76 3657 if (!iova) {
c03ab37c 3658 sglist->dma_length = 0;
f76aec76
KA		 3659 return 0;
3660 }
3661
3662 /*
3663 * Check if DMAR supports zero-length reads on write only
3664 * mappings..
3665 */
3666 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3667 !cap_zlr(iommu->cap))
f76aec76
KA		 3668 prot |= DMA_PTE_READ;
3669 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3670 prot |= DMA_PTE_WRITE;
3671
b536d24d 3672 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3673
f532959b 3674 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3675 if (unlikely(ret)) {
e1605495
DW		 3676 dma_pte_free_pagetable(domain, start_vpfn,
3677 start_vpfn + size - 1);
e1605495
DW		 3678 __free_iova(&domain->iovad, iova);
3679 return 0;
ba395927
KA		 3680 }
3681
1f0ef2aa
DW		 3682 /* it's a non-present to present mapping. Only flush if caching mode */
3683 if (cap_caching_mode(iommu->cap))
ea8ea460 3684 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3685 else
8c11e798 3686 iommu_flush_write_buffer(iommu);
1f0ef2aa 3687
ba395927
KA		 3688 return nelems;
3689}
3690
dfb805e8
FT		 3691static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3692{
3693 return !dma_addr;
3694}
3695
160c1d8e 3696struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3697 .alloc = intel_alloc_coherent,
3698 .free = intel_free_coherent,
ba395927
KA		 3699 .map_sg = intel_map_sg,
3700 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3701 .map_page = intel_map_page,
3702 .unmap_page = intel_unmap_page,
dfb805e8 3703 .mapping_error = intel_mapping_error,
ba395927
KA		 3704};
3705
3706static inline int iommu_domain_cache_init(void)
3707{
3708 int ret = 0;
3709
3710 iommu_domain_cache = kmem_cache_create("iommu_domain",
3711 sizeof(struct dmar_domain),
3712 0,
3713 SLAB_HWCACHE_ALIGN,
3714
3715 NULL);
3716 if (!iommu_domain_cache) {
9f10e5bf 3717 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3718 ret = -ENOMEM;
3719 }
3720
3721 return ret;
3722}
3723
3724static inline int iommu_devinfo_cache_init(void)
3725{
3726 int ret = 0;
3727
3728 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3729 sizeof(struct device_domain_info),
3730 0,
3731 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3732 NULL);
3733 if (!iommu_devinfo_cache) {
9f10e5bf 3734 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3735 ret = -ENOMEM;
3736 }
3737
3738 return ret;
3739}
3740
ba395927
KA		 3741static int __init iommu_init_mempool(void)
3742{
3743 int ret;
3744 ret = iommu_iova_cache_init();
3745 if (ret)
3746 return ret;
3747
3748 ret = iommu_domain_cache_init();
3749 if (ret)
3750 goto domain_error;
3751
3752 ret = iommu_devinfo_cache_init();
3753 if (!ret)
3754 return ret;
3755
3756 kmem_cache_destroy(iommu_domain_cache);
3757domain_error:
85b45456 3758 iommu_iova_cache_destroy();
ba395927
KA		 3759
3760 return -ENOMEM;
3761}
3762
3763static void __init iommu_exit_mempool(void)
3764{
3765 kmem_cache_destroy(iommu_devinfo_cache);
3766 kmem_cache_destroy(iommu_domain_cache);
85b45456 3767 iommu_iova_cache_destroy();
ba395927
KA		 3768}
3769
556ab45f
DW		 3770static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3771{
3772 struct dmar_drhd_unit *drhd;
3773 u32 vtbar;
3774 int rc;
3775
3776 /* We know that this device on this chipset has its own IOMMU.
3777 * If we find it under a different IOMMU, then the BIOS is lying
3778 * to us. Hope that the IOMMU for this device is actually
3779 * disabled, and it needs no translation...
3780 */
3781 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3782 if (rc) {
3783 /* "can't" happen */
3784 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3785 return;
3786 }
3787 vtbar &= 0xffff0000;
3788
3789 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3790 drhd = dmar_find_matched_drhd_unit(pdev);
3791 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3792 TAINT_FIRMWARE_WORKAROUND,
3793 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3794 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3795}
3796DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3797
ba395927
KA		 3798static void __init init_no_remapping_devices(void)
3799{
3800 struct dmar_drhd_unit *drhd;
832bd858 3801 struct device *dev;
b683b230 3802 int i;
ba395927
KA		 3803
3804 for_each_drhd_unit(drhd) {
3805 if (!drhd->include_all) {
b683b230
JL		 3806 for_each_active_dev_scope(drhd->devices,
3807 drhd->devices_cnt, i, dev)
3808 break;
832bd858 3809 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3810 if (i == drhd->devices_cnt)
3811 drhd->ignored = 1;
3812 }
3813 }
3814
7c919779 3815 for_each_active_drhd_unit(drhd) {
7c919779 3816 if (drhd->include_all)
ba395927
KA		 3817 continue;
3818
b683b230
JL		 3819 for_each_active_dev_scope(drhd->devices,
3820 drhd->devices_cnt, i, dev)
832bd858 3821 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3822 break;
ba395927
KA		 3823 if (i < drhd->devices_cnt)
3824 continue;
3825
c0771df8
DW		 3826 /* This IOMMU has *only* gfx devices. Either bypass it or
3827 set the gfx_mapped flag, as appropriate */
3828 if (dmar_map_gfx) {
3829 intel_iommu_gfx_mapped = 1;
3830 } else {
3831 drhd->ignored = 1;
b683b230
JL		 3832 for_each_active_dev_scope(drhd->devices,
3833 drhd->devices_cnt, i, dev)
832bd858 3834 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3835 }
3836 }
3837}
3838
f59c7b69
FY		 3839#ifdef CONFIG_SUSPEND
3840static int init_iommu_hw(void)
3841{
3842 struct dmar_drhd_unit *drhd;
3843 struct intel_iommu *iommu = NULL;
3844
3845 for_each_active_iommu(iommu, drhd)
3846 if (iommu->qi)
3847 dmar_reenable_qi(iommu);
3848
b779260b
JC		 3849 for_each_iommu(iommu, drhd) {
3850 if (drhd->ignored) {
3851 /*
3852 * we always have to disable PMRs or DMA may fail on
3853 * this device
3854 */
3855 if (force_on)
3856 iommu_disable_protect_mem_regions(iommu);
3857 continue;
3858 }
3859
f59c7b69
FY		 3860 iommu_flush_write_buffer(iommu);
3861
3862 iommu_set_root_entry(iommu);
3863
3864 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3865 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3866 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3867 iommu_enable_translation(iommu);
b94996c9 3868 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3869 }
3870
3871 return 0;
3872}
3873
3874static void iommu_flush_all(void)
3875{
3876 struct dmar_drhd_unit *drhd;
3877 struct intel_iommu *iommu;
3878
3879 for_each_active_iommu(iommu, drhd) {
3880 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3881 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3882 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3883 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3884 }
3885}
3886
134fac3f 3887static int iommu_suspend(void)
f59c7b69
FY		 3888{
3889 struct dmar_drhd_unit *drhd;
3890 struct intel_iommu *iommu = NULL;
3891 unsigned long flag;
3892
3893 for_each_active_iommu(iommu, drhd) {
3894 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3895 GFP_ATOMIC);
3896 if (!iommu->iommu_state)
3897 goto nomem;
3898 }
3899
3900 iommu_flush_all();
3901
3902 for_each_active_iommu(iommu, drhd) {
3903 iommu_disable_translation(iommu);
3904
1f5b3c3f 3905 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3906
3907 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3908 readl(iommu->reg + DMAR_FECTL_REG);
3909 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3910 readl(iommu->reg + DMAR_FEDATA_REG);
3911 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3912 readl(iommu->reg + DMAR_FEADDR_REG);
3913 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3914 readl(iommu->reg + DMAR_FEUADDR_REG);
3915
1f5b3c3f 3916 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3917 }
3918 return 0;
3919
3920nomem:
3921 for_each_active_iommu(iommu, drhd)
3922 kfree(iommu->iommu_state);
3923
3924 return -ENOMEM;
3925}
3926
134fac3f 3927static void iommu_resume(void)
f59c7b69
FY		 3928{
3929 struct dmar_drhd_unit *drhd;
3930 struct intel_iommu *iommu = NULL;
3931 unsigned long flag;
3932
3933 if (init_iommu_hw()) {
b779260b
JC		 3934 if (force_on)
3935 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3936 else
3937 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3938 return;
f59c7b69
FY		 3939 }
3940
3941 for_each_active_iommu(iommu, drhd) {
3942
1f5b3c3f 3943 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3944
3945 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3946 iommu->reg + DMAR_FECTL_REG);
3947 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3948 iommu->reg + DMAR_FEDATA_REG);
3949 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3950 iommu->reg + DMAR_FEADDR_REG);
3951 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3952 iommu->reg + DMAR_FEUADDR_REG);
3953
1f5b3c3f 3954 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3955 }
3956
3957 for_each_active_iommu(iommu, drhd)
3958 kfree(iommu->iommu_state);
f59c7b69
FY		 3959}
3960
134fac3f 3961static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3962 .resume = iommu_resume,
3963 .suspend = iommu_suspend,
3964};
3965
134fac3f 3966static void __init init_iommu_pm_ops(void)
f59c7b69 3967{
134fac3f 3968 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3969}
3970
3971#else
99592ba4 3972static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3973#endif /* CONFIG_PM */
3974
318fe7df 3975
c2a0b538 3976int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3977{
3978 struct acpi_dmar_reserved_memory *rmrr;
3979 struct dmar_rmrr_unit *rmrru;
3980
3981 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3982 if (!rmrru)
3983 return -ENOMEM;
3984
3985 rmrru->hdr = header;
3986 rmrr = (struct acpi_dmar_reserved_memory *)header;
3987 rmrru->base_address = rmrr->base_address;
3988 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3989 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3990 ((void *)rmrr) + rmrr->header.length,
3991 &rmrru->devices_cnt);
3992 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3993 kfree(rmrru);
3994 return -ENOMEM;
3995 }
318fe7df 3996
2e455289 3997 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3998
2e455289 3999 return 0;
318fe7df
SS		 4000}
4001
6b197249
JL		 4002static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4003{
4004 struct dmar_atsr_unit *atsru;
4005 struct acpi_dmar_atsr *tmp;
4006
4007 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4008 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4009 if (atsr->segment != tmp->segment)
4010 continue;
4011 if (atsr->header.length != tmp->header.length)
4012 continue;
4013 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4014 return atsru;
4015 }
4016
4017 return NULL;
4018}
4019
4020int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4021{
4022 struct acpi_dmar_atsr *atsr;
4023 struct dmar_atsr_unit *atsru;
4024
6b197249
JL		 4025 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4026 return 0;
4027
318fe7df 4028 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4029 atsru = dmar_find_atsr(atsr);
4030 if (atsru)
4031 return 0;
4032
4033 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4034 if (!atsru)
4035 return -ENOMEM;
4036
6b197249
JL		 4037 /*
4038 * If memory is allocated from slab by ACPI _DSM method, we need to
4039 * copy the memory content because the memory buffer will be freed
4040 * on return.
4041 */
4042 atsru->hdr = (void *)(atsru + 1);
4043 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4044 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4045 if (!atsru->include_all) {
4046 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4047 (void *)atsr + atsr->header.length,
4048 &atsru->devices_cnt);
4049 if (atsru->devices_cnt && atsru->devices == NULL) {
4050 kfree(atsru);
4051 return -ENOMEM;
4052 }
4053 }
318fe7df 4054
0e242612 4055 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4056
4057 return 0;
4058}
4059
9bdc531e
JL		 4060static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4061{
4062 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4063 kfree(atsru);
4064}
4065
6b197249
JL		 4066int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4067{
4068 struct acpi_dmar_atsr *atsr;
4069 struct dmar_atsr_unit *atsru;
4070
4071 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4072 atsru = dmar_find_atsr(atsr);
4073 if (atsru) {
4074 list_del_rcu(&atsru->list);
4075 synchronize_rcu();
4076 intel_iommu_free_atsr(atsru);
4077 }
4078
4079 return 0;
4080}
4081
4082int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4083{
4084 int i;
4085 struct device *dev;
4086 struct acpi_dmar_atsr *atsr;
4087 struct dmar_atsr_unit *atsru;
4088
4089 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4090 atsru = dmar_find_atsr(atsr);
4091 if (!atsru)
4092 return 0;
4093
4094 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4095 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4096 i, dev)
4097 return -EBUSY;
4098
4099 return 0;
4100}
4101
ffebeb46
JL		 4102static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4103{
4104 int sp, ret = 0;
4105 struct intel_iommu *iommu = dmaru->iommu;
4106
4107 if (g_iommus[iommu->seq_id])
4108 return 0;
4109
4110 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4111 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4112 iommu->name);
4113 return -ENXIO;
4114 }
4115 if (!ecap_sc_support(iommu->ecap) &&
4116 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4117 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4118 iommu->name);
4119 return -ENXIO;
4120 }
4121 sp = domain_update_iommu_superpage(iommu) - 1;
4122 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4123 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4124 iommu->name);
4125 return -ENXIO;
4126 }
4127
4128 /*
4129 * Disable translation if already enabled prior to OS handover.
4130 */
4131 if (iommu->gcmd & DMA_GCMD_TE)
4132 iommu_disable_translation(iommu);
4133
4134 g_iommus[iommu->seq_id] = iommu;
4135 ret = iommu_init_domains(iommu);
4136 if (ret == 0)
4137 ret = iommu_alloc_root_entry(iommu);
4138 if (ret)
4139 goto out;
4140
4141 if (dmaru->ignored) {
4142 /*
4143 * we always have to disable PMRs or DMA may fail on this device
4144 */
4145 if (force_on)
4146 iommu_disable_protect_mem_regions(iommu);
4147 return 0;
4148 }
4149
4150 intel_iommu_init_qi(iommu);
4151 iommu_flush_write_buffer(iommu);
4152 ret = dmar_set_interrupt(iommu);
4153 if (ret)
4154 goto disable_iommu;
4155
4156 iommu_set_root_entry(iommu);
4157 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4158 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4159 iommu_enable_translation(iommu);
4160
4161 if (si_domain) {
4162 ret = iommu_attach_domain(si_domain, iommu);
4163 if (ret < 0 || si_domain->id != ret)
4164 goto disable_iommu;
4165 domain_attach_iommu(si_domain, iommu);
4166 }
4167
4168 iommu_disable_protect_mem_regions(iommu);
4169 return 0;
4170
4171disable_iommu:
4172 disable_dmar_iommu(iommu);
4173out:
4174 free_dmar_iommu(iommu);
4175 return ret;
4176}
4177
6b197249
JL		 4178int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4179{
ffebeb46
JL		 4180 int ret = 0;
4181 struct intel_iommu *iommu = dmaru->iommu;
4182
4183 if (!intel_iommu_enabled)
4184 return 0;
4185 if (iommu == NULL)
4186 return -EINVAL;
4187
4188 if (insert) {
4189 ret = intel_iommu_add(dmaru);
4190 } else {
4191 disable_dmar_iommu(iommu);
4192 free_dmar_iommu(iommu);
4193 }
4194
4195 return ret;
6b197249
JL		 4196}
4197
9bdc531e
JL		 4198static void intel_iommu_free_dmars(void)
4199{
4200 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4201 struct dmar_atsr_unit *atsru, *atsr_n;
4202
4203 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4204 list_del(&rmrru->list);
4205 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4206 kfree(rmrru);
318fe7df
SS		 4207 }
4208
9bdc531e
JL		 4209 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4210 list_del(&atsru->list);
4211 intel_iommu_free_atsr(atsru);
4212 }
318fe7df
SS		 4213}
4214
4215int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4216{
b683b230 4217 int i, ret = 1;
318fe7df 4218 struct pci_bus *bus;
832bd858
DW		 4219 struct pci_dev *bridge = NULL;
4220 struct device *tmp;
318fe7df
SS		 4221 struct acpi_dmar_atsr *atsr;
4222 struct dmar_atsr_unit *atsru;
4223
4224 dev = pci_physfn(dev);
318fe7df 4225 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4226 bridge = bus->self;
318fe7df 4227 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4228 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4229 return 0;
b5f82ddf 4230 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4231 break;
318fe7df 4232 }
b5f82ddf
JL		 4233 if (!bridge)
4234 return 0;
318fe7df 4235
0e242612 4236 rcu_read_lock();
b5f82ddf
JL		 4237 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4238 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4239 if (atsr->segment != pci_domain_nr(dev->bus))
4240 continue;
4241
b683b230 4242 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4243 if (tmp == &bridge->dev)
b683b230 4244 goto out;
b5f82ddf
JL		 4245
4246 if (atsru->include_all)
b683b230 4247 goto out;
b5f82ddf 4248 }
b683b230
JL		 4249 ret = 0;
4250out:
0e242612 4251 rcu_read_unlock();
318fe7df 4252
b683b230 4253 return ret;
318fe7df
SS		 4254}
4255
59ce0515
JL		 4256int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4257{
4258 int ret = 0;
4259 struct dmar_rmrr_unit *rmrru;
4260 struct dmar_atsr_unit *atsru;
4261 struct acpi_dmar_atsr *atsr;
4262 struct acpi_dmar_reserved_memory *rmrr;
4263
4264 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4265 return 0;
4266
4267 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4268 rmrr = container_of(rmrru->hdr,
4269 struct acpi_dmar_reserved_memory, header);
4270 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4271 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4272 ((void *)rmrr) + rmrr->header.length,
4273 rmrr->segment, rmrru->devices,
4274 rmrru->devices_cnt);
27e24950 4275 if(ret < 0)
59ce0515
JL		 4276 return ret;
4277 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4278 dmar_remove_dev_scope(info, rmrr->segment,
4279 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4280 }
4281 }
4282
4283 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4284 if (atsru->include_all)
4285 continue;
4286
4287 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4288 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4289 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4290 (void *)atsr + atsr->header.length,
4291 atsr->segment, atsru->devices,
4292 atsru->devices_cnt);
4293 if (ret > 0)
4294 break;
4295 else if(ret < 0)
4296 return ret;
4297 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4298 if (dmar_remove_dev_scope(info, atsr->segment,
4299 atsru->devices, atsru->devices_cnt))
4300 break;
4301 }
4302 }
4303
4304 return 0;
4305}
4306
99dcaded
FY		 4307/*
4308 * Here we only respond to action of unbound device from driver.
4309 *
4310 * Added device is not attached to its DMAR domain here yet. That will happen
4311 * when mapping the device to iova.
4312 */
4313static int device_notifier(struct notifier_block *nb,
4314 unsigned long action, void *data)
4315{
4316 struct device *dev = data;
99dcaded
FY		 4317 struct dmar_domain *domain;
4318
3d89194a 4319 if (iommu_dummy(dev))
44cd613c
DW		 4320 return 0;
4321
1196c2fb 4322 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4323 return 0;
4324
1525a29a 4325 domain = find_domain(dev);
99dcaded
FY		 4326 if (!domain)
4327 return 0;
4328
3a5670e8 4329 down_read(&dmar_global_lock);
bf9c9eda 4330 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4331 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4332 domain_exit(domain);
3a5670e8 4333 up_read(&dmar_global_lock);
a97590e5 4334
99dcaded
FY		 4335 return 0;
4336}
4337
4338static struct notifier_block device_nb = {
4339 .notifier_call = device_notifier,
4340};
4341
75f05569
JL		 4342static int intel_iommu_memory_notifier(struct notifier_block *nb,
4343 unsigned long val, void *v)
4344{
4345 struct memory_notify *mhp = v;
4346 unsigned long long start, end;
4347 unsigned long start_vpfn, last_vpfn;
4348
4349 switch (val) {
4350 case MEM_GOING_ONLINE:
4351 start = mhp->start_pfn << PAGE_SHIFT;
4352 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4353 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4354 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4355 start, end);
4356 return NOTIFY_BAD;
4357 }
4358 break;
4359
4360 case MEM_OFFLINE:
4361 case MEM_CANCEL_ONLINE:
4362 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4363 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4364 while (start_vpfn <= last_vpfn) {
4365 struct iova *iova;
4366 struct dmar_drhd_unit *drhd;
4367 struct intel_iommu *iommu;
ea8ea460 4368 struct page *freelist;
75f05569
JL		 4369
4370 iova = find_iova(&si_domain->iovad, start_vpfn);
4371 if (iova == NULL) {
9f10e5bf 4372 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4373 start_vpfn);
4374 break;
4375 }
4376
4377 iova = split_and_remove_iova(&si_domain->iovad, iova,
4378 start_vpfn, last_vpfn);
4379 if (iova == NULL) {
9f10e5bf 4380 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4381 start_vpfn, last_vpfn);
4382 return NOTIFY_BAD;
4383 }
4384
ea8ea460
DW		 4385 freelist = domain_unmap(si_domain, iova->pfn_lo,
4386 iova->pfn_hi);
4387
75f05569
JL		 4388 rcu_read_lock();
4389 for_each_active_iommu(iommu, drhd)
4390 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4391 iova->pfn_lo, iova_size(iova),
ea8ea460 4392 !freelist, 0);
75f05569 4393 rcu_read_unlock();
ea8ea460 4394 dma_free_pagelist(freelist);
75f05569
JL		 4395
4396 start_vpfn = iova->pfn_hi + 1;
4397 free_iova_mem(iova);
4398 }
4399 break;
4400 }
4401
4402 return NOTIFY_OK;
4403}
4404
4405static struct notifier_block intel_iommu_memory_nb = {
4406 .notifier_call = intel_iommu_memory_notifier,
4407 .priority = 0
4408};
4409
a5459cfe
AW		 4410
4411static ssize_t intel_iommu_show_version(struct device *dev,
4412 struct device_attribute *attr,
4413 char *buf)
4414{
4415 struct intel_iommu *iommu = dev_get_drvdata(dev);
4416 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4417 return sprintf(buf, "%d:%d\n",
4418 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4419}
4420static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4421
4422static ssize_t intel_iommu_show_address(struct device *dev,
4423 struct device_attribute *attr,
4424 char *buf)
4425{
4426 struct intel_iommu *iommu = dev_get_drvdata(dev);
4427 return sprintf(buf, "%llx\n", iommu->reg_phys);
4428}
4429static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4430
4431static ssize_t intel_iommu_show_cap(struct device *dev,
4432 struct device_attribute *attr,
4433 char *buf)
4434{
4435 struct intel_iommu *iommu = dev_get_drvdata(dev);
4436 return sprintf(buf, "%llx\n", iommu->cap);
4437}
4438static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4439
4440static ssize_t intel_iommu_show_ecap(struct device *dev,
4441 struct device_attribute *attr,
4442 char *buf)
4443{
4444 struct intel_iommu *iommu = dev_get_drvdata(dev);
4445 return sprintf(buf, "%llx\n", iommu->ecap);
4446}
4447static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4448
4449static struct attribute *intel_iommu_attrs[] = {
4450 &dev_attr_version.attr,
4451 &dev_attr_address.attr,
4452 &dev_attr_cap.attr,
4453 &dev_attr_ecap.attr,
4454 NULL,
4455};
4456
4457static struct attribute_group intel_iommu_group = {
4458 .name = "intel-iommu",
4459 .attrs = intel_iommu_attrs,
4460};
4461
4462const struct attribute_group *intel_iommu_groups[] = {
4463 &intel_iommu_group,
4464 NULL,
4465};
4466
ba395927
KA		 4467int __init intel_iommu_init(void)
4468{
9bdc531e 4469 int ret = -ENODEV;
3a93c841 4470 struct dmar_drhd_unit *drhd;
7c919779 4471 struct intel_iommu *iommu;
ba395927 4472
a59b50e9
JC		 4473 /* VT-d is required for a TXT/tboot launch, so enforce that */
4474 force_on = tboot_force_iommu();
4475
3a5670e8
JL		 4476 if (iommu_init_mempool()) {
4477 if (force_on)
4478 panic("tboot: Failed to initialize iommu memory\n");
4479 return -ENOMEM;
4480 }
4481
4482 down_write(&dmar_global_lock);
a59b50e9
JC		 4483 if (dmar_table_init()) {
4484 if (force_on)
4485 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4486 goto out_free_dmar;
a59b50e9 4487 }
ba395927 4488
c2c7286a 4489 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4490 if (force_on)
4491 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4492 goto out_free_dmar;
a59b50e9 4493 }
1886e8a9 4494
75f1cdf1 4495 if (no_iommu || dmar_disabled)
9bdc531e 4496 goto out_free_dmar;
2ae21010 4497
318fe7df 4498 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4499 pr_info("No RMRR found\n");
318fe7df
SS		 4500
4501 if (list_empty(&dmar_atsr_units))
9f10e5bf 4502 pr_info("No ATSR found\n");
318fe7df 4503
51a63e67
JC		 4504 if (dmar_init_reserved_ranges()) {
4505 if (force_on)
4506 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4507 goto out_free_reserved_range;
51a63e67 4508 }
ba395927
KA		 4509
4510 init_no_remapping_devices();
4511
b779260b 4512 ret = init_dmars();
ba395927 4513 if (ret) {
a59b50e9
JC		 4514 if (force_on)
4515 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4516 pr_err("Initialization failed\n");
9bdc531e 4517 goto out_free_reserved_range;
ba395927 4518 }
3a5670e8 4519 up_write(&dmar_global_lock);
9f10e5bf 4520 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4521
5e0d2a6f 4522 init_timer(&unmap_timer);
75f1cdf1
FT		 4523#ifdef CONFIG_SWIOTLB
4524 swiotlb = 0;
4525#endif
19943b0e 4526 dma_ops = &intel_dma_ops;
4ed0d3e6 4527
134fac3f 4528 init_iommu_pm_ops();
a8bcbb0d 4529
a5459cfe
AW		 4530 for_each_active_iommu(iommu, drhd)
4531 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4532 intel_iommu_groups,
4533 iommu->name);
4534
4236d97d 4535 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4536 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4537 if (si_domain && !hw_pass_through)
4538 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4539
8bc1f85c
ED		 4540 intel_iommu_enabled = 1;
4541
ba395927 4542 return 0;
9bdc531e
JL		 4543
4544out_free_reserved_range:
4545 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4546out_free_dmar:
4547 intel_iommu_free_dmars();
3a5670e8
JL		 4548 up_write(&dmar_global_lock);
4549 iommu_exit_mempool();
9bdc531e 4550 return ret;
ba395927 4551}
e820482c 4552
579305f7
AW		 4553static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4554{
4555 struct intel_iommu *iommu = opaque;
4556
4557 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4558 return 0;
4559}
4560
4561/*
4562 * NB - intel-iommu lacks any sort of reference counting for the users of
4563 * dependent devices. If multiple endpoints have intersecting dependent
4564 * devices, unbinding the driver from any one of them will possibly leave
4565 * the others unable to operate.
4566 */
3199aa6b 4567static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4568 struct device *dev)
3199aa6b 4569{
0bcb3e28 4570 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4571 return;
4572
579305f7 4573 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4574}
4575
2c2e2c38 4576static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4577 struct device *dev)
c7151a8d 4578{
bca2b916 4579 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4580 struct intel_iommu *iommu;
4581 unsigned long flags;
2f119c78 4582 bool found = false;
156baca8 4583 u8 bus, devfn;
c7151a8d 4584
bf9c9eda 4585 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4586 if (!iommu)
4587 return;
4588
4589 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4590 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4591 if (info->iommu == iommu && info->bus == bus &&
4592 info->devfn == devfn) {
109b9b04 4593 unlink_domain_info(info);
c7151a8d
WH		 4594 spin_unlock_irqrestore(&device_domain_lock, flags);
4595
93a23a72 4596 iommu_disable_dev_iotlb(info);
c7151a8d 4597 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4598 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4599 free_devinfo_mem(info);
4600
4601 spin_lock_irqsave(&device_domain_lock, flags);
4602
4603 if (found)
4604 break;
4605 else
4606 continue;
4607 }
4608
4609 /* if there is no other devices under the same iommu
4610 * owned by this domain, clear this iommu in iommu_bmp
4611 * update iommu count and coherency
4612 */
8bbc4410 4613 if (info->iommu == iommu)
2f119c78 4614 found = true;
c7151a8d
WH		 4615 }
4616
3e7abe25
RD		 4617 spin_unlock_irqrestore(&device_domain_lock, flags);
4618
c7151a8d 4619 if (found == 0) {
fb170fb4
JL		 4620 domain_detach_iommu(domain, iommu);
4621 if (!domain_type_is_vm_or_si(domain))
4622 iommu_detach_domain(domain, iommu);
c7151a8d 4623 }
c7151a8d
WH		 4624}
4625
2c2e2c38 4626static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4627{
4628 int adjust_width;
4629
0fb5fe87
RM		 4630 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4631 DMA_32BIT_PFN);
5e98c4b1
WH		 4632 domain_reserve_special_ranges(domain);
4633
4634 /* calculate AGAW */
4635 domain->gaw = guest_width;
4636 adjust_width = guestwidth_to_adjustwidth(guest_width);
4637 domain->agaw = width_to_agaw(adjust_width);
4638
5e98c4b1 4639 domain->iommu_coherency = 0;
c5b15255 4640 domain->iommu_snooping = 0;
6dd9a7c7 4641 domain->iommu_superpage = 0;
fe40f1e0 4642 domain->max_addr = 0;
5e98c4b1
WH		 4643
4644 /* always allocate the top pgd */
4c923d47 4645 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4646 if (!domain->pgd)
4647 return -ENOMEM;
4648 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4649 return 0;
4650}
4651
00a77deb 4652static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4653{
5d450806 4654 struct dmar_domain *dmar_domain;
00a77deb
JR		 4655 struct iommu_domain *domain;
4656
4657 if (type != IOMMU_DOMAIN_UNMANAGED)
4658 return NULL;
38717946 4659
ab8dfe25 4660 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4661 if (!dmar_domain) {
9f10e5bf 4662 pr_err("Can't allocate dmar_domain\n");
00a77deb 4663 return NULL;
38717946 4664 }
2c2e2c38 4665 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4666 pr_err("Domain initialization failed\n");
92d03cc8 4667 domain_exit(dmar_domain);
00a77deb 4668 return NULL;
38717946 4669 }
8140a95d 4670 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4671
00a77deb 4672 domain = &dmar_domain->domain;
8a0e715b
JR		 4673 domain->geometry.aperture_start = 0;
4674 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4675 domain->geometry.force_aperture = true;
4676
00a77deb 4677 return domain;
38717946 4678}
38717946 4679
00a77deb 4680static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4681{
00a77deb 4682 domain_exit(to_dmar_domain(domain));
38717946 4683}
38717946 4684
4c5478c9
JR		 4685static int intel_iommu_attach_device(struct iommu_domain *domain,
4686 struct device *dev)
38717946 4687{
00a77deb 4688 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4689 struct intel_iommu *iommu;
4690 int addr_width;
156baca8 4691 u8 bus, devfn;
faa3d6f5 4692
c875d2c1
AW		 4693 if (device_is_rmrr_locked(dev)) {
4694 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4695 return -EPERM;
4696 }
4697
7207d8f9
DW		 4698 /* normally dev is not mapped */
4699 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4700 struct dmar_domain *old_domain;
4701
1525a29a 4702 old_domain = find_domain(dev);
faa3d6f5 4703 if (old_domain) {
ab8dfe25 4704 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4705 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4706 else
4707 domain_remove_dev_info(old_domain);
62c22167
JR		 4708
4709 if (!domain_type_is_vm_or_si(old_domain) &&
4710 list_empty(&old_domain->devices))
4711 domain_exit(old_domain);
faa3d6f5
WH		 4712 }
4713 }
4714
156baca8 4715 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4716 if (!iommu)
4717 return -ENODEV;
4718
4719 /* check if this iommu agaw is sufficient for max mapped address */
4720 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4721 if (addr_width > cap_mgaw(iommu->cap))
4722 addr_width = cap_mgaw(iommu->cap);
4723
4724 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4725 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4726 "sufficient for the mapped address (%llx)\n",
a99c47a2 4727 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4728 return -EFAULT;
4729 }
a99c47a2
TL		 4730 dmar_domain->gaw = addr_width;
4731
4732 /*
4733 * Knock out extra levels of page tables if necessary
4734 */
4735 while (iommu->agaw < dmar_domain->agaw) {
4736 struct dma_pte *pte;
4737
4738 pte = dmar_domain->pgd;
4739 if (dma_pte_present(pte)) {
25cbff16
SY		 4740 dmar_domain->pgd = (struct dma_pte *)
4741 phys_to_virt(dma_pte_addr(pte));
7a661013 4742 free_pgtable_page(pte);
a99c47a2
TL		 4743 }
4744 dmar_domain->agaw--;
4745 }
fe40f1e0 4746
5913c9bf 4747 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4748}
38717946 4749
4c5478c9
JR		 4750static void intel_iommu_detach_device(struct iommu_domain *domain,
4751 struct device *dev)
38717946 4752{
00a77deb 4753 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4754}
c7151a8d 4755
b146a1c9
JR		 4756static int intel_iommu_map(struct iommu_domain *domain,
4757 unsigned long iova, phys_addr_t hpa,
5009065d 4758 size_t size, int iommu_prot)
faa3d6f5 4759{
00a77deb 4760 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4761 u64 max_addr;
dde57a21 4762 int prot = 0;
faa3d6f5 4763 int ret;
fe40f1e0 4764
dde57a21
JR		 4765 if (iommu_prot & IOMMU_READ)
4766 prot |= DMA_PTE_READ;
4767 if (iommu_prot & IOMMU_WRITE)
4768 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4769 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4770 prot |= DMA_PTE_SNP;
dde57a21 4771
163cc52c 4772 max_addr = iova + size;
dde57a21 4773 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4774 u64 end;
4775
4776 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4777 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4778 if (end < max_addr) {
9f10e5bf 4779 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4780 "sufficient for the mapped address (%llx)\n",
8954da1f 4781 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4782 return -EFAULT;
4783 }
dde57a21 4784 dmar_domain->max_addr = max_addr;
fe40f1e0 4785 }
ad051221
DW		 4786 /* Round up size to next multiple of PAGE_SIZE, if it and
4787 the low bits of hpa would take us onto the next page */
88cb6a74 4788 size = aligned_nrpages(hpa, size);
ad051221
DW		 4789 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4790 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4791 return ret;
38717946 4792}
38717946 4793
5009065d 4794static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4795 unsigned long iova, size_t size)
38717946 4796{
00a77deb 4797 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4798 struct page *freelist = NULL;
4799 struct intel_iommu *iommu;
4800 unsigned long start_pfn, last_pfn;
4801 unsigned int npages;
4802 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4803
4804 /* Cope with horrid API which requires us to unmap more than the
4805 size argument if it happens to be a large-page mapping. */
4806 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4807 BUG();
4808
4809 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4810 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4811
ea8ea460
DW		 4812 start_pfn = iova >> VTD_PAGE_SHIFT;
4813 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4814
4815 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4816
4817 npages = last_pfn - start_pfn + 1;
4818
4819 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4820 iommu = g_iommus[iommu_id];
4821
4822 /*
4823 * find bit position of dmar_domain
4824 */
4825 ndomains = cap_ndoms(iommu->cap);
4826 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4827 if (iommu->domains[num] == dmar_domain)
4828 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4829 npages, !freelist, 0);
4830 }
4831
4832 }
4833
4834 dma_free_pagelist(freelist);
fe40f1e0 4835
163cc52c
DW		 4836 if (dmar_domain->max_addr == iova + size)
4837 dmar_domain->max_addr = iova;
b146a1c9 4838
5cf0a76f 4839 return size;
38717946 4840}
38717946 4841
d14d6577 4842static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4843 dma_addr_t iova)
38717946 4844{
00a77deb 4845 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4846 struct dma_pte *pte;
5cf0a76f 4847 int level = 0;
faa3d6f5 4848 u64 phys = 0;
38717946 4849
5cf0a76f 4850 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4851 if (pte)
faa3d6f5 4852 phys = dma_pte_addr(pte);
38717946 4853
faa3d6f5 4854 return phys;
38717946 4855}
a8bcbb0d 4856
5d587b8d 4857static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4858{
dbb9fd86 4859 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4860 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4861 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4862 return irq_remapping_enabled == 1;
dbb9fd86 4863
5d587b8d 4864 return false;
dbb9fd86
SY		 4865}
4866
abdfdde2
AW		 4867static int intel_iommu_add_device(struct device *dev)
4868{
a5459cfe 4869 struct intel_iommu *iommu;
abdfdde2 4870 struct iommu_group *group;
156baca8 4871 u8 bus, devfn;
70ae6f0d 4872
a5459cfe
AW		 4873 iommu = device_to_iommu(dev, &bus, &devfn);
4874 if (!iommu)
70ae6f0d
AW		 4875 return -ENODEV;
4876
a5459cfe 4877 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4878
e17f9ff4 4879 group = iommu_group_get_for_dev(dev);
783f157b 4880
e17f9ff4
AW		 4881 if (IS_ERR(group))
4882 return PTR_ERR(group);
bcb71abe 4883
abdfdde2 4884 iommu_group_put(group);
e17f9ff4 4885 return 0;
abdfdde2 4886}
70ae6f0d 4887
abdfdde2
AW		 4888static void intel_iommu_remove_device(struct device *dev)
4889{
a5459cfe
AW		 4890 struct intel_iommu *iommu;
4891 u8 bus, devfn;
4892
4893 iommu = device_to_iommu(dev, &bus, &devfn);
4894 if (!iommu)
4895 return;
4896
abdfdde2 4897 iommu_group_remove_device(dev);
a5459cfe
AW		 4898
4899 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4900}
4901
b22f6434 4902static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4903 .capable = intel_iommu_capable,
00a77deb
JR		 4904 .domain_alloc = intel_iommu_domain_alloc,
4905 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4906 .attach_dev = intel_iommu_attach_device,
4907 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4908 .map = intel_iommu_map,
4909 .unmap = intel_iommu_unmap,
315786eb 4910 .map_sg = default_iommu_map_sg,
a8bcbb0d 4911 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4912 .add_device = intel_iommu_add_device,
4913 .remove_device = intel_iommu_remove_device,
6d1c56a9 4914 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4915};
9af88143 4916
9452618e
DV		 4917static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4918{
4919 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4920 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4921 dmar_map_gfx = 0;
4922}
4923
4924DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4925DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4926DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4927DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4928DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4929DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4930DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4931
d34d6517 4932static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4933{
4934 /*
4935 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4936 * but needs it. Same seems to hold for the desktop versions.
9af88143 4937 */
9f10e5bf 4938 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4939 rwbf_quirk = 1;
4940}
4941
4942DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4943DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4944DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4945DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4946DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4947DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4948DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4949
eecfd57f
AJ		 4950#define GGC 0x52
4951#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4952#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4953#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4954#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4955#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4956#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4957#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4958#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4959
d34d6517 4960static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4961{
4962 unsigned short ggc;
4963
eecfd57f 4964 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4965 return;
4966
eecfd57f 4967 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4968 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4969 dmar_map_gfx = 0;
6fbcfb3e
DW		 4970 } else if (dmar_map_gfx) {
4971 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4972 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4973 intel_iommu_strict = 1;
4974 }
9eecabcb
DW		 4975}
4976DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4977DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4978DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4979DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4980
e0fc7e0b
DW		 4981/* On Tylersburg chipsets, some BIOSes have been known to enable the
4982 ISOCH DMAR unit for the Azalia sound device, but not give it any
4983 TLB entries, which causes it to deadlock. Check for that. We do
4984 this in a function called from init_dmars(), instead of in a PCI
4985 quirk, because we don't want to print the obnoxious "BIOS broken"
4986 message if VT-d is actually disabled.
4987*/
4988static void __init check_tylersburg_isoch(void)
4989{
4990 struct pci_dev *pdev;
4991 uint32_t vtisochctrl;
4992
4993 /* If there's no Azalia in the system anyway, forget it. */
4994 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4995 if (!pdev)
4996 return;
4997 pci_dev_put(pdev);
4998
4999 /* System Management Registers. Might be hidden, in which case
5000 we can't do the sanity check. But that's OK, because the
5001 known-broken BIOSes _don't_ actually hide it, so far. */
5002 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5003 if (!pdev)
5004 return;
5005
5006 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5007 pci_dev_put(pdev);
5008 return;
5009 }
5010
5011 pci_dev_put(pdev);
5012
5013 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5014 if (vtisochctrl & 1)
5015 return;
5016
5017 /* Drop all bits other than the number of TLB entries */
5018 vtisochctrl &= 0x1c;
5019
5020 /* If we have the recommended number of TLB entries (16), fine. */
5021 if (vtisochctrl == 0x10)
5022 return;
5023
5024 /* Zero TLB entries? You get to ride the short bus to school. */
5025 if (!vtisochctrl) {
5026 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5027 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5028 dmi_get_system_info(DMI_BIOS_VENDOR),
5029 dmi_get_system_info(DMI_BIOS_VERSION),
5030 dmi_get_system_info(DMI_PRODUCT_VERSION));
5031 iommu_identity_mapping |= IDENTMAP_AZALIA;
5032 return;
5033 }
9f10e5bf
JR		 5034
5035 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5036 vtisochctrl);
5037}
Linux 6.x block layer and io_uring tree(s)