projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
paride: convert pcd to blk-mq
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
aa473240 36#include <linux/cpu.h>
5e0d2a6f 37#include <linux/timer.h>
dfddb969 38#include <linux/io.h>
38717946 39#include <linux/iova.h>
5d450806 40#include <linux/iommu.h>
38717946 41#include <linux/intel-iommu.h>
134fac3f 42#include <linux/syscore_ops.h>
69575d38 43#include <linux/tboot.h>
adb2fe02 44#include <linux/dmi.h>
5cdede24 45#include <linux/pci-ats.h>
0ee332c1 46#include <linux/memblock.h>
36746436 47#include <linux/dma-contiguous.h>
fec777c3 48#include <linux/dma-direct.h>
091d42e4 49#include <linux/crash_dump.h>
8a8f422d 50#include <asm/irq_remapping.h>
ba395927 51#include <asm/cacheflush.h>
46a7fa27 52#include <asm/iommu.h>
ba395927 53
078e1ee2 54#include "irq_remapping.h"
56283174 55#include "intel-pasid.h"
078e1ee2 56
5b6985ce
FY		 57#define ROOT_SIZE VTD_PAGE_SIZE
58#define CONTEXT_SIZE VTD_PAGE_SIZE
59
ba395927 60#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 61#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 62#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 63#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 64
65#define IOAPIC_RANGE_START (0xfee00000)
66#define IOAPIC_RANGE_END (0xfeefffff)
67#define IOVA_START_ADDR (0x1000)
68
5e3b4a15 69#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
ba395927 70
4ed0d3e6 71#define MAX_AGAW_WIDTH 64
5c645b35 72#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 73
2ebe3151
DW		 74#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
75#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
76
77/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
78 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
79#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
80 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
81#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 82
1b722500
RM		 83/* IO virtual address start page frame number */
84#define IOVA_START_PFN (1)
85
f27be03b 86#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
5e0d2a6f 87
df08cdc7
AM		 88/* page table handling */
89#define LEVEL_STRIDE (9)
90#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
6d1c56a9
OBC		 92/*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
df08cdc7
AM		 110static inline int agaw_to_level(int agaw)
111{
112 return agaw + 2;
113}
114
115static inline int agaw_to_width(int agaw)
116{
5c645b35 117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 118}
119
120static inline int width_to_agaw(int width)
121{
5c645b35 122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 123}
124
125static inline unsigned int level_to_offset_bits(int level)
126{
127 return (level - 1) * LEVEL_STRIDE;
128}
129
130static inline int pfn_level_offset(unsigned long pfn, int level)
131{
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133}
134
135static inline unsigned long level_mask(int level)
136{
137 return -1UL << level_to_offset_bits(level);
138}
139
140static inline unsigned long level_size(int level)
141{
142 return 1UL << level_to_offset_bits(level);
143}
144
145static inline unsigned long align_to_level(unsigned long pfn, int level)
146{
147 return (pfn + level_size(level) - 1) & level_mask(level);
148}
fd18de50 149
6dd9a7c7
YS		 150static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151{
5c645b35 152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 153}
154
dd4e8319
DW		 155/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158{
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160}
161
162static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163{
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165}
166static inline unsigned long page_to_dma_pfn(struct page *pg)
167{
168 return mm_to_dma_pfn(page_to_pfn(pg));
169}
170static inline unsigned long virt_to_dma_pfn(void *p)
171{
172 return page_to_dma_pfn(virt_to_page(p));
173}
174
d9630fe9
WH		 175/* global iommu list, set NULL for ignored DMAR units */
176static struct intel_iommu **g_iommus;
177
e0fc7e0b 178static void __init check_tylersburg_isoch(void);
9af88143
DW		 179static int rwbf_quirk;
180
b779260b
JC		 181/*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185static int force_on = 0;
bfd20f1c 186int intel_iommu_tboot_noforce;
b779260b 187
46b08e1a
MM		 188/*
189 * 0: Present
190 * 1-11: Reserved
191 * 12-63: Context Ptr (12 - (haw-1))
192 * 64-127: Reserved
193 */
194struct root_entry {
03ecc32c
DW		 195 u64 lo;
196 u64 hi;
46b08e1a
MM		 197};
198#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 199
091d42e4
JR		 200/*
201 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
202 * if marked present.
203 */
204static phys_addr_t root_entry_lctp(struct root_entry *re)
205{
206 if (!(re->lo & 1))
207 return 0;
208
209 return re->lo & VTD_PAGE_MASK;
210}
211
212/*
213 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
214 * if marked present.
215 */
216static phys_addr_t root_entry_uctp(struct root_entry *re)
217{
218 if (!(re->hi & 1))
219 return 0;
46b08e1a 220
091d42e4
JR		 221 return re->hi & VTD_PAGE_MASK;
222}
7a8fc25e
MM		 223/*
224 * low 64 bits:
225 * 0: present
226 * 1: fault processing disable
227 * 2-3: translation type
228 * 12-63: address space root
229 * high 64 bits:
230 * 0-2: address width
231 * 3-6: aval
232 * 8-23: domain id
233 */
234struct context_entry {
235 u64 lo;
236 u64 hi;
237};
c07e7d21 238
cf484d0e
JR		 239static inline void context_clear_pasid_enable(struct context_entry *context)
240{
241 context->lo &= ~(1ULL << 11);
242}
243
244static inline bool context_pasid_enabled(struct context_entry *context)
245{
246 return !!(context->lo & (1ULL << 11));
247}
248
249static inline void context_set_copied(struct context_entry *context)
250{
251 context->hi |= (1ull << 3);
252}
253
254static inline bool context_copied(struct context_entry *context)
255{
256 return !!(context->hi & (1ULL << 3));
257}
258
259static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 260{
261 return (context->lo & 1);
262}
cf484d0e
JR		 263
264static inline bool context_present(struct context_entry *context)
265{
266 return context_pasid_enabled(context) ?
267 __context_present(context) :
268 __context_present(context) && !context_copied(context);
269}
270
c07e7d21
MM		 271static inline void context_set_present(struct context_entry *context)
272{
273 context->lo |= 1;
274}
275
276static inline void context_set_fault_enable(struct context_entry *context)
277{
278 context->lo &= (((u64)-1) << 2) | 1;
279}
280
c07e7d21
MM		 281static inline void context_set_translation_type(struct context_entry *context,
282 unsigned long value)
283{
284 context->lo &= (((u64)-1) << 4) | 3;
285 context->lo |= (value & 3) << 2;
286}
287
288static inline void context_set_address_root(struct context_entry *context,
289 unsigned long value)
290{
1a2262f9 291 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 292 context->lo |= value & VTD_PAGE_MASK;
293}
294
295static inline void context_set_address_width(struct context_entry *context,
296 unsigned long value)
297{
298 context->hi |= value & 7;
299}
300
301static inline void context_set_domain_id(struct context_entry *context,
302 unsigned long value)
303{
304 context->hi |= (value & ((1 << 16) - 1)) << 8;
305}
306
dbcd861f
JR		 307static inline int context_domain_id(struct context_entry *c)
308{
309 return((c->hi >> 8) & 0xffff);
310}
311
c07e7d21
MM		 312static inline void context_clear_entry(struct context_entry *context)
313{
314 context->lo = 0;
315 context->hi = 0;
316}
7a8fc25e 317
622ba12a
MM		 318/*
319 * 0: readable
320 * 1: writable
321 * 2-6: reserved
322 * 7: super page
9cf06697
SY		 323 * 8-10: available
324 * 11: snoop behavior
622ba12a
MM		 325 * 12-63: Host physcial address
326 */
327struct dma_pte {
328 u64 val;
329};
622ba12a 330
19c239ce
MM		 331static inline void dma_clear_pte(struct dma_pte *pte)
332{
333 pte->val = 0;
334}
335
19c239ce
MM		 336static inline u64 dma_pte_addr(struct dma_pte *pte)
337{
c85994e4
DW		 338#ifdef CONFIG_64BIT
339 return pte->val & VTD_PAGE_MASK;
340#else
341 /* Must have a full atomic 64-bit read */
1a8bd481 342 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 343#endif
19c239ce
MM		 344}
345
19c239ce
MM		 346static inline bool dma_pte_present(struct dma_pte *pte)
347{
348 return (pte->val & 3) != 0;
349}
622ba12a 350
4399c8bf
AK		 351static inline bool dma_pte_superpage(struct dma_pte *pte)
352{
c3c75eb7 353 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 354}
355
75e6bf96
DW		 356static inline int first_pte_in_page(struct dma_pte *pte)
357{
358 return !((unsigned long)pte & ~VTD_PAGE_MASK);
359}
360
2c2e2c38
FY		 361/*
362 * This domain is a statically identity mapping domain.
363 * 1. This domain creats a static 1:1 mapping to all usable memory.
364 * 2. It maps to each iommu if successful.
365 * 3. Each iommu mapps to this domain if successful.
366 */
19943b0e
DW		 367static struct dmar_domain *si_domain;
368static int hw_pass_through = 1;
2c2e2c38 369
28ccce0d
JR		 370/*
371 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 372 * across iommus may be owned in one domain, e.g. kvm guest.
373 */
ab8dfe25 374#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 375
2c2e2c38 376/* si_domain contains mulitple devices */
ab8dfe25 377#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 378
29a27719
JR		 379#define for_each_domain_iommu(idx, domain) \
380 for (idx = 0; idx < g_num_of_iommus; idx++) \
381 if (domain->iommu_refcnt[idx])
382
b94e4117
JL		 383struct dmar_rmrr_unit {
384 struct list_head list; /* list of rmrr units */
385 struct acpi_dmar_header *hdr; /* ACPI header */
386 u64 base_address; /* reserved base address*/
387 u64 end_address; /* reserved end address */
832bd858 388 struct dmar_dev_scope *devices; /* target devices */
b94e4117 389 int devices_cnt; /* target device count */
0659b8dc 390 struct iommu_resv_region *resv; /* reserved region handle */
b94e4117
JL		 391};
392
393struct dmar_atsr_unit {
394 struct list_head list; /* list of ATSR units */
395 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 396 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 397 int devices_cnt; /* target device count */
398 u8 include_all:1; /* include all ports */
399};
400
401static LIST_HEAD(dmar_atsr_units);
402static LIST_HEAD(dmar_rmrr_units);
403
404#define for_each_rmrr_units(rmrr) \
405 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
406
5e0d2a6f 407/* bitmap for indexing intel_iommus */
5e0d2a6f 408static int g_num_of_iommus;
409
92d03cc8 410static void domain_exit(struct dmar_domain *domain);
ba395927 411static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 412static void dmar_remove_one_dev_info(struct dmar_domain *domain,
413 struct device *dev);
127c7615 414static void __dmar_remove_one_dev_info(struct device_domain_info *info);
2452d9db
JR		 415static void domain_context_clear(struct intel_iommu *iommu,
416 struct device *dev);
2a46ddf7
JL		 417static int domain_detach_iommu(struct dmar_domain *domain,
418 struct intel_iommu *iommu);
ba395927 419
d3f13810 420#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 421int dmar_disabled = 0;
422#else
423int dmar_disabled = 1;
d3f13810 424#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 425
8bc1f85c
ED		 426int intel_iommu_enabled = 0;
427EXPORT_SYMBOL_GPL(intel_iommu_enabled);
428
2d9e667e 429static int dmar_map_gfx = 1;
7d3b03ce 430static int dmar_forcedac;
5e0d2a6f 431static int intel_iommu_strict;
6dd9a7c7 432static int intel_iommu_superpage = 1;
c83b2f20 433static int intel_iommu_ecs = 1;
2db1581e 434static int intel_iommu_pasid28;
ae853ddb 435static int iommu_identity_mapping;
c83b2f20 436
ae853ddb
DW		 437#define IDENTMAP_ALL 1
438#define IDENTMAP_GFX 2
439#define IDENTMAP_AZALIA 4
c83b2f20 440
2db1581e
LB		 441/* Broadwell and Skylake have broken ECS support — normal so-called "second
442 * level" translation of DMA requests-without-PASID doesn't actually happen
443 * unless you also set the NESTE bit in an extended context-entry. Which of
444 * course means that SVM doesn't work because it's trying to do nested
445 * translation of the physical addresses it finds in the process page tables,
446 * through the IOVA->phys mapping found in the "second level" page tables.
447 *
448 * The VT-d specification was retroactively changed to change the definition
449 * of the capability bits and pretend that Broadwell/Skylake never happened...
450 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
451 * for some reason it was the PASID capability bit which was redefined (from
452 * bit 28 on BDW/SKL to bit 40 in future).
453 *
454 * So our test for ECS needs to eschew those implementations which set the old
455 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
456 * Unless we are working around the 'pasid28' limitations, that is, by putting
457 * the device into passthrough mode for normal DMA and thus masking the bug.
458 */
459#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
460 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
461/* PASID support is thus enabled if ECS is enabled and *either* of the old
462 * or new capability bits are set. */
463#define pasid_enabled(iommu) (ecs_enabled(iommu) && \
464 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
ba395927 465
c0771df8
DW		 466int intel_iommu_gfx_mapped;
467EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
468
ba395927
KA		 469#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
470static DEFINE_SPINLOCK(device_domain_lock);
471static LIST_HEAD(device_domain_list);
472
85319dcc
LB		 473/*
474 * Iterate over elements in device_domain_list and call the specified
475 * callback @fn against each element. This helper should only be used
476 * in the context where the device_domain_lock has already been holden.
477 */
478int for_each_device_domain(int (*fn)(struct device_domain_info *info,
479 void *data), void *data)
480{
481 int ret = 0;
482 struct device_domain_info *info;
483
484 assert_spin_locked(&device_domain_lock);
485 list_for_each_entry(info, &device_domain_list, global) {
486 ret = fn(info, data);
487 if (ret)
488 return ret;
489 }
490
491 return 0;
492}
493
b0119e87 494const struct iommu_ops intel_iommu_ops;
a8bcbb0d 495
4158c2ec
JR		 496static bool translation_pre_enabled(struct intel_iommu *iommu)
497{
498 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
499}
500
091d42e4
JR		 501static void clear_translation_pre_enabled(struct intel_iommu *iommu)
502{
503 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
504}
505
4158c2ec
JR		 506static void init_translation_status(struct intel_iommu *iommu)
507{
508 u32 gsts;
509
510 gsts = readl(iommu->reg + DMAR_GSTS_REG);
511 if (gsts & DMA_GSTS_TES)
512 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
513}
514
00a77deb
JR		 515/* Convert generic 'struct iommu_domain to private struct dmar_domain */
516static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
517{
518 return container_of(dom, struct dmar_domain, domain);
519}
520
ba395927
KA		 521static int __init intel_iommu_setup(char *str)
522{
523 if (!str)
524 return -EINVAL;
525 while (*str) {
0cd5c3c8
KM		 526 if (!strncmp(str, "on", 2)) {
527 dmar_disabled = 0;
9f10e5bf 528 pr_info("IOMMU enabled\n");
0cd5c3c8 529 } else if (!strncmp(str, "off", 3)) {
ba395927 530 dmar_disabled = 1;
9f10e5bf 531 pr_info("IOMMU disabled\n");
ba395927
KA		 532 } else if (!strncmp(str, "igfx_off", 8)) {
533 dmar_map_gfx = 0;
9f10e5bf 534 pr_info("Disable GFX device mapping\n");
7d3b03ce 535 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 536 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 537 dmar_forcedac = 1;
5e0d2a6f 538 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 539 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 540 intel_iommu_strict = 1;
6dd9a7c7 541 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 542 pr_info("Disable supported super page\n");
6dd9a7c7 543 intel_iommu_superpage = 0;
c83b2f20
DW		 544 } else if (!strncmp(str, "ecs_off", 7)) {
545 printk(KERN_INFO
546 "Intel-IOMMU: disable extended context table support\n");
547 intel_iommu_ecs = 0;
2db1581e
LB		 548 } else if (!strncmp(str, "pasid28", 7)) {
549 printk(KERN_INFO
550 "Intel-IOMMU: enable pre-production PASID support\n");
551 intel_iommu_pasid28 = 1;
552 iommu_identity_mapping |= IDENTMAP_GFX;
bfd20f1c
SL		 553 } else if (!strncmp(str, "tboot_noforce", 13)) {
554 printk(KERN_INFO
555 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
556 intel_iommu_tboot_noforce = 1;
ba395927
KA		 557 }
558
559 str += strcspn(str, ",");
560 while (*str == ',')
561 str++;
562 }
563 return 0;
564}
565__setup("intel_iommu=", intel_iommu_setup);
566
567static struct kmem_cache *iommu_domain_cache;
568static struct kmem_cache *iommu_devinfo_cache;
ba395927 569
9452d5bf
JR		 570static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
571{
8bf47816
JR		 572 struct dmar_domain **domains;
573 int idx = did >> 8;
574
575 domains = iommu->domains[idx];
576 if (!domains)
577 return NULL;
578
579 return domains[did & 0xff];
9452d5bf
JR		 580}
581
582static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
583 struct dmar_domain *domain)
584{
8bf47816
JR		 585 struct dmar_domain **domains;
586 int idx = did >> 8;
587
588 if (!iommu->domains[idx]) {
589 size_t size = 256 * sizeof(struct dmar_domain *);
590 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
591 }
592
593 domains = iommu->domains[idx];
594 if (WARN_ON(!domains))
595 return;
596 else
597 domains[did & 0xff] = domain;
9452d5bf
JR		 598}
599
9ddbfb42 600void *alloc_pgtable_page(int node)
eb3fa7cb 601{
4c923d47
SS		 602 struct page *page;
603 void *vaddr = NULL;
eb3fa7cb 604
4c923d47
SS		 605 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
606 if (page)
607 vaddr = page_address(page);
eb3fa7cb 608 return vaddr;
ba395927
KA		 609}
610
9ddbfb42 611void free_pgtable_page(void *vaddr)
ba395927
KA		 612{
613 free_page((unsigned long)vaddr);
614}
615
616static inline void *alloc_domain_mem(void)
617{
354bb65e 618 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 619}
620
38717946 621static void free_domain_mem(void *vaddr)
ba395927
KA		 622{
623 kmem_cache_free(iommu_domain_cache, vaddr);
624}
625
626static inline void * alloc_devinfo_mem(void)
627{
354bb65e 628 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 629}
630
631static inline void free_devinfo_mem(void *vaddr)
632{
633 kmem_cache_free(iommu_devinfo_cache, vaddr);
634}
635
ab8dfe25
JL		 636static inline int domain_type_is_vm(struct dmar_domain *domain)
637{
638 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
639}
640
28ccce0d
JR		 641static inline int domain_type_is_si(struct dmar_domain *domain)
642{
643 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
644}
645
ab8dfe25
JL		 646static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
647{
648 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
649 DOMAIN_FLAG_STATIC_IDENTITY);
650}
1b573683 651
162d1b10
JL		 652static inline int domain_pfn_supported(struct dmar_domain *domain,
653 unsigned long pfn)
654{
655 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
656
657 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
658}
659
4ed0d3e6 660static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 661{
662 unsigned long sagaw;
663 int agaw = -1;
664
665 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 666 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 667 agaw >= 0; agaw--) {
668 if (test_bit(agaw, &sagaw))
669 break;
670 }
671
672 return agaw;
673}
674
4ed0d3e6
FY		 675/*
676 * Calculate max SAGAW for each iommu.
677 */
678int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
679{
680 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
681}
682
683/*
684 * calculate agaw for each iommu.
685 * "SAGAW" may be different across iommus, use a default agaw, and
686 * get a supported less agaw for iommus that don't support the default agaw.
687 */
688int iommu_calculate_agaw(struct intel_iommu *iommu)
689{
690 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
691}
692
2c2e2c38 693/* This functionin only returns single iommu in a domain */
9ddbfb42 694struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
8c11e798
WH		 695{
696 int iommu_id;
697
2c2e2c38 698 /* si_domain and vm domain should not get here. */
ab8dfe25 699 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 700 for_each_domain_iommu(iommu_id, domain)
701 break;
702
8c11e798
WH		 703 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
704 return NULL;
705
706 return g_iommus[iommu_id];
707}
708
8e604097
WH		 709static void domain_update_iommu_coherency(struct dmar_domain *domain)
710{
d0501960
DW		 711 struct dmar_drhd_unit *drhd;
712 struct intel_iommu *iommu;
2f119c78
QL		 713 bool found = false;
714 int i;
2e12bc29 715
d0501960 716 domain->iommu_coherency = 1;
8e604097 717
29a27719 718 for_each_domain_iommu(i, domain) {
2f119c78 719 found = true;
8e604097
WH		 720 if (!ecap_coherent(g_iommus[i]->ecap)) {
721 domain->iommu_coherency = 0;
722 break;
723 }
8e604097 724 }
d0501960
DW		 725 if (found)
726 return;
727
728 /* No hardware attached; use lowest common denominator */
729 rcu_read_lock();
730 for_each_active_iommu(iommu, drhd) {
731 if (!ecap_coherent(iommu->ecap)) {
732 domain->iommu_coherency = 0;
733 break;
734 }
735 }
736 rcu_read_unlock();
8e604097
WH		 737}
738
161f6934 739static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 740{
161f6934
JL		 741 struct dmar_drhd_unit *drhd;
742 struct intel_iommu *iommu;
743 int ret = 1;
58c610bd 744
161f6934
JL		 745 rcu_read_lock();
746 for_each_active_iommu(iommu, drhd) {
747 if (iommu != skip) {
748 if (!ecap_sc_support(iommu->ecap)) {
749 ret = 0;
750 break;
751 }
58c610bd 752 }
58c610bd 753 }
161f6934
JL		 754 rcu_read_unlock();
755
756 return ret;
58c610bd
SY		 757}
758
161f6934 759static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 760{
8140a95d 761 struct dmar_drhd_unit *drhd;
161f6934 762 struct intel_iommu *iommu;
8140a95d 763 int mask = 0xf;
6dd9a7c7
YS		 764
765 if (!intel_iommu_superpage) {
161f6934 766 return 0;
6dd9a7c7
YS		 767 }
768
8140a95d 769 /* set iommu_superpage to the smallest common denominator */
0e242612 770 rcu_read_lock();
8140a95d 771 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 772 if (iommu != skip) {
773 mask &= cap_super_page_val(iommu->cap);
774 if (!mask)
775 break;
6dd9a7c7
YS		 776 }
777 }
0e242612
JL		 778 rcu_read_unlock();
779
161f6934 780 return fls(mask);
6dd9a7c7
YS		 781}
782
58c610bd
SY		 783/* Some capabilities may be different across iommus */
784static void domain_update_iommu_cap(struct dmar_domain *domain)
785{
786 domain_update_iommu_coherency(domain);
161f6934
JL		 787 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
788 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 789}
790
03ecc32c
DW		 791static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
792 u8 bus, u8 devfn, int alloc)
793{
794 struct root_entry *root = &iommu->root_entry[bus];
795 struct context_entry *context;
796 u64 *entry;
797
4df4eab1 798 entry = &root->lo;
c83b2f20 799 if (ecs_enabled(iommu)) {
03ecc32c
DW		 800 if (devfn >= 0x80) {
801 devfn -= 0x80;
802 entry = &root->hi;
803 }
804 devfn *= 2;
805 }
03ecc32c
DW		 806 if (*entry & 1)
807 context = phys_to_virt(*entry & VTD_PAGE_MASK);
808 else {
809 unsigned long phy_addr;
810 if (!alloc)
811 return NULL;
812
813 context = alloc_pgtable_page(iommu->node);
814 if (!context)
815 return NULL;
816
817 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
818 phy_addr = virt_to_phys((void *)context);
819 *entry = phy_addr | 1;
820 __iommu_flush_cache(iommu, entry, sizeof(*entry));
821 }
822 return &context[devfn];
823}
824
4ed6a540
DW		 825static int iommu_dummy(struct device *dev)
826{
827 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
828}
829
156baca8 830static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 831{
832 struct dmar_drhd_unit *drhd = NULL;
b683b230 833 struct intel_iommu *iommu;
156baca8
DW		 834 struct device *tmp;
835 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 836 u16 segment = 0;
c7151a8d
WH		 837 int i;
838
4ed6a540
DW		 839 if (iommu_dummy(dev))
840 return NULL;
841
156baca8 842 if (dev_is_pci(dev)) {
1c387188
AR		 843 struct pci_dev *pf_pdev;
844
156baca8 845 pdev = to_pci_dev(dev);
5823e330
JD		 846
847#ifdef CONFIG_X86
848 /* VMD child devices currently cannot be handled individually */
849 if (is_vmd(pdev->bus))
850 return NULL;
851#endif
852
1c387188
AR		 853 /* VFs aren't listed in scope tables; we need to look up
854 * the PF instead to find the IOMMU. */
855 pf_pdev = pci_physfn(pdev);
856 dev = &pf_pdev->dev;
156baca8 857 segment = pci_domain_nr(pdev->bus);
ca5b74d2 858 } else if (has_acpi_companion(dev))
156baca8
DW		 859 dev = &ACPI_COMPANION(dev)->dev;
860
0e242612 861 rcu_read_lock();
b683b230 862 for_each_active_iommu(iommu, drhd) {
156baca8 863 if (pdev && segment != drhd->segment)
276dbf99 864 continue;
c7151a8d 865
b683b230 866 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 867 drhd->devices_cnt, i, tmp) {
868 if (tmp == dev) {
1c387188
AR		 869 /* For a VF use its original BDF# not that of the PF
870 * which we used for the IOMMU lookup. Strictly speaking
871 * we could do this for all PCI devices; we only need to
872 * get the BDF# from the scope table for ACPI matches. */
5003ae1e 873 if (pdev && pdev->is_virtfn)
1c387188
AR		 874 goto got_pdev;
875
156baca8
DW		 876 *bus = drhd->devices[i].bus;
877 *devfn = drhd->devices[i].devfn;
b683b230 878 goto out;
156baca8
DW		 879 }
880
881 if (!pdev || !dev_is_pci(tmp))
882 continue;
883
884 ptmp = to_pci_dev(tmp);
885 if (ptmp->subordinate &&
886 ptmp->subordinate->number <= pdev->bus->number &&
887 ptmp->subordinate->busn_res.end >= pdev->bus->number)
888 goto got_pdev;
924b6231 889 }
c7151a8d 890
156baca8
DW		 891 if (pdev && drhd->include_all) {
892 got_pdev:
893 *bus = pdev->bus->number;
894 *devfn = pdev->devfn;
b683b230 895 goto out;
156baca8 896 }
c7151a8d 897 }
b683b230 898 iommu = NULL;
156baca8 899 out:
0e242612 900 rcu_read_unlock();
c7151a8d 901
b683b230 902 return iommu;
c7151a8d
WH		 903}
904
5331fe6f
WH		 905static void domain_flush_cache(struct dmar_domain *domain,
906 void *addr, int size)
907{
908 if (!domain->iommu_coherency)
909 clflush_cache_range(addr, size);
910}
911
ba395927
KA		 912static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
913{
ba395927 914 struct context_entry *context;
03ecc32c 915 int ret = 0;
ba395927
KA		 916 unsigned long flags;
917
918 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 919 context = iommu_context_addr(iommu, bus, devfn, 0);
920 if (context)
921 ret = context_present(context);
ba395927
KA		 922 spin_unlock_irqrestore(&iommu->lock, flags);
923 return ret;
924}
925
ba395927
KA		 926static void free_context_table(struct intel_iommu *iommu)
927{
ba395927
KA		 928 int i;
929 unsigned long flags;
930 struct context_entry *context;
931
932 spin_lock_irqsave(&iommu->lock, flags);
933 if (!iommu->root_entry) {
934 goto out;
935 }
936 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 937 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 938 if (context)
939 free_pgtable_page(context);
03ecc32c 940
c83b2f20 941 if (!ecs_enabled(iommu))
03ecc32c
DW		 942 continue;
943
944 context = iommu_context_addr(iommu, i, 0x80, 0);
945 if (context)
946 free_pgtable_page(context);
947
ba395927
KA		 948 }
949 free_pgtable_page(iommu->root_entry);
950 iommu->root_entry = NULL;
951out:
952 spin_unlock_irqrestore(&iommu->lock, flags);
953}
954
b026fd28 955static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 956 unsigned long pfn, int *target_level)
ba395927 957{
ba395927
KA		 958 struct dma_pte *parent, *pte = NULL;
959 int level = agaw_to_level(domain->agaw);
4399c8bf 960 int offset;
ba395927
KA		 961
962 BUG_ON(!domain->pgd);
f9423606 963
162d1b10 964 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 965 /* Address beyond IOMMU's addressing capabilities. */
966 return NULL;
967
ba395927
KA		 968 parent = domain->pgd;
969
5cf0a76f 970 while (1) {
ba395927
KA		 971 void *tmp_page;
972
b026fd28 973 offset = pfn_level_offset(pfn, level);
ba395927 974 pte = &parent[offset];
5cf0a76f 975 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 976 break;
5cf0a76f 977 if (level == *target_level)
ba395927
KA		 978 break;
979
19c239ce 980 if (!dma_pte_present(pte)) {
c85994e4
DW		 981 uint64_t pteval;
982
4c923d47 983 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 984
206a73c1 985 if (!tmp_page)
ba395927 986 return NULL;
206a73c1 987
c85994e4 988 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 989 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 990 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 991 /* Someone else set it while we were thinking; use theirs. */
992 free_pgtable_page(tmp_page);
effad4b5 993 else
c85994e4 994 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 995 }
5cf0a76f
DW		 996 if (level == 1)
997 break;
998
19c239ce 999 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1000 level--;
1001 }
1002
5cf0a76f
DW		 1003 if (!*target_level)
1004 *target_level = level;
1005
ba395927
KA		 1006 return pte;
1007}
1008
6dd9a7c7 1009
ba395927 1010/* return address's pte at specific level */
90dcfb5e
DW		 1011static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1012 unsigned long pfn,
6dd9a7c7 1013 int level, int *large_page)
ba395927
KA		 1014{
1015 struct dma_pte *parent, *pte = NULL;
1016 int total = agaw_to_level(domain->agaw);
1017 int offset;
1018
1019 parent = domain->pgd;
1020 while (level <= total) {
90dcfb5e 1021 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1022 pte = &parent[offset];
1023 if (level == total)
1024 return pte;
1025
6dd9a7c7
YS		 1026 if (!dma_pte_present(pte)) {
1027 *large_page = total;
ba395927 1028 break;
6dd9a7c7
YS		 1029 }
1030
e16922af 1031 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1032 *large_page = total;
1033 return pte;
1034 }
1035
19c239ce 1036 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1037 total--;
1038 }
1039 return NULL;
1040}
1041
ba395927 1042/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1043static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1044 unsigned long start_pfn,
1045 unsigned long last_pfn)
ba395927 1046{
6dd9a7c7 1047 unsigned int large_page = 1;
310a5ab9 1048 struct dma_pte *first_pte, *pte;
66eae846 1049
162d1b10
JL		 1050 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1051 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1052 BUG_ON(start_pfn > last_pfn);
ba395927 1053
04b18e65 1054 /* we don't need lock here; nobody else touches the iova range */
59c36286 1055 do {
6dd9a7c7
YS		 1056 large_page = 1;
1057 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1058 if (!pte) {
6dd9a7c7 1059 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1060 continue;
1061 }
6dd9a7c7 1062 do {
310a5ab9 1063 dma_clear_pte(pte);
6dd9a7c7 1064 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1065 pte++;
75e6bf96
DW		 1066 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1067
310a5ab9
DW		 1068 domain_flush_cache(domain, first_pte,
1069 (void *)pte - (void *)first_pte);
59c36286
DW		 1070
1071 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1072}
1073
3269ee0b 1074static void dma_pte_free_level(struct dmar_domain *domain, int level,
bc24c571
DD		 1075 int retain_level, struct dma_pte *pte,
1076 unsigned long pfn, unsigned long start_pfn,
1077 unsigned long last_pfn)
3269ee0b
AW		 1078{
1079 pfn = max(start_pfn, pfn);
1080 pte = &pte[pfn_level_offset(pfn, level)];
1081
1082 do {
1083 unsigned long level_pfn;
1084 struct dma_pte *level_pte;
1085
1086 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1087 goto next;
1088
f7116e11 1089 level_pfn = pfn & level_mask(level);
3269ee0b
AW		 1090 level_pte = phys_to_virt(dma_pte_addr(pte));
1091
bc24c571
DD		 1092 if (level > 2) {
1093 dma_pte_free_level(domain, level - 1, retain_level,
1094 level_pte, level_pfn, start_pfn,
1095 last_pfn);
1096 }
3269ee0b 1097
bc24c571
DD		 1098 /*
1099 * Free the page table if we're below the level we want to
1100 * retain and the range covers the entire table.
1101 */
1102 if (level < retain_level && !(start_pfn > level_pfn ||
08336fd2 1103 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1104 dma_clear_pte(pte);
1105 domain_flush_cache(domain, pte, sizeof(*pte));
1106 free_pgtable_page(level_pte);
1107 }
1108next:
1109 pfn += level_size(level);
1110 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1111}
1112
bc24c571
DD		 1113/*
1114 * clear last level (leaf) ptes and free page table pages below the
1115 * level we wish to keep intact.
1116 */
ba395927 1117static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b 1118 unsigned long start_pfn,
bc24c571
DD		 1119 unsigned long last_pfn,
1120 int retain_level)
ba395927 1121{
162d1b10
JL		 1122 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1123 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1124 BUG_ON(start_pfn > last_pfn);
ba395927 1125
d41a4adb
JL		 1126 dma_pte_clear_range(domain, start_pfn, last_pfn);
1127
f3a0a52f 1128 /* We don't need lock here; nobody else touches the iova range */
bc24c571 1129 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
3269ee0b 1130 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1131
ba395927 1132 /* free pgd */
d794dc9b 1133 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1134 free_pgtable_page(domain->pgd);
1135 domain->pgd = NULL;
1136 }
1137}
1138
ea8ea460
DW		 1139/* When a page at a given level is being unlinked from its parent, we don't
1140 need to *modify* it at all. All we need to do is make a list of all the
1141 pages which can be freed just as soon as we've flushed the IOTLB and we
1142 know the hardware page-walk will no longer touch them.
1143 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1144 be freed. */
1145static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1146 int level, struct dma_pte *pte,
1147 struct page *freelist)
1148{
1149 struct page *pg;
1150
1151 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1152 pg->freelist = freelist;
1153 freelist = pg;
1154
1155 if (level == 1)
1156 return freelist;
1157
adeb2590
JL		 1158 pte = page_address(pg);
1159 do {
ea8ea460
DW		 1160 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1161 freelist = dma_pte_list_pagetables(domain, level - 1,
1162 pte, freelist);
adeb2590
JL		 1163 pte++;
1164 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1165
1166 return freelist;
1167}
1168
1169static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1170 struct dma_pte *pte, unsigned long pfn,
1171 unsigned long start_pfn,
1172 unsigned long last_pfn,
1173 struct page *freelist)
1174{
1175 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1176
1177 pfn = max(start_pfn, pfn);
1178 pte = &pte[pfn_level_offset(pfn, level)];
1179
1180 do {
1181 unsigned long level_pfn;
1182
1183 if (!dma_pte_present(pte))
1184 goto next;
1185
1186 level_pfn = pfn & level_mask(level);
1187
1188 /* If range covers entire pagetable, free it */
1189 if (start_pfn <= level_pfn &&
1190 last_pfn >= level_pfn + level_size(level) - 1) {
1191 /* These suborbinate page tables are going away entirely. Don't
1192 bother to clear them; we're just going to *free* them. */
1193 if (level > 1 && !dma_pte_superpage(pte))
1194 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1195
1196 dma_clear_pte(pte);
1197 if (!first_pte)
1198 first_pte = pte;
1199 last_pte = pte;
1200 } else if (level > 1) {
1201 /* Recurse down into a level that isn't *entirely* obsolete */
1202 freelist = dma_pte_clear_level(domain, level - 1,
1203 phys_to_virt(dma_pte_addr(pte)),
1204 level_pfn, start_pfn, last_pfn,
1205 freelist);
1206 }
1207next:
1208 pfn += level_size(level);
1209 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1210
1211 if (first_pte)
1212 domain_flush_cache(domain, first_pte,
1213 (void *)++last_pte - (void *)first_pte);
1214
1215 return freelist;
1216}
1217
1218/* We can't just free the pages because the IOMMU may still be walking
1219 the page tables, and may have cached the intermediate levels. The
1220 pages can only be freed after the IOTLB flush has been done. */
b690420a
JR		 1221static struct page *domain_unmap(struct dmar_domain *domain,
1222 unsigned long start_pfn,
1223 unsigned long last_pfn)
ea8ea460 1224{
ea8ea460
DW		 1225 struct page *freelist = NULL;
1226
162d1b10
JL		 1227 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1228 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1229 BUG_ON(start_pfn > last_pfn);
1230
1231 /* we don't need lock here; nobody else touches the iova range */
1232 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1233 domain->pgd, 0, start_pfn, last_pfn, NULL);
1234
1235 /* free pgd */
1236 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1237 struct page *pgd_page = virt_to_page(domain->pgd);
1238 pgd_page->freelist = freelist;
1239 freelist = pgd_page;
1240
1241 domain->pgd = NULL;
1242 }
1243
1244 return freelist;
1245}
1246
b690420a 1247static void dma_free_pagelist(struct page *freelist)
ea8ea460
DW		 1248{
1249 struct page *pg;
1250
1251 while ((pg = freelist)) {
1252 freelist = pg->freelist;
1253 free_pgtable_page(page_address(pg));
1254 }
1255}
1256
13cf0174
JR		 1257static void iova_entry_free(unsigned long data)
1258{
1259 struct page *freelist = (struct page *)data;
1260
1261 dma_free_pagelist(freelist);
1262}
1263
ba395927
KA		 1264/* iommu handling */
1265static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1266{
1267 struct root_entry *root;
1268 unsigned long flags;
1269
4c923d47 1270 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1271 if (!root) {
9f10e5bf 1272 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1273 iommu->name);
ba395927 1274 return -ENOMEM;
ffebeb46 1275 }
ba395927 1276
5b6985ce 1277 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1278
1279 spin_lock_irqsave(&iommu->lock, flags);
1280 iommu->root_entry = root;
1281 spin_unlock_irqrestore(&iommu->lock, flags);
1282
1283 return 0;
1284}
1285
ba395927
KA		 1286static void iommu_set_root_entry(struct intel_iommu *iommu)
1287{
03ecc32c 1288 u64 addr;
c416daa9 1289 u32 sts;
ba395927
KA		 1290 unsigned long flag;
1291
03ecc32c 1292 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1293 if (ecs_enabled(iommu))
03ecc32c 1294 addr |= DMA_RTADDR_RTT;
ba395927 1295
1f5b3c3f 1296 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1297 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1298
c416daa9 1299 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1300
1301 /* Make sure hardware complete it */
1302 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1303 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1304
1f5b3c3f 1305 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1306}
1307
1308static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1309{
1310 u32 val;
1311 unsigned long flag;
1312
9af88143 1313 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1314 return;
ba395927 1315
1f5b3c3f 1316 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1317 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1318
1319 /* Make sure hardware complete it */
1320 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1321 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1322
1f5b3c3f 1323 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1324}
1325
1326/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1327static void __iommu_flush_context(struct intel_iommu *iommu,
1328 u16 did, u16 source_id, u8 function_mask,
1329 u64 type)
ba395927
KA		 1330{
1331 u64 val = 0;
1332 unsigned long flag;
1333
ba395927
KA		 1334 switch (type) {
1335 case DMA_CCMD_GLOBAL_INVL:
1336 val = DMA_CCMD_GLOBAL_INVL;
1337 break;
1338 case DMA_CCMD_DOMAIN_INVL:
1339 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1340 break;
1341 case DMA_CCMD_DEVICE_INVL:
1342 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1343 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1344 break;
1345 default:
1346 BUG();
1347 }
1348 val |= DMA_CCMD_ICC;
1349
1f5b3c3f 1350 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1351 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1352
1353 /* Make sure hardware complete it */
1354 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1355 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1356
1f5b3c3f 1357 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1358}
1359
ba395927 1360/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1361static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1362 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1363{
1364 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1365 u64 val = 0, val_iva = 0;
1366 unsigned long flag;
1367
ba395927
KA		 1368 switch (type) {
1369 case DMA_TLB_GLOBAL_FLUSH:
1370 /* global flush doesn't need set IVA_REG */
1371 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1372 break;
1373 case DMA_TLB_DSI_FLUSH:
1374 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1375 break;
1376 case DMA_TLB_PSI_FLUSH:
1377 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1378 /* IH bit is passed in as part of address */
ba395927
KA		 1379 val_iva = size_order | addr;
1380 break;
1381 default:
1382 BUG();
1383 }
1384 /* Note: set drain read/write */
1385#if 0
1386 /*
1387 * This is probably to be super secure.. Looks like we can
1388 * ignore it without any impact.
1389 */
1390 if (cap_read_drain(iommu->cap))
1391 val |= DMA_TLB_READ_DRAIN;
1392#endif
1393 if (cap_write_drain(iommu->cap))
1394 val |= DMA_TLB_WRITE_DRAIN;
1395
1f5b3c3f 1396 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1397 /* Note: Only uses first TLB reg currently */
1398 if (val_iva)
1399 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1400 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1401
1402 /* Make sure hardware complete it */
1403 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1404 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1405
1f5b3c3f 1406 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1407
1408 /* check IOTLB invalidation granularity */
1409 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1410 pr_err("Flush IOTLB failed\n");
ba395927 1411 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1412 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1413 (unsigned long long)DMA_TLB_IIRG(type),
1414 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1415}
1416
64ae892b
DW		 1417static struct device_domain_info *
1418iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1419 u8 bus, u8 devfn)
93a23a72 1420{
93a23a72 1421 struct device_domain_info *info;
93a23a72 1422
55d94043
JR		 1423 assert_spin_locked(&device_domain_lock);
1424
93a23a72
YZ		 1425 if (!iommu->qi)
1426 return NULL;
1427
93a23a72 1428 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1429 if (info->iommu == iommu && info->bus == bus &&
1430 info->devfn == devfn) {
b16d0cb9
DW		 1431 if (info->ats_supported && info->dev)
1432 return info;
93a23a72
YZ		 1433 break;
1434 }
93a23a72 1435
b16d0cb9 1436 return NULL;
93a23a72
YZ		 1437}
1438
0824c592
OP		 1439static void domain_update_iotlb(struct dmar_domain *domain)
1440{
1441 struct device_domain_info *info;
1442 bool has_iotlb_device = false;
1443
1444 assert_spin_locked(&device_domain_lock);
1445
1446 list_for_each_entry(info, &domain->devices, link) {
1447 struct pci_dev *pdev;
1448
1449 if (!info->dev || !dev_is_pci(info->dev))
1450 continue;
1451
1452 pdev = to_pci_dev(info->dev);
1453 if (pdev->ats_enabled) {
1454 has_iotlb_device = true;
1455 break;
1456 }
1457 }
1458
1459 domain->has_iotlb_device = has_iotlb_device;
1460}
1461
93a23a72 1462static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1463{
fb0cc3aa
BH		 1464 struct pci_dev *pdev;
1465
0824c592
OP		 1466 assert_spin_locked(&device_domain_lock);
1467
0bcb3e28 1468 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1469 return;
1470
fb0cc3aa 1471 pdev = to_pci_dev(info->dev);
1c48db44
JP		 1472 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1473 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1474 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1475 * reserved, which should be set to 0.
1476 */
1477 if (!ecap_dit(info->iommu->ecap))
1478 info->pfsid = 0;
1479 else {
1480 struct pci_dev *pf_pdev;
1481
1482 /* pdev will be returned if device is not a vf */
1483 pf_pdev = pci_physfn(pdev);
1484 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1485 }
fb0cc3aa 1486
b16d0cb9
DW		 1487#ifdef CONFIG_INTEL_IOMMU_SVM
1488 /* The PCIe spec, in its wisdom, declares that the behaviour of
1489 the device if you enable PASID support after ATS support is
1490 undefined. So always enable PASID support on devices which
1491 have it, even if we can't yet know if we're ever going to
1492 use it. */
1493 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1494 info->pasid_enabled = 1;
1495
1496 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1497 info->pri_enabled = 1;
1498#endif
1499 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1500 info->ats_enabled = 1;
0824c592 1501 domain_update_iotlb(info->domain);
b16d0cb9
DW		 1502 info->ats_qdep = pci_ats_queue_depth(pdev);
1503 }
93a23a72
YZ		 1504}
1505
1506static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1507{
b16d0cb9
DW		 1508 struct pci_dev *pdev;
1509
0824c592
OP		 1510 assert_spin_locked(&device_domain_lock);
1511
da972fb1 1512 if (!dev_is_pci(info->dev))
93a23a72
YZ		 1513 return;
1514
b16d0cb9
DW		 1515 pdev = to_pci_dev(info->dev);
1516
1517 if (info->ats_enabled) {
1518 pci_disable_ats(pdev);
1519 info->ats_enabled = 0;
0824c592 1520 domain_update_iotlb(info->domain);
b16d0cb9
DW		 1521 }
1522#ifdef CONFIG_INTEL_IOMMU_SVM
1523 if (info->pri_enabled) {
1524 pci_disable_pri(pdev);
1525 info->pri_enabled = 0;
1526 }
1527 if (info->pasid_enabled) {
1528 pci_disable_pasid(pdev);
1529 info->pasid_enabled = 0;
1530 }
1531#endif
93a23a72
YZ		 1532}
1533
1534static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1535 u64 addr, unsigned mask)
1536{
1537 u16 sid, qdep;
1538 unsigned long flags;
1539 struct device_domain_info *info;
1540
0824c592
OP		 1541 if (!domain->has_iotlb_device)
1542 return;
1543
93a23a72
YZ		 1544 spin_lock_irqsave(&device_domain_lock, flags);
1545 list_for_each_entry(info, &domain->devices, link) {
b16d0cb9 1546 if (!info->ats_enabled)
93a23a72
YZ		 1547 continue;
1548
1549 sid = info->bus << 8 | info->devfn;
b16d0cb9 1550 qdep = info->ats_qdep;
1c48db44
JP		 1551 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1552 qdep, addr, mask);
93a23a72
YZ		 1553 }
1554 spin_unlock_irqrestore(&device_domain_lock, flags);
1555}
1556
a1ddcbe9
JR		 1557static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1558 struct dmar_domain *domain,
1559 unsigned long pfn, unsigned int pages,
1560 int ih, int map)
ba395927 1561{
9dd2fe89 1562 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1563 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1564 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1565
ba395927
KA		 1566 BUG_ON(pages == 0);
1567
ea8ea460
DW		 1568 if (ih)
1569 ih = 1 << 6;
ba395927 1570 /*
9dd2fe89
YZ		 1571 * Fallback to domain selective flush if no PSI support or the size is
1572 * too big.
ba395927
KA		 1573 * PSI requires page size to be 2 ^ x, and the base address is naturally
1574 * aligned to the size
1575 */
9dd2fe89
YZ		 1576 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1577 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1578 DMA_TLB_DSI_FLUSH);
9dd2fe89 1579 else
ea8ea460 1580 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1581 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1582
1583 /*
82653633
NA		 1584 * In caching mode, changes of pages from non-present to present require
1585 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1586 */
82653633 1587 if (!cap_caching_mode(iommu->cap) || !map)
9d2e6505 1588 iommu_flush_dev_iotlb(domain, addr, mask);
ba395927
KA		 1589}
1590
eed91a0b
PX		 1591/* Notification for newly created mappings */
1592static inline void __mapping_notify_one(struct intel_iommu *iommu,
1593 struct dmar_domain *domain,
1594 unsigned long pfn, unsigned int pages)
1595{
1596 /* It's a non-present to present mapping. Only flush if caching mode */
1597 if (cap_caching_mode(iommu->cap))
1598 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1599 else
1600 iommu_flush_write_buffer(iommu);
1601}
1602
13cf0174
JR		 1603static void iommu_flush_iova(struct iova_domain *iovad)
1604{
1605 struct dmar_domain *domain;
1606 int idx;
1607
1608 domain = container_of(iovad, struct dmar_domain, iovad);
1609
1610 for_each_domain_iommu(idx, domain) {
1611 struct intel_iommu *iommu = g_iommus[idx];
1612 u16 did = domain->iommu_did[iommu->seq_id];
1613
1614 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1615
1616 if (!cap_caching_mode(iommu->cap))
1617 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1618 0, MAX_AGAW_PFN_WIDTH);
1619 }
1620}
1621
f8bab735 1622static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1623{
1624 u32 pmen;
1625 unsigned long flags;
1626
1f5b3c3f 1627 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1628 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1629 pmen &= ~DMA_PMEN_EPM;
1630 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1631
1632 /* wait for the protected region status bit to clear */
1633 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1634 readl, !(pmen & DMA_PMEN_PRS), pmen);
1635
1f5b3c3f 1636 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1637}
1638
2a41ccee 1639static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1640{
1641 u32 sts;
1642 unsigned long flags;
1643
1f5b3c3f 1644 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1645 iommu->gcmd |= DMA_GCMD_TE;
1646 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1647
1648 /* Make sure hardware complete it */
1649 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1650 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1651
1f5b3c3f 1652 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1653}
1654
2a41ccee 1655static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1656{
1657 u32 sts;
1658 unsigned long flag;
1659
1f5b3c3f 1660 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1661 iommu->gcmd &= ~DMA_GCMD_TE;
1662 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1663
1664 /* Make sure hardware complete it */
1665 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1666 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1667
1f5b3c3f 1668 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1669}
1670
3460a6d9 1671
ba395927
KA		 1672static int iommu_init_domains(struct intel_iommu *iommu)
1673{
8bf47816
JR		 1674 u32 ndomains, nlongs;
1675 size_t size;
ba395927
KA		 1676
1677 ndomains = cap_ndoms(iommu->cap);
8bf47816 1678 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1679 iommu->name, ndomains);
ba395927
KA		 1680 nlongs = BITS_TO_LONGS(ndomains);
1681
94a91b50
DD		 1682 spin_lock_init(&iommu->lock);
1683
ba395927
KA		 1684 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1685 if (!iommu->domain_ids) {
9f10e5bf
JR		 1686 pr_err("%s: Allocating domain id array failed\n",
1687 iommu->name);
ba395927
KA		 1688 return -ENOMEM;
1689 }
8bf47816 1690
86f004c7 1691 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
8bf47816
JR		 1692 iommu->domains = kzalloc(size, GFP_KERNEL);
1693
1694 if (iommu->domains) {
1695 size = 256 * sizeof(struct dmar_domain *);
1696 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1697 }
1698
1699 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1700 pr_err("%s: Allocating domain array failed\n",
1701 iommu->name);
852bdb04 1702 kfree(iommu->domain_ids);
8bf47816 1703 kfree(iommu->domains);
852bdb04 1704 iommu->domain_ids = NULL;
8bf47816 1705 iommu->domains = NULL;
ba395927
KA		 1706 return -ENOMEM;
1707 }
1708
8bf47816
JR		 1709
1710
ba395927 1711 /*
c0e8a6c8
JR		 1712 * If Caching mode is set, then invalid translations are tagged
1713 * with domain-id 0, hence we need to pre-allocate it. We also
1714 * use domain-id 0 as a marker for non-allocated domain-id, so
1715 * make sure it is not used for a real domain.
ba395927 1716 */
c0e8a6c8
JR		 1717 set_bit(0, iommu->domain_ids);
1718
ba395927
KA		 1719 return 0;
1720}
ba395927 1721
ffebeb46 1722static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1723{
29a27719 1724 struct device_domain_info *info, *tmp;
55d94043 1725 unsigned long flags;
ba395927 1726
29a27719
JR		 1727 if (!iommu->domains || !iommu->domain_ids)
1728 return;
a4eaa86c 1729
bea64033 1730again:
55d94043 1731 spin_lock_irqsave(&device_domain_lock, flags);
29a27719
JR		 1732 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1733 struct dmar_domain *domain;
1734
1735 if (info->iommu != iommu)
1736 continue;
1737
1738 if (!info->dev || !info->domain)
1739 continue;
1740
1741 domain = info->domain;
1742
bea64033 1743 __dmar_remove_one_dev_info(info);
29a27719 1744
bea64033
JR		 1745 if (!domain_type_is_vm_or_si(domain)) {
1746 /*
1747 * The domain_exit() function can't be called under
1748 * device_domain_lock, as it takes this lock itself.
1749 * So release the lock here and re-run the loop
1750 * afterwards.
1751 */
1752 spin_unlock_irqrestore(&device_domain_lock, flags);
29a27719 1753 domain_exit(domain);
bea64033
JR		 1754 goto again;
1755 }
ba395927 1756 }
55d94043 1757 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1758
1759 if (iommu->gcmd & DMA_GCMD_TE)
1760 iommu_disable_translation(iommu);
ffebeb46 1761}
ba395927 1762
ffebeb46
JL		 1763static void free_dmar_iommu(struct intel_iommu *iommu)
1764{
1765 if ((iommu->domains) && (iommu->domain_ids)) {
86f004c7 1766 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
8bf47816
JR		 1767 int i;
1768
1769 for (i = 0; i < elems; i++)
1770 kfree(iommu->domains[i]);
ffebeb46
JL		 1771 kfree(iommu->domains);
1772 kfree(iommu->domain_ids);
1773 iommu->domains = NULL;
1774 iommu->domain_ids = NULL;
1775 }
ba395927 1776
d9630fe9
WH		 1777 g_iommus[iommu->seq_id] = NULL;
1778
ba395927
KA		 1779 /* free context mapping */
1780 free_context_table(iommu);
8a94ade4
DW		 1781
1782#ifdef CONFIG_INTEL_IOMMU_SVM
a222a7f0
DW		 1783 if (pasid_enabled(iommu)) {
1784 if (ecap_prs(iommu->ecap))
1785 intel_svm_finish_prq(iommu);
d9737953 1786 intel_svm_exit(iommu);
a222a7f0 1787 }
8a94ade4 1788#endif
ba395927
KA		 1789}
1790
ab8dfe25 1791static struct dmar_domain *alloc_domain(int flags)
ba395927 1792{
ba395927 1793 struct dmar_domain *domain;
ba395927
KA		 1794
1795 domain = alloc_domain_mem();
1796 if (!domain)
1797 return NULL;
1798
ab8dfe25 1799 memset(domain, 0, sizeof(*domain));
4c923d47 1800 domain->nid = -1;
ab8dfe25 1801 domain->flags = flags;
0824c592 1802 domain->has_iotlb_device = false;
92d03cc8 1803 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1804
1805 return domain;
1806}
1807
d160aca5
JR		 1808/* Must be called with iommu->lock */
1809static int domain_attach_iommu(struct dmar_domain *domain,
fb170fb4
JL		 1810 struct intel_iommu *iommu)
1811{
44bde614 1812 unsigned long ndomains;
55d94043 1813 int num;
44bde614 1814
55d94043 1815 assert_spin_locked(&device_domain_lock);
d160aca5 1816 assert_spin_locked(&iommu->lock);
ba395927 1817
29a27719
JR		 1818 domain->iommu_refcnt[iommu->seq_id] += 1;
1819 domain->iommu_count += 1;
1820 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
fb170fb4 1821 ndomains = cap_ndoms(iommu->cap);
d160aca5
JR		 1822 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1823
1824 if (num >= ndomains) {
1825 pr_err("%s: No free domain ids\n", iommu->name);
1826 domain->iommu_refcnt[iommu->seq_id] -= 1;
1827 domain->iommu_count -= 1;
55d94043 1828 return -ENOSPC;
2c2e2c38 1829 }
ba395927 1830
d160aca5
JR		 1831 set_bit(num, iommu->domain_ids);
1832 set_iommu_domain(iommu, num, domain);
1833
1834 domain->iommu_did[iommu->seq_id] = num;
1835 domain->nid = iommu->node;
fb170fb4 1836
fb170fb4
JL		 1837 domain_update_iommu_cap(domain);
1838 }
d160aca5 1839
55d94043 1840 return 0;
fb170fb4
JL		 1841}
1842
1843static int domain_detach_iommu(struct dmar_domain *domain,
1844 struct intel_iommu *iommu)
1845{
d160aca5 1846 int num, count = INT_MAX;
d160aca5 1847
55d94043 1848 assert_spin_locked(&device_domain_lock);
d160aca5 1849 assert_spin_locked(&iommu->lock);
fb170fb4 1850
29a27719
JR		 1851 domain->iommu_refcnt[iommu->seq_id] -= 1;
1852 count = --domain->iommu_count;
1853 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
d160aca5
JR		 1854 num = domain->iommu_did[iommu->seq_id];
1855 clear_bit(num, iommu->domain_ids);
1856 set_iommu_domain(iommu, num, NULL);
fb170fb4 1857
fb170fb4 1858 domain_update_iommu_cap(domain);
c0e8a6c8 1859 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4 1860 }
fb170fb4
JL		 1861
1862 return count;
1863}
1864
ba395927 1865static struct iova_domain reserved_iova_list;
8a443df4 1866static struct lock_class_key reserved_rbtree_key;
ba395927 1867
51a63e67 1868static int dmar_init_reserved_ranges(void)
ba395927
KA		 1869{
1870 struct pci_dev *pdev = NULL;
1871 struct iova *iova;
1872 int i;
ba395927 1873
aa3ac946 1874 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
ba395927 1875
8a443df4
MG		 1876 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1877 &reserved_rbtree_key);
1878
ba395927
KA		 1879 /* IOAPIC ranges shouldn't be accessed by DMA */
1880 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1881 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1882 if (!iova) {
9f10e5bf 1883 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1884 return -ENODEV;
1885 }
ba395927
KA		 1886
1887 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1888 for_each_pci_dev(pdev) {
1889 struct resource *r;
1890
1891 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1892 r = &pdev->resource[i];
1893 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1894 continue;
1a4a4551
DW		 1895 iova = reserve_iova(&reserved_iova_list,
1896 IOVA_PFN(r->start),
1897 IOVA_PFN(r->end));
51a63e67 1898 if (!iova) {
9f10e5bf 1899 pr_err("Reserve iova failed\n");
51a63e67
JC		 1900 return -ENODEV;
1901 }
ba395927
KA		 1902 }
1903 }
51a63e67 1904 return 0;
ba395927
KA		 1905}
1906
1907static void domain_reserve_special_ranges(struct dmar_domain *domain)
1908{
1909 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1910}
1911
1912static inline int guestwidth_to_adjustwidth(int gaw)
1913{
1914 int agaw;
1915 int r = (gaw - 12) % 9;
1916
1917 if (r == 0)
1918 agaw = gaw;
1919 else
1920 agaw = gaw + 9 - r;
1921 if (agaw > 64)
1922 agaw = 64;
1923 return agaw;
1924}
1925
dc534b25
JR		 1926static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1927 int guest_width)
ba395927 1928{
ba395927
KA		 1929 int adjust_width, agaw;
1930 unsigned long sagaw;
13cf0174 1931 int err;
ba395927 1932
aa3ac946 1933 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
13cf0174
JR		 1934
1935 err = init_iova_flush_queue(&domain->iovad,
1936 iommu_flush_iova, iova_entry_free);
1937 if (err)
1938 return err;
1939
ba395927
KA		 1940 domain_reserve_special_ranges(domain);
1941
1942 /* calculate AGAW */
ba395927
KA		 1943 if (guest_width > cap_mgaw(iommu->cap))
1944 guest_width = cap_mgaw(iommu->cap);
1945 domain->gaw = guest_width;
1946 adjust_width = guestwidth_to_adjustwidth(guest_width);
1947 agaw = width_to_agaw(adjust_width);
1948 sagaw = cap_sagaw(iommu->cap);
1949 if (!test_bit(agaw, &sagaw)) {
1950 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1951 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1952 agaw = find_next_bit(&sagaw, 5, agaw);
1953 if (agaw >= 5)
1954 return -ENODEV;
1955 }
1956 domain->agaw = agaw;
ba395927 1957
8e604097
WH		 1958 if (ecap_coherent(iommu->ecap))
1959 domain->iommu_coherency = 1;
1960 else
1961 domain->iommu_coherency = 0;
1962
58c610bd
SY		 1963 if (ecap_sc_support(iommu->ecap))
1964 domain->iommu_snooping = 1;
1965 else
1966 domain->iommu_snooping = 0;
1967
214e39aa
DW		 1968 if (intel_iommu_superpage)
1969 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1970 else
1971 domain->iommu_superpage = 0;
1972
4c923d47 1973 domain->nid = iommu->node;
c7151a8d 1974
ba395927 1975 /* always allocate the top pgd */
4c923d47 1976 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1977 if (!domain->pgd)
1978 return -ENOMEM;
5b6985ce 1979 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1980 return 0;
1981}
1982
1983static void domain_exit(struct dmar_domain *domain)
1984{
ea8ea460 1985 struct page *freelist = NULL;
ba395927
KA		 1986
1987 /* Domain 0 is reserved, so dont process it */
1988 if (!domain)
1989 return;
1990
d160aca5
JR		 1991 /* Remove associated devices and clear attached or cached domains */
1992 rcu_read_lock();
ba395927 1993 domain_remove_dev_info(domain);
d160aca5 1994 rcu_read_unlock();
92d03cc8 1995
ba395927
KA		 1996 /* destroy iovas */
1997 put_iova_domain(&domain->iovad);
ba395927 1998
ea8ea460 1999 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 2000
ea8ea460
DW		 2001 dma_free_pagelist(freelist);
2002
ba395927
KA		 2003 free_domain_mem(domain);
2004}
2005
64ae892b
DW		 2006static int domain_context_mapping_one(struct dmar_domain *domain,
2007 struct intel_iommu *iommu,
28ccce0d 2008 u8 bus, u8 devfn)
ba395927 2009{
c6c2cebd 2010 u16 did = domain->iommu_did[iommu->seq_id];
28ccce0d
JR		 2011 int translation = CONTEXT_TT_MULTI_LEVEL;
2012 struct device_domain_info *info = NULL;
ba395927 2013 struct context_entry *context;
ba395927 2014 unsigned long flags;
ea6606b0 2015 struct dma_pte *pgd;
55d94043 2016 int ret, agaw;
28ccce0d 2017
c6c2cebd
JR		 2018 WARN_ON(did == 0);
2019
28ccce0d
JR		 2020 if (hw_pass_through && domain_type_is_si(domain))
2021 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 2022
2023 pr_debug("Set context mapping for %02x:%02x.%d\n",
2024 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 2025
ba395927 2026 BUG_ON(!domain->pgd);
5331fe6f 2027
55d94043
JR		 2028 spin_lock_irqsave(&device_domain_lock, flags);
2029 spin_lock(&iommu->lock);
2030
2031 ret = -ENOMEM;
03ecc32c 2032 context = iommu_context_addr(iommu, bus, devfn, 1);
ba395927 2033 if (!context)
55d94043 2034 goto out_unlock;
ba395927 2035
55d94043
JR		 2036 ret = 0;
2037 if (context_present(context))
2038 goto out_unlock;
cf484d0e 2039
aec0e861
XP		 2040 /*
2041 * For kdump cases, old valid entries may be cached due to the
2042 * in-flight DMA and copied pgtable, but there is no unmapping
2043 * behaviour for them, thus we need an explicit cache flush for
2044 * the newly-mapped device. For kdump, at this point, the device
2045 * is supposed to finish reset at its driver probe stage, so no
2046 * in-flight DMA will exist, and we don't need to worry anymore
2047 * hereafter.
2048 */
2049 if (context_copied(context)) {
2050 u16 did_old = context_domain_id(context);
2051
b117e038 2052 if (did_old < cap_ndoms(iommu->cap)) {
aec0e861
XP		 2053 iommu->flush.flush_context(iommu, did_old,
2054 (((u16)bus) << 8) | devfn,
2055 DMA_CCMD_MASK_NOBIT,
2056 DMA_CCMD_DEVICE_INVL);
f73a7eee
KA		 2057 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2058 DMA_TLB_DSI_FLUSH);
2059 }
aec0e861
XP		 2060 }
2061
ea6606b0
WH		 2062 pgd = domain->pgd;
2063
de24e553 2064 context_clear_entry(context);
c6c2cebd 2065 context_set_domain_id(context, did);
ea6606b0 2066
de24e553
JR		 2067 /*
2068 * Skip top levels of page tables for iommu which has less agaw
2069 * than default. Unnecessary for PT mode.
2070 */
93a23a72 2071 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553 2072 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
55d94043 2073 ret = -ENOMEM;
de24e553 2074 pgd = phys_to_virt(dma_pte_addr(pgd));
55d94043
JR		 2075 if (!dma_pte_present(pgd))
2076 goto out_unlock;
ea6606b0 2077 }
4ed0d3e6 2078
64ae892b 2079 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
b16d0cb9
DW		 2080 if (info && info->ats_supported)
2081 translation = CONTEXT_TT_DEV_IOTLB;
2082 else
2083 translation = CONTEXT_TT_MULTI_LEVEL;
de24e553 2084
93a23a72
YZ		 2085 context_set_address_root(context, virt_to_phys(pgd));
2086 context_set_address_width(context, iommu->agaw);
de24e553
JR		 2087 } else {
2088 /*
2089 * In pass through mode, AW must be programmed to
2090 * indicate the largest AGAW value supported by
2091 * hardware. And ASR is ignored by hardware.
2092 */
2093 context_set_address_width(context, iommu->msagaw);
93a23a72 2094 }
4ed0d3e6
FY		 2095
2096 context_set_translation_type(context, translation);
c07e7d21
MM		 2097 context_set_fault_enable(context);
2098 context_set_present(context);
5331fe6f 2099 domain_flush_cache(domain, context, sizeof(*context));
ba395927 2100
4c25a2c1
DW		 2101 /*
2102 * It's a non-present to present mapping. If hardware doesn't cache
2103 * non-present entry we only need to flush the write-buffer. If the
2104 * _does_ cache non-present entries, then it does so in the special
2105 * domain #0, which we have to flush:
2106 */
2107 if (cap_caching_mode(iommu->cap)) {
2108 iommu->flush.flush_context(iommu, 0,
2109 (((u16)bus) << 8) | devfn,
2110 DMA_CCMD_MASK_NOBIT,
2111 DMA_CCMD_DEVICE_INVL);
c6c2cebd 2112 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 2113 } else {
ba395927 2114 iommu_flush_write_buffer(iommu);
4c25a2c1 2115 }
93a23a72 2116 iommu_enable_dev_iotlb(info);
c7151a8d 2117
55d94043
JR		 2118 ret = 0;
2119
2120out_unlock:
2121 spin_unlock(&iommu->lock);
2122 spin_unlock_irqrestore(&device_domain_lock, flags);
fb170fb4 2123
5c365d18 2124 return ret;
ba395927
KA		 2125}
2126
579305f7
AW		 2127struct domain_context_mapping_data {
2128 struct dmar_domain *domain;
2129 struct intel_iommu *iommu;
579305f7
AW		 2130};
2131
2132static int domain_context_mapping_cb(struct pci_dev *pdev,
2133 u16 alias, void *opaque)
2134{
2135 struct domain_context_mapping_data *data = opaque;
2136
2137 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2138 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2139}
2140
ba395927 2141static int
28ccce0d 2142domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2143{
64ae892b 2144 struct intel_iommu *iommu;
156baca8 2145 u8 bus, devfn;
579305f7 2146 struct domain_context_mapping_data data;
64ae892b 2147
e1f167f3 2148 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2149 if (!iommu)
2150 return -ENODEV;
ba395927 2151
579305f7 2152 if (!dev_is_pci(dev))
28ccce0d 2153 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2154
2155 data.domain = domain;
2156 data.iommu = iommu;
579305f7
AW		 2157
2158 return pci_for_each_dma_alias(to_pci_dev(dev),
2159 &domain_context_mapping_cb, &data);
2160}
2161
2162static int domain_context_mapped_cb(struct pci_dev *pdev,
2163 u16 alias, void *opaque)
2164{
2165 struct intel_iommu *iommu = opaque;
2166
2167 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2168}
2169
e1f167f3 2170static int domain_context_mapped(struct device *dev)
ba395927 2171{
5331fe6f 2172 struct intel_iommu *iommu;
156baca8 2173 u8 bus, devfn;
5331fe6f 2174
e1f167f3 2175 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2176 if (!iommu)
2177 return -ENODEV;
ba395927 2178
579305f7
AW		 2179 if (!dev_is_pci(dev))
2180 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2181
579305f7
AW		 2182 return !pci_for_each_dma_alias(to_pci_dev(dev),
2183 domain_context_mapped_cb, iommu);
ba395927
KA		 2184}
2185
f532959b
FY		 2186/* Returns a number of VTD pages, but aligned to MM page size */
2187static inline unsigned long aligned_nrpages(unsigned long host_addr,
2188 size_t size)
2189{
2190 host_addr &= ~PAGE_MASK;
2191 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2192}
2193
6dd9a7c7
YS		 2194/* Return largest possible superpage level for a given mapping */
2195static inline int hardware_largepage_caps(struct dmar_domain *domain,
2196 unsigned long iov_pfn,
2197 unsigned long phy_pfn,
2198 unsigned long pages)
2199{
2200 int support, level = 1;
2201 unsigned long pfnmerge;
2202
2203 support = domain->iommu_superpage;
2204
2205 /* To use a large page, the virtual *and* physical addresses
2206 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2207 of them will mean we have to use smaller pages. So just
2208 merge them and check both at once. */
2209 pfnmerge = iov_pfn | phy_pfn;
2210
2211 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2212 pages >>= VTD_STRIDE_SHIFT;
2213 if (!pages)
2214 break;
2215 pfnmerge >>= VTD_STRIDE_SHIFT;
2216 level++;
2217 support--;
2218 }
2219 return level;
2220}
2221
9051aa02
DW		 2222static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2223 struct scatterlist *sg, unsigned long phys_pfn,
2224 unsigned long nr_pages, int prot)
e1605495
DW		 2225{
2226 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2227 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2228 unsigned long sg_res = 0;
6dd9a7c7
YS		 2229 unsigned int largepage_lvl = 0;
2230 unsigned long lvl_pages = 0;
e1605495 2231
162d1b10 2232 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2233
2234 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2235 return -EINVAL;
2236
2237 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2238
cc4f14aa
JL		 2239 if (!sg) {
2240 sg_res = nr_pages;
9051aa02
DW		 2241 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2242 }
2243
6dd9a7c7 2244 while (nr_pages > 0) {
c85994e4
DW		 2245 uint64_t tmp;
2246
e1605495 2247 if (!sg_res) {
29a90b70
RM		 2248 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2249
f532959b 2250 sg_res = aligned_nrpages(sg->offset, sg->length);
29a90b70 2251 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
e1605495 2252 sg->dma_length = sg->length;
29a90b70 2253 pteval = (sg_phys(sg) - pgoff) | prot;
6dd9a7c7 2254 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2255 }
6dd9a7c7 2256
e1605495 2257 if (!pte) {
6dd9a7c7
YS		 2258 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2259
5cf0a76f 2260 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2261 if (!pte)
2262 return -ENOMEM;
6dd9a7c7 2263 /* It is large page*/
6491d4d0 2264 if (largepage_lvl > 1) {
ba2374fd
CZ		 2265 unsigned long nr_superpages, end_pfn;
2266
6dd9a7c7 2267 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb 2268 lvl_pages = lvl_to_nr_pages(largepage_lvl);
ba2374fd
CZ		 2269
2270 nr_superpages = sg_res / lvl_pages;
2271 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2272
d41a4adb
JL		 2273 /*
2274 * Ensure that old small page tables are
ba2374fd 2275 * removed to make room for superpage(s).
bc24c571
DD		 2276 * We're adding new large pages, so make sure
2277 * we don't remove their parent tables.
d41a4adb 2278 */
bc24c571
DD		 2279 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2280 largepage_lvl + 1);
6491d4d0 2281 } else {
6dd9a7c7 2282 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2283 }
6dd9a7c7 2284
e1605495
DW		 2285 }
2286 /* We don't need lock here, nobody else
2287 * touches the iova range
2288 */
7766a3fb 2289 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2290 if (tmp) {
1bf20f0d 2291 static int dumps = 5;
9f10e5bf
JR		 2292 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2293 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2294 if (dumps) {
2295 dumps--;
2296 debug_dma_dump_mappings(NULL);
2297 }
2298 WARN_ON(1);
2299 }
6dd9a7c7
YS		 2300
2301 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2302
2303 BUG_ON(nr_pages < lvl_pages);
2304 BUG_ON(sg_res < lvl_pages);
2305
2306 nr_pages -= lvl_pages;
2307 iov_pfn += lvl_pages;
2308 phys_pfn += lvl_pages;
2309 pteval += lvl_pages * VTD_PAGE_SIZE;
2310 sg_res -= lvl_pages;
2311
2312 /* If the next PTE would be the first in a new page, then we
2313 need to flush the cache on the entries we've just written.
2314 And then we'll need to recalculate 'pte', so clear it and
2315 let it get set again in the if (!pte) block above.
2316
2317 If we're done (!nr_pages) we need to flush the cache too.
2318
2319 Also if we've been setting superpages, we may need to
2320 recalculate 'pte' and switch back to smaller pages for the
2321 end of the mapping, if the trailing size is not enough to
2322 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2323 pte++;
6dd9a7c7
YS		 2324 if (!nr_pages || first_pte_in_page(pte) ||
2325 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2326 domain_flush_cache(domain, first_pte,
2327 (void *)pte - (void *)first_pte);
2328 pte = NULL;
2329 }
6dd9a7c7
YS		 2330
2331 if (!sg_res && nr_pages)
e1605495
DW		 2332 sg = sg_next(sg);
2333 }
2334 return 0;
2335}
2336
87684fd9
PX		 2337static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2338 struct scatterlist *sg, unsigned long phys_pfn,
2339 unsigned long nr_pages, int prot)
2340{
2341 int ret;
2342 struct intel_iommu *iommu;
2343
2344 /* Do the real mapping first */
2345 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2346 if (ret)
2347 return ret;
2348
2349 /* Notify about the new mapping */
2350 if (domain_type_is_vm(domain)) {
2351 /* VM typed domains can have more than one IOMMUs */
2352 int iommu_id;
2353 for_each_domain_iommu(iommu_id, domain) {
2354 iommu = g_iommus[iommu_id];
2355 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2356 }
2357 } else {
2358 /* General domains only have one IOMMU */
2359 iommu = domain_get_iommu(domain);
2360 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2361 }
2362
2363 return 0;
2364}
2365
9051aa02
DW		 2366static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2367 struct scatterlist *sg, unsigned long nr_pages,
2368 int prot)
ba395927 2369{
87684fd9 2370 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
9051aa02 2371}
6f6a00e4 2372
9051aa02
DW		 2373static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2374 unsigned long phys_pfn, unsigned long nr_pages,
2375 int prot)
2376{
87684fd9 2377 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2378}
2379
2452d9db 2380static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2381{
5082219b
FS		 2382 unsigned long flags;
2383 struct context_entry *context;
2384 u16 did_old;
2385
c7151a8d
WH		 2386 if (!iommu)
2387 return;
8c11e798 2388
5082219b
FS		 2389 spin_lock_irqsave(&iommu->lock, flags);
2390 context = iommu_context_addr(iommu, bus, devfn, 0);
2391 if (!context) {
2392 spin_unlock_irqrestore(&iommu->lock, flags);
2393 return;
2394 }
2395 did_old = context_domain_id(context);
2396 context_clear_entry(context);
2397 __iommu_flush_cache(iommu, context, sizeof(*context));
2398 spin_unlock_irqrestore(&iommu->lock, flags);
2399 iommu->flush.flush_context(iommu,
2400 did_old,
2401 (((u16)bus) << 8) | devfn,
2402 DMA_CCMD_MASK_NOBIT,
2403 DMA_CCMD_DEVICE_INVL);
2404 iommu->flush.flush_iotlb(iommu,
2405 did_old,
2406 0,
2407 0,
2408 DMA_TLB_DSI_FLUSH);
ba395927
KA		 2409}
2410
109b9b04
DW		 2411static inline void unlink_domain_info(struct device_domain_info *info)
2412{
2413 assert_spin_locked(&device_domain_lock);
2414 list_del(&info->link);
2415 list_del(&info->global);
2416 if (info->dev)
0bcb3e28 2417 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2418}
2419
ba395927
KA		 2420static void domain_remove_dev_info(struct dmar_domain *domain)
2421{
3a74ca01 2422 struct device_domain_info *info, *tmp;
fb170fb4 2423 unsigned long flags;
ba395927
KA		 2424
2425 spin_lock_irqsave(&device_domain_lock, flags);
76f45fe3 2426 list_for_each_entry_safe(info, tmp, &domain->devices, link)
127c7615 2427 __dmar_remove_one_dev_info(info);
ba395927
KA		 2428 spin_unlock_irqrestore(&device_domain_lock, flags);
2429}
2430
2431/*
2432 * find_domain
1525a29a 2433 * Note: we use struct device->archdata.iommu stores the info
ba395927 2434 */
1525a29a 2435static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2436{
2437 struct device_domain_info *info;
2438
2439 /* No lock here, assumes no domain exit in normal case */
1525a29a 2440 info = dev->archdata.iommu;
b316d02a 2441 if (likely(info))
ba395927
KA		 2442 return info->domain;
2443 return NULL;
2444}
2445
5a8f40e8 2446static inline struct device_domain_info *
745f2586
JL		 2447dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2448{
2449 struct device_domain_info *info;
2450
2451 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2452 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2453 info->devfn == devfn)
5a8f40e8 2454 return info;
745f2586
JL		 2455
2456 return NULL;
2457}
2458
5db31569
JR		 2459static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2460 int bus, int devfn,
2461 struct device *dev,
2462 struct dmar_domain *domain)
745f2586 2463{
5a8f40e8 2464 struct dmar_domain *found = NULL;
745f2586
JL		 2465 struct device_domain_info *info;
2466 unsigned long flags;
d160aca5 2467 int ret;
745f2586
JL		 2468
2469 info = alloc_devinfo_mem();
2470 if (!info)
b718cd3d 2471 return NULL;
745f2586 2472
745f2586
JL		 2473 info->bus = bus;
2474 info->devfn = devfn;
b16d0cb9
DW		 2475 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2476 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2477 info->ats_qdep = 0;
745f2586
JL		 2478 info->dev = dev;
2479 info->domain = domain;
5a8f40e8 2480 info->iommu = iommu;
cc580e41 2481 info->pasid_table = NULL;
745f2586 2482
b16d0cb9
DW		 2483 if (dev && dev_is_pci(dev)) {
2484 struct pci_dev *pdev = to_pci_dev(info->dev);
2485
cef74409
GK		 2486 if (!pci_ats_disabled() &&
2487 ecap_dev_iotlb_support(iommu->ecap) &&
b16d0cb9
DW		 2488 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2489 dmar_find_matched_atsr_unit(pdev))
2490 info->ats_supported = 1;
2491
2492 if (ecs_enabled(iommu)) {
2493 if (pasid_enabled(iommu)) {
2494 int features = pci_pasid_features(pdev);
2495 if (features >= 0)
2496 info->pasid_supported = features | 1;
2497 }
2498
2499 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2500 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2501 info->pri_supported = 1;
2502 }
2503 }
2504
745f2586
JL		 2505 spin_lock_irqsave(&device_domain_lock, flags);
2506 if (dev)
0bcb3e28 2507 found = find_domain(dev);
f303e507
JR		 2508
2509 if (!found) {
5a8f40e8 2510 struct device_domain_info *info2;
41e80dca 2511 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
f303e507
JR		 2512 if (info2) {
2513 found = info2->domain;
2514 info2->dev = dev;
2515 }
5a8f40e8 2516 }
f303e507 2517
745f2586
JL		 2518 if (found) {
2519 spin_unlock_irqrestore(&device_domain_lock, flags);
2520 free_devinfo_mem(info);
b718cd3d
DW		 2521 /* Caller must free the original domain */
2522 return found;
745f2586
JL		 2523 }
2524
d160aca5
JR		 2525 spin_lock(&iommu->lock);
2526 ret = domain_attach_iommu(domain, iommu);
2527 spin_unlock(&iommu->lock);
2528
2529 if (ret) {
c6c2cebd 2530 spin_unlock_irqrestore(&device_domain_lock, flags);
499f3aa4 2531 free_devinfo_mem(info);
c6c2cebd
JR		 2532 return NULL;
2533 }
c6c2cebd 2534
b718cd3d
DW		 2535 list_add(&info->link, &domain->devices);
2536 list_add(&info->global, &device_domain_list);
2537 if (dev)
2538 dev->archdata.iommu = info;
a7fc93fe
LB		 2539
2540 if (dev && dev_is_pci(dev) && info->pasid_supported) {
2541 ret = intel_pasid_alloc_table(dev);
2542 if (ret) {
be9e6598
LB		 2543 pr_warn("No pasid table for %s, pasid disabled\n",
2544 dev_name(dev));
2545 info->pasid_supported = 0;
a7fc93fe
LB		 2546 }
2547 }
b718cd3d
DW		 2548 spin_unlock_irqrestore(&device_domain_lock, flags);
2549
cc4e2575
JR		 2550 if (dev && domain_context_mapping(domain, dev)) {
2551 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2552 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2553 return NULL;
2554 }
2555
b718cd3d 2556 return domain;
745f2586
JL		 2557}
2558
579305f7
AW		 2559static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2560{
2561 *(u16 *)opaque = alias;
2562 return 0;
2563}
2564
76208356 2565static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
ba395927 2566{
cc4e2575 2567 struct device_domain_info *info = NULL;
76208356 2568 struct dmar_domain *domain = NULL;
579305f7 2569 struct intel_iommu *iommu;
fcc35c63 2570 u16 dma_alias;
ba395927 2571 unsigned long flags;
aa4d066a 2572 u8 bus, devfn;
ba395927 2573
579305f7
AW		 2574 iommu = device_to_iommu(dev, &bus, &devfn);
2575 if (!iommu)
2576 return NULL;
2577
146922ec
DW		 2578 if (dev_is_pci(dev)) {
2579 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2580
579305f7
AW		 2581 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2582
2583 spin_lock_irqsave(&device_domain_lock, flags);
2584 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2585 PCI_BUS_NUM(dma_alias),
2586 dma_alias & 0xff);
2587 if (info) {
2588 iommu = info->iommu;
2589 domain = info->domain;
5a8f40e8 2590 }
579305f7 2591 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2592
76208356 2593 /* DMA alias already has a domain, use it */
579305f7 2594 if (info)
76208356 2595 goto out;
579305f7 2596 }
ba395927 2597
146922ec 2598 /* Allocate and initialize new domain for the device */
ab8dfe25 2599 domain = alloc_domain(0);
745f2586 2600 if (!domain)
579305f7 2601 return NULL;
dc534b25 2602 if (domain_init(domain, iommu, gaw)) {
579305f7
AW		 2603 domain_exit(domain);
2604 return NULL;
2c2e2c38 2605 }
ba395927 2606
76208356 2607out:
579305f7 2608
76208356
JR		 2609 return domain;
2610}
579305f7 2611
76208356
JR		 2612static struct dmar_domain *set_domain_for_dev(struct device *dev,
2613 struct dmar_domain *domain)
2614{
2615 struct intel_iommu *iommu;
2616 struct dmar_domain *tmp;
2617 u16 req_id, dma_alias;
2618 u8 bus, devfn;
2619
2620 iommu = device_to_iommu(dev, &bus, &devfn);
2621 if (!iommu)
2622 return NULL;
2623
2624 req_id = ((u16)bus << 8) | devfn;
2625
2626 if (dev_is_pci(dev)) {
2627 struct pci_dev *pdev = to_pci_dev(dev);
2628
2629 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2630
2631 /* register PCI DMA alias device */
2632 if (req_id != dma_alias) {
2633 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2634 dma_alias & 0xff, NULL, domain);
2635
2636 if (!tmp || tmp != domain)
2637 return tmp;
2638 }
ba395927
KA		 2639 }
2640
5db31569 2641 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
76208356
JR		 2642 if (!tmp || tmp != domain)
2643 return tmp;
2644
2645 return domain;
2646}
579305f7 2647
76208356
JR		 2648static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2649{
2650 struct dmar_domain *domain, *tmp;
2651
2652 domain = find_domain(dev);
2653 if (domain)
2654 goto out;
2655
2656 domain = find_or_alloc_domain(dev, gaw);
2657 if (!domain)
2658 goto out;
2659
2660 tmp = set_domain_for_dev(dev, domain);
2661 if (!tmp || domain != tmp) {
579305f7
AW		 2662 domain_exit(domain);
2663 domain = tmp;
2664 }
b718cd3d 2665
76208356
JR		 2666out:
2667
b718cd3d 2668 return domain;
ba395927
KA		 2669}
2670
b213203e
DW		 2671static int iommu_domain_identity_map(struct dmar_domain *domain,
2672 unsigned long long start,
2673 unsigned long long end)
ba395927 2674{
c5395d5c
DW		 2675 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2676 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2677
2678 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2679 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2680 pr_err("Reserving iova failed\n");
b213203e 2681 return -ENOMEM;
ba395927
KA		 2682 }
2683
af1089ce 2684 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2685 /*
2686 * RMRR range might have overlap with physical memory range,
2687 * clear it first
2688 */
c5395d5c 2689 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2690
87684fd9
PX		 2691 return __domain_mapping(domain, first_vpfn, NULL,
2692 first_vpfn, last_vpfn - first_vpfn + 1,
2693 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2694}
2695
d66ce54b
JR		 2696static int domain_prepare_identity_map(struct device *dev,
2697 struct dmar_domain *domain,
2698 unsigned long long start,
2699 unsigned long long end)
b213203e 2700{
19943b0e
DW		 2701 /* For _hardware_ passthrough, don't bother. But for software
2702 passthrough, we do it anyway -- it may indicate a memory
2703 range which is reserved in E820, so which didn't get set
2704 up to start with in si_domain */
2705 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2706 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2707 dev_name(dev), start, end);
19943b0e
DW		 2708 return 0;
2709 }
2710
9f10e5bf
JR		 2711 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2712 dev_name(dev), start, end);
2713
5595b528
DW		 2714 if (end < start) {
2715 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2716 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2717 dmi_get_system_info(DMI_BIOS_VENDOR),
2718 dmi_get_system_info(DMI_BIOS_VERSION),
2719 dmi_get_system_info(DMI_PRODUCT_VERSION));
d66ce54b 2720 return -EIO;
5595b528
DW		 2721 }
2722
2ff729f5
DW		 2723 if (end >> agaw_to_width(domain->agaw)) {
2724 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2725 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2726 agaw_to_width(domain->agaw),
2727 dmi_get_system_info(DMI_BIOS_VENDOR),
2728 dmi_get_system_info(DMI_BIOS_VERSION),
2729 dmi_get_system_info(DMI_PRODUCT_VERSION));
d66ce54b 2730 return -EIO;
2ff729f5 2731 }
19943b0e 2732
d66ce54b
JR		 2733 return iommu_domain_identity_map(domain, start, end);
2734}
ba395927 2735
d66ce54b
JR		 2736static int iommu_prepare_identity_map(struct device *dev,
2737 unsigned long long start,
2738 unsigned long long end)
2739{
2740 struct dmar_domain *domain;
2741 int ret;
2742
2743 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2744 if (!domain)
2745 return -ENOMEM;
2746
2747 ret = domain_prepare_identity_map(dev, domain, start, end);
2748 if (ret)
2749 domain_exit(domain);
b213203e 2750
ba395927 2751 return ret;
ba395927
KA		 2752}
2753
2754static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2755 struct device *dev)
ba395927 2756{
0b9d9753 2757 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2758 return 0;
0b9d9753
DW		 2759 return iommu_prepare_identity_map(dev, rmrr->base_address,
2760 rmrr->end_address);
ba395927
KA		 2761}
2762
d3f13810 2763#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2764static inline void iommu_prepare_isa(void)
2765{
2766 struct pci_dev *pdev;
2767 int ret;
2768
2769 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2770 if (!pdev)
2771 return;
2772
9f10e5bf 2773 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2774 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2775
2776 if (ret)
9f10e5bf 2777 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2778
9b27e82d 2779 pci_dev_put(pdev);
49a0429e
KA		 2780}
2781#else
2782static inline void iommu_prepare_isa(void)
2783{
2784 return;
2785}
d3f13810 2786#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2787
2c2e2c38 2788static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2789
071e1374 2790static int __init si_domain_init(int hw)
2c2e2c38 2791{
c7ab48d2 2792 int nid, ret = 0;
2c2e2c38 2793
ab8dfe25 2794 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2795 if (!si_domain)
2796 return -EFAULT;
2797
2c2e2c38
FY		 2798 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2799 domain_exit(si_domain);
2800 return -EFAULT;
2801 }
2802
0dc79715 2803 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2804
19943b0e
DW		 2805 if (hw)
2806 return 0;
2807
c7ab48d2 2808 for_each_online_node(nid) {
5dfe8660
TH		 2809 unsigned long start_pfn, end_pfn;
2810 int i;
2811
2812 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2813 ret = iommu_domain_identity_map(si_domain,
2814 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2815 if (ret)
2816 return ret;
2817 }
c7ab48d2
DW		 2818 }
2819
2c2e2c38
FY		 2820 return 0;
2821}
2822
9b226624 2823static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2824{
2825 struct device_domain_info *info;
2826
2827 if (likely(!iommu_identity_mapping))
2828 return 0;
2829
9b226624 2830 info = dev->archdata.iommu;
cb452a40
MT		 2831 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2832 return (info->domain == si_domain);
2c2e2c38 2833
2c2e2c38
FY		 2834 return 0;
2835}
2836
28ccce0d 2837static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2838{
0ac72664 2839 struct dmar_domain *ndomain;
5a8f40e8 2840 struct intel_iommu *iommu;
156baca8 2841 u8 bus, devfn;
2c2e2c38 2842
5913c9bf 2843 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2844 if (!iommu)
2845 return -ENODEV;
2846
5db31569 2847 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2848 if (ndomain != domain)
2849 return -EBUSY;
2c2e2c38
FY		 2850
2851 return 0;
2852}
2853
0b9d9753 2854static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2855{
2856 struct dmar_rmrr_unit *rmrr;
832bd858 2857 struct device *tmp;
ea2447f7
TM		 2858 int i;
2859
0e242612 2860 rcu_read_lock();
ea2447f7 2861 for_each_rmrr_units(rmrr) {
b683b230
JL		 2862 /*
2863 * Return TRUE if this RMRR contains the device that
2864 * is passed in.
2865 */
2866 for_each_active_dev_scope(rmrr->devices,
2867 rmrr->devices_cnt, i, tmp)
0b9d9753 2868 if (tmp == dev) {
0e242612 2869 rcu_read_unlock();
ea2447f7 2870 return true;
b683b230 2871 }
ea2447f7 2872 }
0e242612 2873 rcu_read_unlock();
ea2447f7
TM		 2874 return false;
2875}
2876
c875d2c1
AW		 2877/*
2878 * There are a couple cases where we need to restrict the functionality of
2879 * devices associated with RMRRs. The first is when evaluating a device for
2880 * identity mapping because problems exist when devices are moved in and out
2881 * of domains and their respective RMRR information is lost. This means that
2882 * a device with associated RMRRs will never be in a "passthrough" domain.
2883 * The second is use of the device through the IOMMU API. This interface
2884 * expects to have full control of the IOVA space for the device. We cannot
2885 * satisfy both the requirement that RMRR access is maintained and have an
2886 * unencumbered IOVA space. We also have no ability to quiesce the device's
2887 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2888 * We therefore prevent devices associated with an RMRR from participating in
2889 * the IOMMU API, which eliminates them from device assignment.
2890 *
2891 * In both cases we assume that PCI USB devices with RMRRs have them largely
2892 * for historical reasons and that the RMRR space is not actively used post
2893 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2894 *
2895 * The same exception is made for graphics devices, with the requirement that
2896 * any use of the RMRR regions will be torn down before assigning the device
2897 * to a guest.
c875d2c1
AW		 2898 */
2899static bool device_is_rmrr_locked(struct device *dev)
2900{
2901 if (!device_has_rmrr(dev))
2902 return false;
2903
2904 if (dev_is_pci(dev)) {
2905 struct pci_dev *pdev = to_pci_dev(dev);
2906
18436afd 2907 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2908 return false;
2909 }
2910
2911 return true;
2912}
2913
3bdb2591 2914static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2915{
ea2447f7 2916
3bdb2591
DW		 2917 if (dev_is_pci(dev)) {
2918 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2919
c875d2c1 2920 if (device_is_rmrr_locked(dev))
3bdb2591 2921 return 0;
e0fc7e0b 2922
3bdb2591
DW		 2923 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2924 return 1;
e0fc7e0b 2925
3bdb2591
DW		 2926 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2927 return 1;
6941af28 2928
3bdb2591 2929 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2930 return 0;
3bdb2591
DW		 2931
2932 /*
2933 * We want to start off with all devices in the 1:1 domain, and
2934 * take them out later if we find they can't access all of memory.
2935 *
2936 * However, we can't do this for PCI devices behind bridges,
2937 * because all PCI devices behind the same bridge will end up
2938 * with the same source-id on their transactions.
2939 *
2940 * Practically speaking, we can't change things around for these
2941 * devices at run-time, because we can't be sure there'll be no
2942 * DMA transactions in flight for any of their siblings.
2943 *
2944 * So PCI devices (unless they're on the root bus) as well as
2945 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2946 * the 1:1 domain, just in _case_ one of their siblings turns out
2947 * not to be able to map all of memory.
2948 */
2949 if (!pci_is_pcie(pdev)) {
2950 if (!pci_is_root_bus(pdev->bus))
2951 return 0;
2952 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2953 return 0;
2954 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2955 return 0;
3bdb2591
DW		 2956 } else {
2957 if (device_has_rmrr(dev))
2958 return 0;
2959 }
3dfc813d 2960
3bdb2591 2961 /*
3dfc813d 2962 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2963 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2964 * take them out of the 1:1 domain later.
2965 */
8fcc5372
CW		 2966 if (!startup) {
2967 /*
2968 * If the device's dma_mask is less than the system's memory
2969 * size then this is not a candidate for identity mapping.
2970 */
3bdb2591 2971 u64 dma_mask = *dev->dma_mask;
8fcc5372 2972
3bdb2591
DW		 2973 if (dev->coherent_dma_mask &&
2974 dev->coherent_dma_mask < dma_mask)
2975 dma_mask = dev->coherent_dma_mask;
8fcc5372 2976
3bdb2591 2977 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2978 }
6941af28
DW		 2979
2980 return 1;
2981}
2982
cf04eee8
DW		 2983static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2984{
2985 int ret;
2986
2987 if (!iommu_should_identity_map(dev, 1))
2988 return 0;
2989
28ccce0d 2990 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2991 if (!ret)
9f10e5bf
JR		 2992 pr_info("%s identity mapping for device %s\n",
2993 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2994 else if (ret == -ENODEV)
2995 /* device not associated with an iommu */
2996 ret = 0;
2997
2998 return ret;
2999}
3000
3001
071e1374 3002static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 3003{
2c2e2c38 3004 struct pci_dev *pdev = NULL;
cf04eee8
DW		 3005 struct dmar_drhd_unit *drhd;
3006 struct intel_iommu *iommu;
3007 struct device *dev;
3008 int i;
3009 int ret = 0;
2c2e2c38 3010
2c2e2c38 3011 for_each_pci_dev(pdev) {
cf04eee8
DW		 3012 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3013 if (ret)
3014 return ret;
3015 }
3016
3017 for_each_active_iommu(iommu, drhd)
3018 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3019 struct acpi_device_physical_node *pn;
3020 struct acpi_device *adev;
3021
3022 if (dev->bus != &acpi_bus_type)
3023 continue;
86080ccc 3024
cf04eee8
DW		 3025 adev= to_acpi_device(dev);
3026 mutex_lock(&adev->physical_node_lock);
3027 list_for_each_entry(pn, &adev->physical_node_list, node) {
3028 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3029 if (ret)
3030 break;
eae460b6 3031 }
cf04eee8
DW		 3032 mutex_unlock(&adev->physical_node_lock);
3033 if (ret)
3034 return ret;
62edf5dc 3035 }
2c2e2c38
FY		 3036
3037 return 0;
3038}
3039
ffebeb46
JL		 3040static void intel_iommu_init_qi(struct intel_iommu *iommu)
3041{
3042 /*
3043 * Start from the sane iommu hardware state.
3044 * If the queued invalidation is already initialized by us
3045 * (for example, while enabling interrupt-remapping) then
3046 * we got the things already rolling from a sane state.
3047 */
3048 if (!iommu->qi) {
3049 /*
3050 * Clear any previous faults.
3051 */
3052 dmar_fault(-1, iommu);
3053 /*
3054 * Disable queued invalidation if supported and already enabled
3055 * before OS handover.
3056 */
3057 dmar_disable_qi(iommu);
3058 }
3059
3060 if (dmar_enable_qi(iommu)) {
3061 /*
3062 * Queued Invalidate not enabled, use Register Based Invalidate
3063 */
3064 iommu->flush.flush_context = __iommu_flush_context;
3065 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 3066 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 3067 iommu->name);
3068 } else {
3069 iommu->flush.flush_context = qi_flush_context;
3070 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 3071 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 3072 }
3073}
3074
091d42e4 3075static int copy_context_table(struct intel_iommu *iommu,
dfddb969 3076 struct root_entry *old_re,
091d42e4
JR		 3077 struct context_entry **tbl,
3078 int bus, bool ext)
3079{
dbcd861f 3080 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
543c8dcf 3081 struct context_entry *new_ce = NULL, ce;
dfddb969 3082 struct context_entry *old_ce = NULL;
543c8dcf 3083 struct root_entry re;
091d42e4
JR		 3084 phys_addr_t old_ce_phys;
3085
3086 tbl_idx = ext ? bus * 2 : bus;
dfddb969 3087 memcpy(&re, old_re, sizeof(re));
091d42e4
JR		 3088
3089 for (devfn = 0; devfn < 256; devfn++) {
3090 /* First calculate the correct index */
3091 idx = (ext ? devfn * 2 : devfn) % 256;
3092
3093 if (idx == 0) {
3094 /* First save what we may have and clean up */
3095 if (new_ce) {
3096 tbl[tbl_idx] = new_ce;
3097 __iommu_flush_cache(iommu, new_ce,
3098 VTD_PAGE_SIZE);
3099 pos = 1;
3100 }
3101
3102 if (old_ce)
3103 iounmap(old_ce);
3104
3105 ret = 0;
3106 if (devfn < 0x80)
543c8dcf 3107 old_ce_phys = root_entry_lctp(&re);
091d42e4 3108 else
543c8dcf 3109 old_ce_phys = root_entry_uctp(&re);
091d42e4
JR		 3110
3111 if (!old_ce_phys) {
3112 if (ext && devfn == 0) {
3113 /* No LCTP, try UCTP */
3114 devfn = 0x7f;
3115 continue;
3116 } else {
3117 goto out;
3118 }
3119 }
3120
3121 ret = -ENOMEM;
dfddb969
DW		 3122 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3123 MEMREMAP_WB);
091d42e4
JR		 3124 if (!old_ce)
3125 goto out;
3126
3127 new_ce = alloc_pgtable_page(iommu->node);
3128 if (!new_ce)
3129 goto out_unmap;
3130
3131 ret = 0;
3132 }
3133
3134 /* Now copy the context entry */
dfddb969 3135 memcpy(&ce, old_ce + idx, sizeof(ce));
091d42e4 3136
cf484d0e 3137 if (!__context_present(&ce))
091d42e4
JR		 3138 continue;
3139
dbcd861f
JR		 3140 did = context_domain_id(&ce);
3141 if (did >= 0 && did < cap_ndoms(iommu->cap))
3142 set_bit(did, iommu->domain_ids);
3143
cf484d0e
JR		 3144 /*
3145 * We need a marker for copied context entries. This
3146 * marker needs to work for the old format as well as
3147 * for extended context entries.
3148 *
3149 * Bit 67 of the context entry is used. In the old
3150 * format this bit is available to software, in the
3151 * extended format it is the PGE bit, but PGE is ignored
3152 * by HW if PASIDs are disabled (and thus still
3153 * available).
3154 *
3155 * So disable PASIDs first and then mark the entry
3156 * copied. This means that we don't copy PASID
3157 * translations from the old kernel, but this is fine as
3158 * faults there are not fatal.
3159 */
3160 context_clear_pasid_enable(&ce);
3161 context_set_copied(&ce);
3162
091d42e4
JR		 3163 new_ce[idx] = ce;
3164 }
3165
3166 tbl[tbl_idx + pos] = new_ce;
3167
3168 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3169
3170out_unmap:
dfddb969 3171 memunmap(old_ce);
091d42e4
JR		 3172
3173out:
3174 return ret;
3175}
3176
3177static int copy_translation_tables(struct intel_iommu *iommu)
3178{
3179 struct context_entry **ctxt_tbls;
dfddb969 3180 struct root_entry *old_rt;
091d42e4
JR		 3181 phys_addr_t old_rt_phys;
3182 int ctxt_table_entries;
3183 unsigned long flags;
3184 u64 rtaddr_reg;
3185 int bus, ret;
c3361f2f 3186 bool new_ext, ext;
091d42e4
JR		 3187
3188 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3189 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 3190 new_ext = !!ecap_ecs(iommu->ecap);
3191
3192 /*
3193 * The RTT bit can only be changed when translation is disabled,
3194 * but disabling translation means to open a window for data
3195 * corruption. So bail out and don't copy anything if we would
3196 * have to change the bit.
3197 */
3198 if (new_ext != ext)
3199 return -EINVAL;
091d42e4
JR		 3200
3201 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3202 if (!old_rt_phys)
3203 return -EINVAL;
3204
dfddb969 3205 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
091d42e4
JR		 3206 if (!old_rt)
3207 return -ENOMEM;
3208
3209 /* This is too big for the stack - allocate it from slab */
3210 ctxt_table_entries = ext ? 512 : 256;
3211 ret = -ENOMEM;
6396bb22 3212 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
091d42e4
JR		 3213 if (!ctxt_tbls)
3214 goto out_unmap;
3215
3216 for (bus = 0; bus < 256; bus++) {
3217 ret = copy_context_table(iommu, &old_rt[bus],
3218 ctxt_tbls, bus, ext);
3219 if (ret) {
3220 pr_err("%s: Failed to copy context table for bus %d\n",
3221 iommu->name, bus);
3222 continue;
3223 }
3224 }
3225
3226 spin_lock_irqsave(&iommu->lock, flags);
3227
3228 /* Context tables are copied, now write them to the root_entry table */
3229 for (bus = 0; bus < 256; bus++) {
3230 int idx = ext ? bus * 2 : bus;
3231 u64 val;
3232
3233 if (ctxt_tbls[idx]) {
3234 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3235 iommu->root_entry[bus].lo = val;
3236 }
3237
3238 if (!ext || !ctxt_tbls[idx + 1])
3239 continue;
3240
3241 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3242 iommu->root_entry[bus].hi = val;
3243 }
3244
3245 spin_unlock_irqrestore(&iommu->lock, flags);
3246
3247 kfree(ctxt_tbls);
3248
3249 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3250
3251 ret = 0;
3252
3253out_unmap:
dfddb969 3254 memunmap(old_rt);
091d42e4
JR		 3255
3256 return ret;
3257}
3258
b779260b 3259static int __init init_dmars(void)
ba395927
KA		 3260{
3261 struct dmar_drhd_unit *drhd;
3262 struct dmar_rmrr_unit *rmrr;
a87f4918 3263 bool copied_tables = false;
832bd858 3264 struct device *dev;
ba395927 3265 struct intel_iommu *iommu;
13cf0174 3266 int i, ret;
2c2e2c38 3267
ba395927
KA		 3268 /*
3269 * for each drhd
3270 * allocate root
3271 * initialize and program root entry to not present
3272 * endfor
3273 */
3274 for_each_drhd_unit(drhd) {
5e0d2a6f 3275 /*
3276 * lock not needed as this is only incremented in the single
3277 * threaded kernel __init code path all other access are read
3278 * only
3279 */
78d8e704 3280 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3281 g_num_of_iommus++;
3282 continue;
3283 }
9f10e5bf 3284 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3285 }
3286
ffebeb46
JL		 3287 /* Preallocate enough resources for IOMMU hot-addition */
3288 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3289 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3290
d9630fe9
WH		 3291 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3292 GFP_KERNEL);
3293 if (!g_iommus) {
9f10e5bf 3294 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3295 ret = -ENOMEM;
3296 goto error;
3297 }
3298
7c919779 3299 for_each_active_iommu(iommu, drhd) {
56283174
LB		 3300 /*
3301 * Find the max pasid size of all IOMMU's in the system.
3302 * We need to ensure the system pasid table is no bigger
3303 * than the smallest supported.
3304 */
3305 if (pasid_enabled(iommu)) {
3306 u32 temp = 2 << ecap_pss(iommu->ecap);
3307
3308 intel_pasid_max_id = min_t(u32, temp,
3309 intel_pasid_max_id);
3310 }
3311
d9630fe9 3312 g_iommus[iommu->seq_id] = iommu;
ba395927 3313
b63d80d1
JR		 3314 intel_iommu_init_qi(iommu);
3315
e61d98d8
SS		 3316 ret = iommu_init_domains(iommu);
3317 if (ret)
989d51fc 3318 goto free_iommu;
e61d98d8 3319
4158c2ec
JR		 3320 init_translation_status(iommu);
3321
091d42e4
JR		 3322 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3323 iommu_disable_translation(iommu);
3324 clear_translation_pre_enabled(iommu);
3325 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3326 iommu->name);
3327 }
4158c2ec 3328
ba395927
KA		 3329 /*
3330 * TBD:
3331 * we could share the same root & context tables
25985edc 3332 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3333 */
3334 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3335 if (ret)
989d51fc 3336 goto free_iommu;
5f0a7f76 3337
091d42e4
JR		 3338 if (translation_pre_enabled(iommu)) {
3339 pr_info("Translation already enabled - trying to copy translation structures\n");
3340
3341 ret = copy_translation_tables(iommu);
3342 if (ret) {
3343 /*
3344 * We found the IOMMU with translation
3345 * enabled - but failed to copy over the
3346 * old root-entry table. Try to proceed
3347 * by disabling translation now and
3348 * allocating a clean root-entry table.
3349 * This might cause DMAR faults, but
3350 * probably the dump will still succeed.
3351 */
3352 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3353 iommu->name);
3354 iommu_disable_translation(iommu);
3355 clear_translation_pre_enabled(iommu);
3356 } else {
3357 pr_info("Copied translation tables from previous kernel for %s\n",
3358 iommu->name);
a87f4918 3359 copied_tables = true;
091d42e4
JR		 3360 }
3361 }
3362
4ed0d3e6 3363 if (!ecap_pass_through(iommu->ecap))
19943b0e 3364 hw_pass_through = 0;
8a94ade4
DW		 3365#ifdef CONFIG_INTEL_IOMMU_SVM
3366 if (pasid_enabled(iommu))
d9737953 3367 intel_svm_init(iommu);
8a94ade4 3368#endif
ba395927
KA		 3369 }
3370
a4c34ff1
JR		 3371 /*
3372 * Now that qi is enabled on all iommus, set the root entry and flush
3373 * caches. This is required on some Intel X58 chipsets, otherwise the
3374 * flush_context function will loop forever and the boot hangs.
3375 */
3376 for_each_active_iommu(iommu, drhd) {
3377 iommu_flush_write_buffer(iommu);
3378 iommu_set_root_entry(iommu);
3379 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3380 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3381 }
3382
19943b0e 3383 if (iommu_pass_through)
e0fc7e0b
DW		 3384 iommu_identity_mapping |= IDENTMAP_ALL;
3385
d3f13810 3386#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3387 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3388#endif
e0fc7e0b 3389
21e722c4
AR		 3390 check_tylersburg_isoch();
3391
86080ccc
JR		 3392 if (iommu_identity_mapping) {
3393 ret = si_domain_init(hw_pass_through);
3394 if (ret)
3395 goto free_iommu;
3396 }
3397
e0fc7e0b 3398
a87f4918
JR		 3399 /*
3400 * If we copied translations from a previous kernel in the kdump
3401 * case, we can not assign the devices to domains now, as that
3402 * would eliminate the old mappings. So skip this part and defer
3403 * the assignment to device driver initialization time.
3404 */
3405 if (copied_tables)
3406 goto domains_done;
3407
ba395927 3408 /*
19943b0e
DW		 3409 * If pass through is not set or not enabled, setup context entries for
3410 * identity mappings for rmrr, gfx, and isa and may fall back to static
3411 * identity mapping if iommu_identity_mapping is set.
ba395927 3412 */
19943b0e
DW		 3413 if (iommu_identity_mapping) {
3414 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3415 if (ret) {
9f10e5bf 3416 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3417 goto free_iommu;
ba395927
KA		 3418 }
3419 }
ba395927 3420 /*
19943b0e
DW		 3421 * For each rmrr
3422 * for each dev attached to rmrr
3423 * do
3424 * locate drhd for dev, alloc domain for dev
3425 * allocate free domain
3426 * allocate page table entries for rmrr
3427 * if context not allocated for bus
3428 * allocate and init context
3429 * set present in root table for this bus
3430 * init context with domain, translation etc
3431 * endfor
3432 * endfor
ba395927 3433 */
9f10e5bf 3434 pr_info("Setting RMRR:\n");
19943b0e 3435 for_each_rmrr_units(rmrr) {
b683b230
JL		 3436 /* some BIOS lists non-exist devices in DMAR table. */
3437 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3438 i, dev) {
0b9d9753 3439 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3440 if (ret)
9f10e5bf 3441 pr_err("Mapping reserved region failed\n");
ba395927 3442 }
4ed0d3e6 3443 }
49a0429e 3444
19943b0e
DW		 3445 iommu_prepare_isa();
3446
a87f4918
JR		 3447domains_done:
3448
ba395927
KA		 3449 /*
3450 * for each drhd
3451 * enable fault log
3452 * global invalidate context cache
3453 * global invalidate iotlb
3454 * enable translation
3455 */
7c919779 3456 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3457 if (drhd->ignored) {
3458 /*
3459 * we always have to disable PMRs or DMA may fail on
3460 * this device
3461 */
3462 if (force_on)
7c919779 3463 iommu_disable_protect_mem_regions(iommu);
ba395927 3464 continue;
51a63e67 3465 }
ba395927
KA		 3466
3467 iommu_flush_write_buffer(iommu);
3468
a222a7f0
DW		 3469#ifdef CONFIG_INTEL_IOMMU_SVM
3470 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3471 ret = intel_svm_enable_prq(iommu);
3472 if (ret)
3473 goto free_iommu;
3474 }
3475#endif
3460a6d9
KA		 3476 ret = dmar_set_interrupt(iommu);
3477 if (ret)
989d51fc 3478 goto free_iommu;
3460a6d9 3479
8939ddf6
JR		 3480 if (!translation_pre_enabled(iommu))
3481 iommu_enable_translation(iommu);
3482
b94996c9 3483 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3484 }
3485
3486 return 0;
989d51fc
JL		 3487
3488free_iommu:
ffebeb46
JL		 3489 for_each_active_iommu(iommu, drhd) {
3490 disable_dmar_iommu(iommu);
a868e6b7 3491 free_dmar_iommu(iommu);
ffebeb46 3492 }
13cf0174 3493
d9630fe9 3494 kfree(g_iommus);
13cf0174 3495
989d51fc 3496error:
ba395927
KA		 3497 return ret;
3498}
3499
5a5e02a6 3500/* This takes a number of _MM_ pages, not VTD pages */
2aac6304 3501static unsigned long intel_alloc_iova(struct device *dev,
875764de
DW		 3502 struct dmar_domain *domain,
3503 unsigned long nrpages, uint64_t dma_mask)
ba395927 3504{
22e2f9fa 3505 unsigned long iova_pfn = 0;
ba395927 3506
875764de
DW		 3507 /* Restrict dma_mask to the width that the iommu can handle */
3508 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
8f6429c7
RM		 3509 /* Ensure we reserve the whole size-aligned region */
3510 nrpages = __roundup_pow_of_two(nrpages);
875764de
DW		 3511
3512 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3513 /*
3514 * First try to allocate an io virtual address in
284901a9 3515 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3516 * from higher range
ba395927 3517 */
22e2f9fa 3518 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
538d5b33 3519 IOVA_PFN(DMA_BIT_MASK(32)), false);
22e2f9fa
OP		 3520 if (iova_pfn)
3521 return iova_pfn;
875764de 3522 }
538d5b33
TN		 3523 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3524 IOVA_PFN(dma_mask), true);
22e2f9fa 3525 if (unlikely(!iova_pfn)) {
9f10e5bf 3526 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3527 nrpages, dev_name(dev));
2aac6304 3528 return 0;
f76aec76
KA		 3529 }
3530
22e2f9fa 3531 return iova_pfn;
f76aec76
KA		 3532}
3533
9ddbfb42 3534struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
f76aec76 3535{
1c5ebba9 3536 struct dmar_domain *domain, *tmp;
b1ce5b79 3537 struct dmar_rmrr_unit *rmrr;
b1ce5b79
JR		 3538 struct device *i_dev;
3539 int i, ret;
f76aec76 3540
1c5ebba9
JR		 3541 domain = find_domain(dev);
3542 if (domain)
3543 goto out;
3544
3545 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3546 if (!domain)
3547 goto out;
ba395927 3548
b1ce5b79
JR		 3549 /* We have a new domain - setup possible RMRRs for the device */
3550 rcu_read_lock();
3551 for_each_rmrr_units(rmrr) {
3552 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3553 i, i_dev) {
3554 if (i_dev != dev)
3555 continue;
3556
3557 ret = domain_prepare_identity_map(dev, domain,
3558 rmrr->base_address,
3559 rmrr->end_address);
3560 if (ret)
3561 dev_err(dev, "Mapping reserved region failed\n");
3562 }
3563 }
3564 rcu_read_unlock();
3565
1c5ebba9
JR		 3566 tmp = set_domain_for_dev(dev, domain);
3567 if (!tmp || domain != tmp) {
3568 domain_exit(domain);
3569 domain = tmp;
3570 }
3571
3572out:
3573
3574 if (!domain)
3575 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3576
3577
f76aec76
KA		 3578 return domain;
3579}
3580
ecb509ec 3581/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3582static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3583{
3584 int found;
3585
3d89194a 3586 if (iommu_dummy(dev))
1e4c64c4
DW		 3587 return 1;
3588
2c2e2c38 3589 if (!iommu_identity_mapping)
1e4c64c4 3590 return 0;
2c2e2c38 3591
9b226624 3592 found = identity_mapping(dev);
2c2e2c38 3593 if (found) {
ecb509ec 3594 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3595 return 1;
3596 else {
3597 /*
3598 * 32 bit DMA is removed from si_domain and fall back
3599 * to non-identity mapping.
3600 */
e6de0f8d 3601 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3602 pr_info("32bit %s uses non-identity mapping\n",
3603 dev_name(dev));
2c2e2c38
FY		 3604 return 0;
3605 }
3606 } else {
3607 /*
3608 * In case of a detached 64 bit DMA device from vm, the device
3609 * is put into si_domain for identity mapping.
3610 */
ecb509ec 3611 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3612 int ret;
28ccce0d 3613 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3614 if (!ret) {
9f10e5bf
JR		 3615 pr_info("64bit %s uses identity mapping\n",
3616 dev_name(dev));
2c2e2c38
FY		 3617 return 1;
3618 }
3619 }
3620 }
3621
1e4c64c4 3622 return 0;
2c2e2c38
FY		 3623}
3624
5040a918 3625static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3626 size_t size, int dir, u64 dma_mask)
f76aec76 3627{
f76aec76 3628 struct dmar_domain *domain;
5b6985ce 3629 phys_addr_t start_paddr;
2aac6304 3630 unsigned long iova_pfn;
f76aec76 3631 int prot = 0;
6865f0d1 3632 int ret;
8c11e798 3633 struct intel_iommu *iommu;
33041ec0 3634 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3635
3636 BUG_ON(dir == DMA_NONE);
2c2e2c38 3637
5040a918 3638 if (iommu_no_mapping(dev))
6865f0d1 3639 return paddr;
f76aec76 3640
5040a918 3641 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3642 if (!domain)
3643 return 0;
3644
8c11e798 3645 iommu = domain_get_iommu(domain);
88cb6a74 3646 size = aligned_nrpages(paddr, size);
f76aec76 3647
2aac6304
OP		 3648 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3649 if (!iova_pfn)
f76aec76
KA		 3650 goto error;
3651
ba395927
KA		 3652 /*
3653 * Check if DMAR supports zero-length reads on write only
3654 * mappings..
3655 */
3656 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3657 !cap_zlr(iommu->cap))
ba395927
KA		 3658 prot |= DMA_PTE_READ;
3659 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3660 prot |= DMA_PTE_WRITE;
3661 /*
6865f0d1 3662 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3663 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3664 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3665 * is not a big problem
3666 */
2aac6304 3667 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
33041ec0 3668 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3669 if (ret)
3670 goto error;
3671
2aac6304 3672 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
03d6a246
DW		 3673 start_paddr += paddr & ~PAGE_MASK;
3674 return start_paddr;
ba395927 3675
ba395927 3676error:
2aac6304 3677 if (iova_pfn)
22e2f9fa 3678 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
9f10e5bf 3679 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3680 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3681 return 0;
3682}
3683
ffbbef5c
FT		 3684static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3685 unsigned long offset, size_t size,
3686 enum dma_data_direction dir,
00085f1e 3687 unsigned long attrs)
bb9e6d65 3688{
ffbbef5c 3689 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3690 dir, *dev->dma_mask);
bb9e6d65
FT		 3691}
3692
769530e4 3693static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
ba395927 3694{
f76aec76 3695 struct dmar_domain *domain;
d794dc9b 3696 unsigned long start_pfn, last_pfn;
769530e4 3697 unsigned long nrpages;
2aac6304 3698 unsigned long iova_pfn;
8c11e798 3699 struct intel_iommu *iommu;
ea8ea460 3700 struct page *freelist;
ba395927 3701
73676832 3702 if (iommu_no_mapping(dev))
f76aec76 3703 return;
2c2e2c38 3704
1525a29a 3705 domain = find_domain(dev);
ba395927
KA		 3706 BUG_ON(!domain);
3707
8c11e798
WH		 3708 iommu = domain_get_iommu(domain);
3709
2aac6304 3710 iova_pfn = IOVA_PFN(dev_addr);
ba395927 3711
769530e4 3712 nrpages = aligned_nrpages(dev_addr, size);
2aac6304 3713 start_pfn = mm_to_dma_pfn(iova_pfn);
769530e4 3714 last_pfn = start_pfn + nrpages - 1;
ba395927 3715
d794dc9b 3716 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3717 dev_name(dev), start_pfn, last_pfn);
ba395927 3718
ea8ea460 3719 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3720
5e0d2a6f 3721 if (intel_iommu_strict) {
a1ddcbe9 3722 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
769530e4 3723 nrpages, !freelist, 0);
5e0d2a6f 3724 /* free iova */
22e2f9fa 3725 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
ea8ea460 3726 dma_free_pagelist(freelist);
5e0d2a6f 3727 } else {
13cf0174
JR		 3728 queue_iova(&domain->iovad, iova_pfn, nrpages,
3729 (unsigned long)freelist);
5e0d2a6f 3730 /*
3731 * queue up the release of the unmap to save the 1/6th of the
3732 * cpu used up by the iotlb flush operation...
3733 */
5e0d2a6f 3734 }
ba395927
KA		 3735}
3736
d41a4adb
JL		 3737static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3738 size_t size, enum dma_data_direction dir,
00085f1e 3739 unsigned long attrs)
d41a4adb 3740{
769530e4 3741 intel_unmap(dev, dev_addr, size);
d41a4adb
JL		 3742}
3743
5040a918 3744static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc 3745 dma_addr_t *dma_handle, gfp_t flags,
00085f1e 3746 unsigned long attrs)
ba395927 3747{
7ec916f8
CH		 3748 struct page *page = NULL;
3749 int order;
ba395927 3750
7ec916f8
CH		 3751 size = PAGE_ALIGN(size);
3752 order = get_order(size);
36746436 3753
7ec916f8
CH		 3754 if (!iommu_no_mapping(dev))
3755 flags &= ~(GFP_DMA | GFP_DMA32);
3756 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3757 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3758 flags |= GFP_DMA;
3759 else
3760 flags |= GFP_DMA32;
3761 }
3762
3763 if (gfpflags_allow_blocking(flags)) {
3764 unsigned int count = size >> PAGE_SHIFT;
3765
d834c5ab
MS		 3766 page = dma_alloc_from_contiguous(dev, count, order,
3767 flags & __GFP_NOWARN);
7ec916f8
CH		 3768 if (page && iommu_no_mapping(dev) &&
3769 page_to_phys(page) + size > dev->coherent_dma_mask) {
3770 dma_release_from_contiguous(dev, page, count);
3771 page = NULL;
3772 }
3773 }
3774
3775 if (!page)
3776 page = alloc_pages(flags, order);
3777 if (!page)
3778 return NULL;
3779 memset(page_address(page), 0, size);
3780
3781 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3782 DMA_BIDIRECTIONAL,
3783 dev->coherent_dma_mask);
3784 if (*dma_handle)
3785 return page_address(page);
3786 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3787 __free_pages(page, order);
36746436 3788
ba395927
KA		 3789 return NULL;
3790}
3791
5040a918 3792static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
00085f1e 3793 dma_addr_t dma_handle, unsigned long attrs)
ba395927 3794{
7ec916f8
CH		 3795 int order;
3796 struct page *page = virt_to_page(vaddr);
3797
3798 size = PAGE_ALIGN(size);
3799 order = get_order(size);
3800
3801 intel_unmap(dev, dma_handle, size);
3802 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3803 __free_pages(page, order);
ba395927
KA		 3804}
3805
5040a918 3806static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46 3807 int nelems, enum dma_data_direction dir,
00085f1e 3808 unsigned long attrs)
ba395927 3809{
769530e4
OP		 3810 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3811 unsigned long nrpages = 0;
3812 struct scatterlist *sg;
3813 int i;
3814
3815 for_each_sg(sglist, sg, nelems, i) {
3816 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3817 }
3818
3819 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
ba395927
KA		 3820}
3821
ba395927 3822static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3823 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3824{
3825 int i;
c03ab37c 3826 struct scatterlist *sg;
ba395927 3827
c03ab37c 3828 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3829 BUG_ON(!sg_page(sg));
29a90b70 3830 sg->dma_address = sg_phys(sg);
c03ab37c 3831 sg->dma_length = sg->length;
ba395927
KA		 3832 }
3833 return nelems;
3834}
3835
5040a918 3836static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
00085f1e 3837 enum dma_data_direction dir, unsigned long attrs)
ba395927 3838{
ba395927 3839 int i;
ba395927 3840 struct dmar_domain *domain;
f76aec76
KA		 3841 size_t size = 0;
3842 int prot = 0;
2aac6304 3843 unsigned long iova_pfn;
f76aec76 3844 int ret;
c03ab37c 3845 struct scatterlist *sg;
b536d24d 3846 unsigned long start_vpfn;
8c11e798 3847 struct intel_iommu *iommu;
ba395927
KA		 3848
3849 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3850 if (iommu_no_mapping(dev))
3851 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3852
5040a918 3853 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3854 if (!domain)
3855 return 0;
3856
8c11e798
WH		 3857 iommu = domain_get_iommu(domain);
3858
b536d24d 3859 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3860 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3861
2aac6304 3862 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
5040a918 3863 *dev->dma_mask);
2aac6304 3864 if (!iova_pfn) {
c03ab37c 3865 sglist->dma_length = 0;
f76aec76
KA		 3866 return 0;
3867 }
3868
3869 /*
3870 * Check if DMAR supports zero-length reads on write only
3871 * mappings..
3872 */
3873 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3874 !cap_zlr(iommu->cap))
f76aec76
KA		 3875 prot |= DMA_PTE_READ;
3876 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3877 prot |= DMA_PTE_WRITE;
3878
2aac6304 3879 start_vpfn = mm_to_dma_pfn(iova_pfn);
e1605495 3880
f532959b 3881 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3882 if (unlikely(ret)) {
e1605495 3883 dma_pte_free_pagetable(domain, start_vpfn,
bc24c571
DD		 3884 start_vpfn + size - 1,
3885 agaw_to_level(domain->agaw) + 1);
22e2f9fa 3886 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
e1605495 3887 return 0;
ba395927
KA		 3888 }
3889
ba395927
KA		 3890 return nelems;
3891}
3892
dfb805e8
FT		 3893static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3894{
3895 return !dma_addr;
3896}
3897
01e1932a 3898const struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3899 .alloc = intel_alloc_coherent,
3900 .free = intel_free_coherent,
ba395927
KA		 3901 .map_sg = intel_map_sg,
3902 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3903 .map_page = intel_map_page,
3904 .unmap_page = intel_unmap_page,
dfb805e8 3905 .mapping_error = intel_mapping_error,
5860acc1 3906#ifdef CONFIG_X86
fec777c3 3907 .dma_supported = dma_direct_supported,
5860acc1 3908#endif
ba395927
KA		 3909};
3910
3911static inline int iommu_domain_cache_init(void)
3912{
3913 int ret = 0;
3914
3915 iommu_domain_cache = kmem_cache_create("iommu_domain",
3916 sizeof(struct dmar_domain),
3917 0,
3918 SLAB_HWCACHE_ALIGN,
3919
3920 NULL);
3921 if (!iommu_domain_cache) {
9f10e5bf 3922 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3923 ret = -ENOMEM;
3924 }
3925
3926 return ret;
3927}
3928
3929static inline int iommu_devinfo_cache_init(void)
3930{
3931 int ret = 0;
3932
3933 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3934 sizeof(struct device_domain_info),
3935 0,
3936 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3937 NULL);
3938 if (!iommu_devinfo_cache) {
9f10e5bf 3939 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3940 ret = -ENOMEM;
3941 }
3942
3943 return ret;
3944}
3945
ba395927
KA		 3946static int __init iommu_init_mempool(void)
3947{
3948 int ret;
ae1ff3d6 3949 ret = iova_cache_get();
ba395927
KA		 3950 if (ret)
3951 return ret;
3952
3953 ret = iommu_domain_cache_init();
3954 if (ret)
3955 goto domain_error;
3956
3957 ret = iommu_devinfo_cache_init();
3958 if (!ret)
3959 return ret;
3960
3961 kmem_cache_destroy(iommu_domain_cache);
3962domain_error:
ae1ff3d6 3963 iova_cache_put();
ba395927
KA		 3964
3965 return -ENOMEM;
3966}
3967
3968static void __init iommu_exit_mempool(void)
3969{
3970 kmem_cache_destroy(iommu_devinfo_cache);
3971 kmem_cache_destroy(iommu_domain_cache);
ae1ff3d6 3972 iova_cache_put();
ba395927
KA		 3973}
3974
556ab45f
DW		 3975static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3976{
3977 struct dmar_drhd_unit *drhd;
3978 u32 vtbar;
3979 int rc;
3980
3981 /* We know that this device on this chipset has its own IOMMU.
3982 * If we find it under a different IOMMU, then the BIOS is lying
3983 * to us. Hope that the IOMMU for this device is actually
3984 * disabled, and it needs no translation...
3985 */
3986 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3987 if (rc) {
3988 /* "can't" happen */
3989 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3990 return;
3991 }
3992 vtbar &= 0xffff0000;
3993
3994 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3995 drhd = dmar_find_matched_drhd_unit(pdev);
3996 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3997 TAINT_FIRMWARE_WORKAROUND,
3998 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3999 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4000}
4001DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4002
ba395927
KA		 4003static void __init init_no_remapping_devices(void)
4004{
4005 struct dmar_drhd_unit *drhd;
832bd858 4006 struct device *dev;
b683b230 4007 int i;
ba395927
KA		 4008
4009 for_each_drhd_unit(drhd) {
4010 if (!drhd->include_all) {
b683b230
JL		 4011 for_each_active_dev_scope(drhd->devices,
4012 drhd->devices_cnt, i, dev)
4013 break;
832bd858 4014 /* ignore DMAR unit if no devices exist */
ba395927
KA		 4015 if (i == drhd->devices_cnt)
4016 drhd->ignored = 1;
4017 }
4018 }
4019
7c919779 4020 for_each_active_drhd_unit(drhd) {
7c919779 4021 if (drhd->include_all)
ba395927
KA		 4022 continue;
4023
b683b230
JL		 4024 for_each_active_dev_scope(drhd->devices,
4025 drhd->devices_cnt, i, dev)
832bd858 4026 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 4027 break;
ba395927
KA		 4028 if (i < drhd->devices_cnt)
4029 continue;
4030
c0771df8
DW		 4031 /* This IOMMU has *only* gfx devices. Either bypass it or
4032 set the gfx_mapped flag, as appropriate */
4033 if (dmar_map_gfx) {
4034 intel_iommu_gfx_mapped = 1;
4035 } else {
4036 drhd->ignored = 1;
b683b230
JL		 4037 for_each_active_dev_scope(drhd->devices,
4038 drhd->devices_cnt, i, dev)
832bd858 4039 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 4040 }
4041 }
4042}
4043
f59c7b69
FY		 4044#ifdef CONFIG_SUSPEND
4045static int init_iommu_hw(void)
4046{
4047 struct dmar_drhd_unit *drhd;
4048 struct intel_iommu *iommu = NULL;
4049
4050 for_each_active_iommu(iommu, drhd)
4051 if (iommu->qi)
4052 dmar_reenable_qi(iommu);
4053
b779260b
JC		 4054 for_each_iommu(iommu, drhd) {
4055 if (drhd->ignored) {
4056 /*
4057 * we always have to disable PMRs or DMA may fail on
4058 * this device
4059 */
4060 if (force_on)
4061 iommu_disable_protect_mem_regions(iommu);
4062 continue;
4063 }
4064
f59c7b69
FY		 4065 iommu_flush_write_buffer(iommu);
4066
4067 iommu_set_root_entry(iommu);
4068
4069 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 4070 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 4071 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4072 iommu_enable_translation(iommu);
b94996c9 4073 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 4074 }
4075
4076 return 0;
4077}
4078
4079static void iommu_flush_all(void)
4080{
4081 struct dmar_drhd_unit *drhd;
4082 struct intel_iommu *iommu;
4083
4084 for_each_active_iommu(iommu, drhd) {
4085 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 4086 DMA_CCMD_GLOBAL_INVL);
f59c7b69 4087 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 4088 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 4089 }
4090}
4091
134fac3f 4092static int iommu_suspend(void)
f59c7b69
FY		 4093{
4094 struct dmar_drhd_unit *drhd;
4095 struct intel_iommu *iommu = NULL;
4096 unsigned long flag;
4097
4098 for_each_active_iommu(iommu, drhd) {
6396bb22 4099 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
f59c7b69
FY		 4100 GFP_ATOMIC);
4101 if (!iommu->iommu_state)
4102 goto nomem;
4103 }
4104
4105 iommu_flush_all();
4106
4107 for_each_active_iommu(iommu, drhd) {
4108 iommu_disable_translation(iommu);
4109
1f5b3c3f 4110 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 4111
4112 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4113 readl(iommu->reg + DMAR_FECTL_REG);
4114 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4115 readl(iommu->reg + DMAR_FEDATA_REG);
4116 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4117 readl(iommu->reg + DMAR_FEADDR_REG);
4118 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4119 readl(iommu->reg + DMAR_FEUADDR_REG);
4120
1f5b3c3f 4121 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 4122 }
4123 return 0;
4124
4125nomem:
4126 for_each_active_iommu(iommu, drhd)
4127 kfree(iommu->iommu_state);
4128
4129 return -ENOMEM;
4130}
4131
134fac3f 4132static void iommu_resume(void)
f59c7b69
FY		 4133{
4134 struct dmar_drhd_unit *drhd;
4135 struct intel_iommu *iommu = NULL;
4136 unsigned long flag;
4137
4138 if (init_iommu_hw()) {
b779260b
JC		 4139 if (force_on)
4140 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4141 else
4142 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 4143 return;
f59c7b69
FY		 4144 }
4145
4146 for_each_active_iommu(iommu, drhd) {
4147
1f5b3c3f 4148 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 4149
4150 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4151 iommu->reg + DMAR_FECTL_REG);
4152 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4153 iommu->reg + DMAR_FEDATA_REG);
4154 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4155 iommu->reg + DMAR_FEADDR_REG);
4156 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4157 iommu->reg + DMAR_FEUADDR_REG);
4158
1f5b3c3f 4159 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 4160 }
4161
4162 for_each_active_iommu(iommu, drhd)
4163 kfree(iommu->iommu_state);
f59c7b69
FY		 4164}
4165
134fac3f 4166static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 4167 .resume = iommu_resume,
4168 .suspend = iommu_suspend,
4169};
4170
134fac3f 4171static void __init init_iommu_pm_ops(void)
f59c7b69 4172{
134fac3f 4173 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 4174}
4175
4176#else
99592ba4 4177static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 4178#endif /* CONFIG_PM */
4179
318fe7df 4180
c2a0b538 4181int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 4182{
4183 struct acpi_dmar_reserved_memory *rmrr;
0659b8dc 4184 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
318fe7df 4185 struct dmar_rmrr_unit *rmrru;
0659b8dc 4186 size_t length;
318fe7df
SS		 4187
4188 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4189 if (!rmrru)
0659b8dc 4190 goto out;
318fe7df
SS		 4191
4192 rmrru->hdr = header;
4193 rmrr = (struct acpi_dmar_reserved_memory *)header;
4194 rmrru->base_address = rmrr->base_address;
4195 rmrru->end_address = rmrr->end_address;
0659b8dc
EA		 4196
4197 length = rmrr->end_address - rmrr->base_address + 1;
4198 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4199 IOMMU_RESV_DIRECT);
4200 if (!rmrru->resv)
4201 goto free_rmrru;
4202
2e455289
JL		 4203 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4204 ((void *)rmrr) + rmrr->header.length,
4205 &rmrru->devices_cnt);
0659b8dc
EA		 4206 if (rmrru->devices_cnt && rmrru->devices == NULL)
4207 goto free_all;
318fe7df 4208
2e455289 4209 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4210
2e455289 4211 return 0;
0659b8dc
EA		 4212free_all:
4213 kfree(rmrru->resv);
4214free_rmrru:
4215 kfree(rmrru);
4216out:
4217 return -ENOMEM;
318fe7df
SS		 4218}
4219
6b197249
JL		 4220static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4221{
4222 struct dmar_atsr_unit *atsru;
4223 struct acpi_dmar_atsr *tmp;
4224
4225 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4226 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4227 if (atsr->segment != tmp->segment)
4228 continue;
4229 if (atsr->header.length != tmp->header.length)
4230 continue;
4231 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4232 return atsru;
4233 }
4234
4235 return NULL;
4236}
4237
4238int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4239{
4240 struct acpi_dmar_atsr *atsr;
4241 struct dmar_atsr_unit *atsru;
4242
b608fe35 4243 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
6b197249
JL		 4244 return 0;
4245
318fe7df 4246 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4247 atsru = dmar_find_atsr(atsr);
4248 if (atsru)
4249 return 0;
4250
4251 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4252 if (!atsru)
4253 return -ENOMEM;
4254
6b197249
JL		 4255 /*
4256 * If memory is allocated from slab by ACPI _DSM method, we need to
4257 * copy the memory content because the memory buffer will be freed
4258 * on return.
4259 */
4260 atsru->hdr = (void *)(atsru + 1);
4261 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4262 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4263 if (!atsru->include_all) {
4264 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4265 (void *)atsr + atsr->header.length,
4266 &atsru->devices_cnt);
4267 if (atsru->devices_cnt && atsru->devices == NULL) {
4268 kfree(atsru);
4269 return -ENOMEM;
4270 }
4271 }
318fe7df 4272
0e242612 4273 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4274
4275 return 0;
4276}
4277
9bdc531e
JL		 4278static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4279{
4280 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4281 kfree(atsru);
4282}
4283
6b197249
JL		 4284int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4285{
4286 struct acpi_dmar_atsr *atsr;
4287 struct dmar_atsr_unit *atsru;
4288
4289 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4290 atsru = dmar_find_atsr(atsr);
4291 if (atsru) {
4292 list_del_rcu(&atsru->list);
4293 synchronize_rcu();
4294 intel_iommu_free_atsr(atsru);
4295 }
4296
4297 return 0;
4298}
4299
4300int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4301{
4302 int i;
4303 struct device *dev;
4304 struct acpi_dmar_atsr *atsr;
4305 struct dmar_atsr_unit *atsru;
4306
4307 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4308 atsru = dmar_find_atsr(atsr);
4309 if (!atsru)
4310 return 0;
4311
194dc870 4312 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
6b197249
JL		 4313 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4314 i, dev)
4315 return -EBUSY;
194dc870 4316 }
6b197249
JL		 4317
4318 return 0;
4319}
4320
ffebeb46
JL		 4321static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4322{
4323 int sp, ret = 0;
4324 struct intel_iommu *iommu = dmaru->iommu;
4325
4326 if (g_iommus[iommu->seq_id])
4327 return 0;
4328
4329 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4330 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4331 iommu->name);
4332 return -ENXIO;
4333 }
4334 if (!ecap_sc_support(iommu->ecap) &&
4335 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4336 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4337 iommu->name);
4338 return -ENXIO;
4339 }
4340 sp = domain_update_iommu_superpage(iommu) - 1;
4341 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4342 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4343 iommu->name);
4344 return -ENXIO;
4345 }
4346
4347 /*
4348 * Disable translation if already enabled prior to OS handover.
4349 */
4350 if (iommu->gcmd & DMA_GCMD_TE)
4351 iommu_disable_translation(iommu);
4352
4353 g_iommus[iommu->seq_id] = iommu;
4354 ret = iommu_init_domains(iommu);
4355 if (ret == 0)
4356 ret = iommu_alloc_root_entry(iommu);
4357 if (ret)
4358 goto out;
4359
8a94ade4
DW		 4360#ifdef CONFIG_INTEL_IOMMU_SVM
4361 if (pasid_enabled(iommu))
d9737953 4362 intel_svm_init(iommu);
8a94ade4
DW		 4363#endif
4364
ffebeb46
JL		 4365 if (dmaru->ignored) {
4366 /*
4367 * we always have to disable PMRs or DMA may fail on this device
4368 */
4369 if (force_on)
4370 iommu_disable_protect_mem_regions(iommu);
4371 return 0;
4372 }
4373
4374 intel_iommu_init_qi(iommu);
4375 iommu_flush_write_buffer(iommu);
a222a7f0
DW		 4376
4377#ifdef CONFIG_INTEL_IOMMU_SVM
4378 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4379 ret = intel_svm_enable_prq(iommu);
4380 if (ret)
4381 goto disable_iommu;
4382 }
4383#endif
ffebeb46
JL		 4384 ret = dmar_set_interrupt(iommu);
4385 if (ret)
4386 goto disable_iommu;
4387
4388 iommu_set_root_entry(iommu);
4389 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4390 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4391 iommu_enable_translation(iommu);
4392
ffebeb46
JL		 4393 iommu_disable_protect_mem_regions(iommu);
4394 return 0;
4395
4396disable_iommu:
4397 disable_dmar_iommu(iommu);
4398out:
4399 free_dmar_iommu(iommu);
4400 return ret;
4401}
4402
6b197249
JL		 4403int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4404{
ffebeb46
JL		 4405 int ret = 0;
4406 struct intel_iommu *iommu = dmaru->iommu;
4407
4408 if (!intel_iommu_enabled)
4409 return 0;
4410 if (iommu == NULL)
4411 return -EINVAL;
4412
4413 if (insert) {
4414 ret = intel_iommu_add(dmaru);
4415 } else {
4416 disable_dmar_iommu(iommu);
4417 free_dmar_iommu(iommu);
4418 }
4419
4420 return ret;
6b197249
JL		 4421}
4422
9bdc531e
JL		 4423static void intel_iommu_free_dmars(void)
4424{
4425 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4426 struct dmar_atsr_unit *atsru, *atsr_n;
4427
4428 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4429 list_del(&rmrru->list);
4430 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
0659b8dc 4431 kfree(rmrru->resv);
9bdc531e 4432 kfree(rmrru);
318fe7df
SS		 4433 }
4434
9bdc531e
JL		 4435 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4436 list_del(&atsru->list);
4437 intel_iommu_free_atsr(atsru);
4438 }
318fe7df
SS		 4439}
4440
4441int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4442{
b683b230 4443 int i, ret = 1;
318fe7df 4444 struct pci_bus *bus;
832bd858
DW		 4445 struct pci_dev *bridge = NULL;
4446 struct device *tmp;
318fe7df
SS		 4447 struct acpi_dmar_atsr *atsr;
4448 struct dmar_atsr_unit *atsru;
4449
4450 dev = pci_physfn(dev);
318fe7df 4451 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4452 bridge = bus->self;
d14053b3
DW		 4453 /* If it's an integrated device, allow ATS */
4454 if (!bridge)
4455 return 1;
4456 /* Connected via non-PCIe: no ATS */
4457 if (!pci_is_pcie(bridge) ||
62f87c0e 4458 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4459 return 0;
d14053b3 4460 /* If we found the root port, look it up in the ATSR */
b5f82ddf 4461 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4462 break;
318fe7df
SS		 4463 }
4464
0e242612 4465 rcu_read_lock();
b5f82ddf
JL		 4466 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4467 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4468 if (atsr->segment != pci_domain_nr(dev->bus))
4469 continue;
4470
b683b230 4471 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4472 if (tmp == &bridge->dev)
b683b230 4473 goto out;
b5f82ddf
JL		 4474
4475 if (atsru->include_all)
b683b230 4476 goto out;
b5f82ddf 4477 }
b683b230
JL		 4478 ret = 0;
4479out:
0e242612 4480 rcu_read_unlock();
318fe7df 4481
b683b230 4482 return ret;
318fe7df
SS		 4483}
4484
59ce0515
JL		 4485int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4486{
4487 int ret = 0;
4488 struct dmar_rmrr_unit *rmrru;
4489 struct dmar_atsr_unit *atsru;
4490 struct acpi_dmar_atsr *atsr;
4491 struct acpi_dmar_reserved_memory *rmrr;
4492
b608fe35 4493 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
59ce0515
JL		 4494 return 0;
4495
4496 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4497 rmrr = container_of(rmrru->hdr,
4498 struct acpi_dmar_reserved_memory, header);
4499 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4500 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4501 ((void *)rmrr) + rmrr->header.length,
4502 rmrr->segment, rmrru->devices,
4503 rmrru->devices_cnt);
27e24950 4504 if(ret < 0)
59ce0515 4505 return ret;
e6a8c9b3 4506 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
27e24950
JL		 4507 dmar_remove_dev_scope(info, rmrr->segment,
4508 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4509 }
4510 }
4511
4512 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4513 if (atsru->include_all)
4514 continue;
4515
4516 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4517 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4518 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4519 (void *)atsr + atsr->header.length,
4520 atsr->segment, atsru->devices,
4521 atsru->devices_cnt);
4522 if (ret > 0)
4523 break;
4524 else if(ret < 0)
4525 return ret;
e6a8c9b3 4526 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
59ce0515
JL		 4527 if (dmar_remove_dev_scope(info, atsr->segment,
4528 atsru->devices, atsru->devices_cnt))
4529 break;
4530 }
4531 }
4532
4533 return 0;
4534}
4535
99dcaded
FY		 4536/*
4537 * Here we only respond to action of unbound device from driver.
4538 *
4539 * Added device is not attached to its DMAR domain here yet. That will happen
4540 * when mapping the device to iova.
4541 */
4542static int device_notifier(struct notifier_block *nb,
4543 unsigned long action, void *data)
4544{
4545 struct device *dev = data;
99dcaded
FY		 4546 struct dmar_domain *domain;
4547
3d89194a 4548 if (iommu_dummy(dev))
44cd613c
DW		 4549 return 0;
4550
1196c2fb 4551 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4552 return 0;
4553
1525a29a 4554 domain = find_domain(dev);
99dcaded
FY		 4555 if (!domain)
4556 return 0;
4557
e6de0f8d 4558 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4559 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4560 domain_exit(domain);
a97590e5 4561
99dcaded
FY		 4562 return 0;
4563}
4564
4565static struct notifier_block device_nb = {
4566 .notifier_call = device_notifier,
4567};
4568
75f05569
JL		 4569static int intel_iommu_memory_notifier(struct notifier_block *nb,
4570 unsigned long val, void *v)
4571{
4572 struct memory_notify *mhp = v;
4573 unsigned long long start, end;
4574 unsigned long start_vpfn, last_vpfn;
4575
4576 switch (val) {
4577 case MEM_GOING_ONLINE:
4578 start = mhp->start_pfn << PAGE_SHIFT;
4579 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4580 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4581 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4582 start, end);
4583 return NOTIFY_BAD;
4584 }
4585 break;
4586
4587 case MEM_OFFLINE:
4588 case MEM_CANCEL_ONLINE:
4589 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4590 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4591 while (start_vpfn <= last_vpfn) {
4592 struct iova *iova;
4593 struct dmar_drhd_unit *drhd;
4594 struct intel_iommu *iommu;
ea8ea460 4595 struct page *freelist;
75f05569
JL		 4596
4597 iova = find_iova(&si_domain->iovad, start_vpfn);
4598 if (iova == NULL) {
9f10e5bf 4599 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4600 start_vpfn);
4601 break;
4602 }
4603
4604 iova = split_and_remove_iova(&si_domain->iovad, iova,
4605 start_vpfn, last_vpfn);
4606 if (iova == NULL) {
9f10e5bf 4607 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4608 start_vpfn, last_vpfn);
4609 return NOTIFY_BAD;
4610 }
4611
ea8ea460
DW		 4612 freelist = domain_unmap(si_domain, iova->pfn_lo,
4613 iova->pfn_hi);
4614
75f05569
JL		 4615 rcu_read_lock();
4616 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4617 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4618 iova->pfn_lo, iova_size(iova),
ea8ea460 4619 !freelist, 0);
75f05569 4620 rcu_read_unlock();
ea8ea460 4621 dma_free_pagelist(freelist);
75f05569
JL		 4622
4623 start_vpfn = iova->pfn_hi + 1;
4624 free_iova_mem(iova);
4625 }
4626 break;
4627 }
4628
4629 return NOTIFY_OK;
4630}
4631
4632static struct notifier_block intel_iommu_memory_nb = {
4633 .notifier_call = intel_iommu_memory_notifier,
4634 .priority = 0
4635};
4636
22e2f9fa
OP		 4637static void free_all_cpu_cached_iovas(unsigned int cpu)
4638{
4639 int i;
4640
4641 for (i = 0; i < g_num_of_iommus; i++) {
4642 struct intel_iommu *iommu = g_iommus[i];
4643 struct dmar_domain *domain;
0caa7616 4644 int did;
22e2f9fa
OP		 4645
4646 if (!iommu)
4647 continue;
4648
3bd4f911 4649 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
0caa7616 4650 domain = get_iommu_domain(iommu, (u16)did);
22e2f9fa
OP		 4651
4652 if (!domain)
4653 continue;
4654 free_cpu_cached_iovas(cpu, &domain->iovad);
4655 }
4656 }
4657}
4658
21647615 4659static int intel_iommu_cpu_dead(unsigned int cpu)
aa473240 4660{
21647615 4661 free_all_cpu_cached_iovas(cpu);
21647615 4662 return 0;
aa473240
OP		 4663}
4664
161b28aa
JR		 4665static void intel_disable_iommus(void)
4666{
4667 struct intel_iommu *iommu = NULL;
4668 struct dmar_drhd_unit *drhd;
4669
4670 for_each_iommu(iommu, drhd)
4671 iommu_disable_translation(iommu);
4672}
4673
a7fdb6e6
JR		 4674static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4675{
2926a2aa
JR		 4676 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4677
4678 return container_of(iommu_dev, struct intel_iommu, iommu);
a7fdb6e6
JR		 4679}
4680
a5459cfe
AW		 4681static ssize_t intel_iommu_show_version(struct device *dev,
4682 struct device_attribute *attr,
4683 char *buf)
4684{
a7fdb6e6 4685 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
a5459cfe
AW		 4686 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4687 return sprintf(buf, "%d:%d\n",
4688 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4689}
4690static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4691
4692static ssize_t intel_iommu_show_address(struct device *dev,
4693 struct device_attribute *attr,
4694 char *buf)
4695{
a7fdb6e6 4696 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
a5459cfe
AW		 4697 return sprintf(buf, "%llx\n", iommu->reg_phys);
4698}
4699static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4700
4701static ssize_t intel_iommu_show_cap(struct device *dev,
4702 struct device_attribute *attr,
4703 char *buf)
4704{
a7fdb6e6 4705 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
a5459cfe
AW		 4706 return sprintf(buf, "%llx\n", iommu->cap);
4707}
4708static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4709
4710static ssize_t intel_iommu_show_ecap(struct device *dev,
4711 struct device_attribute *attr,
4712 char *buf)
4713{
a7fdb6e6 4714 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
a5459cfe
AW		 4715 return sprintf(buf, "%llx\n", iommu->ecap);
4716}
4717static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4718
2238c082
AW		 4719static ssize_t intel_iommu_show_ndoms(struct device *dev,
4720 struct device_attribute *attr,
4721 char *buf)
4722{
a7fdb6e6 4723 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2238c082
AW		 4724 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4725}
4726static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4727
4728static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4729 struct device_attribute *attr,
4730 char *buf)
4731{
a7fdb6e6 4732 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2238c082
AW		 4733 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4734 cap_ndoms(iommu->cap)));
4735}
4736static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4737
a5459cfe
AW		 4738static struct attribute *intel_iommu_attrs[] = {
4739 &dev_attr_version.attr,
4740 &dev_attr_address.attr,
4741 &dev_attr_cap.attr,
4742 &dev_attr_ecap.attr,
2238c082
AW		 4743 &dev_attr_domains_supported.attr,
4744 &dev_attr_domains_used.attr,
a5459cfe
AW		 4745 NULL,
4746};
4747
4748static struct attribute_group intel_iommu_group = {
4749 .name = "intel-iommu",
4750 .attrs = intel_iommu_attrs,
4751};
4752
4753const struct attribute_group *intel_iommu_groups[] = {
4754 &intel_iommu_group,
4755 NULL,
4756};
4757
ba395927
KA		 4758int __init intel_iommu_init(void)
4759{
9bdc531e 4760 int ret = -ENODEV;
3a93c841 4761 struct dmar_drhd_unit *drhd;
7c919779 4762 struct intel_iommu *iommu;
ba395927 4763
a59b50e9
JC		 4764 /* VT-d is required for a TXT/tboot launch, so enforce that */
4765 force_on = tboot_force_iommu();
4766
3a5670e8
JL		 4767 if (iommu_init_mempool()) {
4768 if (force_on)
4769 panic("tboot: Failed to initialize iommu memory\n");
4770 return -ENOMEM;
4771 }
4772
4773 down_write(&dmar_global_lock);
a59b50e9
JC		 4774 if (dmar_table_init()) {
4775 if (force_on)
4776 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4777 goto out_free_dmar;
a59b50e9 4778 }
ba395927 4779
c2c7286a 4780 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4781 if (force_on)
4782 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4783 goto out_free_dmar;
a59b50e9 4784 }
1886e8a9 4785
ec154bf5
JR		 4786 up_write(&dmar_global_lock);
4787
4788 /*
4789 * The bus notifier takes the dmar_global_lock, so lockdep will
4790 * complain later when we register it under the lock.
4791 */
4792 dmar_register_bus_notifier();
4793
4794 down_write(&dmar_global_lock);
4795
161b28aa 4796 if (no_iommu || dmar_disabled) {
bfd20f1c
SL		 4797 /*
4798 * We exit the function here to ensure IOMMU's remapping and
4799 * mempool aren't setup, which means that the IOMMU's PMRs
4800 * won't be disabled via the call to init_dmars(). So disable
4801 * it explicitly here. The PMRs were setup by tboot prior to
4802 * calling SENTER, but the kernel is expected to reset/tear
4803 * down the PMRs.
4804 */
4805 if (intel_iommu_tboot_noforce) {
4806 for_each_iommu(iommu, drhd)
4807 iommu_disable_protect_mem_regions(iommu);
4808 }
4809
161b28aa
JR		 4810 /*
4811 * Make sure the IOMMUs are switched off, even when we
4812 * boot into a kexec kernel and the previous kernel left
4813 * them enabled
4814 */
4815 intel_disable_iommus();
9bdc531e 4816 goto out_free_dmar;
161b28aa 4817 }
2ae21010 4818
318fe7df 4819 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4820 pr_info("No RMRR found\n");
318fe7df
SS		 4821
4822 if (list_empty(&dmar_atsr_units))
9f10e5bf 4823 pr_info("No ATSR found\n");
318fe7df 4824
51a63e67
JC		 4825 if (dmar_init_reserved_ranges()) {
4826 if (force_on)
4827 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4828 goto out_free_reserved_range;
51a63e67 4829 }
ba395927
KA		 4830
4831 init_no_remapping_devices();
4832
b779260b 4833 ret = init_dmars();
ba395927 4834 if (ret) {
a59b50e9
JC		 4835 if (force_on)
4836 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4837 pr_err("Initialization failed\n");
9bdc531e 4838 goto out_free_reserved_range;
ba395927 4839 }
3a5670e8 4840 up_write(&dmar_global_lock);
9f10e5bf 4841 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4842
4fac8076 4843#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
75f1cdf1
FT		 4844 swiotlb = 0;
4845#endif
19943b0e 4846 dma_ops = &intel_dma_ops;
4ed0d3e6 4847
134fac3f 4848 init_iommu_pm_ops();
a8bcbb0d 4849
39ab9555
JR		 4850 for_each_active_iommu(iommu, drhd) {
4851 iommu_device_sysfs_add(&iommu->iommu, NULL,
4852 intel_iommu_groups,
4853 "%s", iommu->name);
4854 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4855 iommu_device_register(&iommu->iommu);
4856 }
a5459cfe 4857
4236d97d 4858 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4859 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4860 if (si_domain && !hw_pass_through)
4861 register_memory_notifier(&intel_iommu_memory_nb);
21647615
AMG		 4862 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4863 intel_iommu_cpu_dead);
8bc1f85c
ED		 4864 intel_iommu_enabled = 1;
4865
ba395927 4866 return 0;
9bdc531e
JL		 4867
4868out_free_reserved_range:
4869 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4870out_free_dmar:
4871 intel_iommu_free_dmars();
3a5670e8
JL		 4872 up_write(&dmar_global_lock);
4873 iommu_exit_mempool();
9bdc531e 4874 return ret;
ba395927 4875}
e820482c 4876
2452d9db 4877static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
579305f7
AW		 4878{
4879 struct intel_iommu *iommu = opaque;
4880
2452d9db 4881 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 4882 return 0;
4883}
4884
4885/*
4886 * NB - intel-iommu lacks any sort of reference counting for the users of
4887 * dependent devices. If multiple endpoints have intersecting dependent
4888 * devices, unbinding the driver from any one of them will possibly leave
4889 * the others unable to operate.
4890 */
2452d9db 4891static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
3199aa6b 4892{
0bcb3e28 4893 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4894 return;
4895
2452d9db 4896 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
3199aa6b
HW		 4897}
4898
127c7615 4899static void __dmar_remove_one_dev_info(struct device_domain_info *info)
c7151a8d 4900{
c7151a8d
WH		 4901 struct intel_iommu *iommu;
4902 unsigned long flags;
c7151a8d 4903
55d94043
JR		 4904 assert_spin_locked(&device_domain_lock);
4905
127c7615 4906 if (WARN_ON(!info))
c7151a8d
WH		 4907 return;
4908
127c7615 4909 iommu = info->iommu;
c7151a8d 4910
127c7615
JR		 4911 if (info->dev) {
4912 iommu_disable_dev_iotlb(info);
4913 domain_context_clear(iommu, info->dev);
a7fc93fe 4914 intel_pasid_free_table(info->dev);
127c7615 4915 }
c7151a8d 4916
b608ac3b 4917 unlink_domain_info(info);
c7151a8d 4918
d160aca5 4919 spin_lock_irqsave(&iommu->lock, flags);
127c7615 4920 domain_detach_iommu(info->domain, iommu);
d160aca5 4921 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 4922
127c7615 4923 free_devinfo_mem(info);
c7151a8d 4924}
c7151a8d 4925
55d94043
JR		 4926static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4927 struct device *dev)
4928{
127c7615 4929 struct device_domain_info *info;
55d94043 4930 unsigned long flags;
3e7abe25 4931
55d94043 4932 spin_lock_irqsave(&device_domain_lock, flags);
127c7615
JR		 4933 info = dev->archdata.iommu;
4934 __dmar_remove_one_dev_info(info);
55d94043 4935 spin_unlock_irqrestore(&device_domain_lock, flags);
c7151a8d
WH		 4936}
4937
2c2e2c38 4938static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4939{
4940 int adjust_width;
4941
aa3ac946 4942 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5e98c4b1
WH		 4943 domain_reserve_special_ranges(domain);
4944
4945 /* calculate AGAW */
4946 domain->gaw = guest_width;
4947 adjust_width = guestwidth_to_adjustwidth(guest_width);
4948 domain->agaw = width_to_agaw(adjust_width);
4949
5e98c4b1 4950 domain->iommu_coherency = 0;
c5b15255 4951 domain->iommu_snooping = 0;
6dd9a7c7 4952 domain->iommu_superpage = 0;
fe40f1e0 4953 domain->max_addr = 0;
5e98c4b1
WH		 4954
4955 /* always allocate the top pgd */
4c923d47 4956 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4957 if (!domain->pgd)
4958 return -ENOMEM;
4959 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4960 return 0;
4961}
4962
00a77deb 4963static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4964{
5d450806 4965 struct dmar_domain *dmar_domain;
00a77deb
JR		 4966 struct iommu_domain *domain;
4967
4968 if (type != IOMMU_DOMAIN_UNMANAGED)
4969 return NULL;
38717946 4970
ab8dfe25 4971 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4972 if (!dmar_domain) {
9f10e5bf 4973 pr_err("Can't allocate dmar_domain\n");
00a77deb 4974 return NULL;
38717946 4975 }
2c2e2c38 4976 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4977 pr_err("Domain initialization failed\n");
92d03cc8 4978 domain_exit(dmar_domain);
00a77deb 4979 return NULL;
38717946 4980 }
8140a95d 4981 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4982
00a77deb 4983 domain = &dmar_domain->domain;
8a0e715b
JR		 4984 domain->geometry.aperture_start = 0;
4985 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4986 domain->geometry.force_aperture = true;
4987
00a77deb 4988 return domain;
38717946 4989}
38717946 4990
00a77deb 4991static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4992{
00a77deb 4993 domain_exit(to_dmar_domain(domain));
38717946 4994}
38717946 4995
4c5478c9
JR		 4996static int intel_iommu_attach_device(struct iommu_domain *domain,
4997 struct device *dev)
38717946 4998{
00a77deb 4999 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 5000 struct intel_iommu *iommu;
5001 int addr_width;
156baca8 5002 u8 bus, devfn;
faa3d6f5 5003
c875d2c1
AW		 5004 if (device_is_rmrr_locked(dev)) {
5005 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5006 return -EPERM;
5007 }
5008
7207d8f9
DW		 5009 /* normally dev is not mapped */
5010 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 5011 struct dmar_domain *old_domain;
5012
1525a29a 5013 old_domain = find_domain(dev);
faa3d6f5 5014 if (old_domain) {
d160aca5 5015 rcu_read_lock();
de7e8886 5016 dmar_remove_one_dev_info(old_domain, dev);
d160aca5 5017 rcu_read_unlock();
62c22167
JR		 5018
5019 if (!domain_type_is_vm_or_si(old_domain) &&
5020 list_empty(&old_domain->devices))
5021 domain_exit(old_domain);
faa3d6f5
WH		 5022 }
5023 }
5024
156baca8 5025 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 5026 if (!iommu)
5027 return -ENODEV;
5028
5029 /* check if this iommu agaw is sufficient for max mapped address */
5030 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 5031 if (addr_width > cap_mgaw(iommu->cap))
5032 addr_width = cap_mgaw(iommu->cap);
5033
5034 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 5035 pr_err("%s: iommu width (%d) is not "
fe40f1e0 5036 "sufficient for the mapped address (%llx)\n",
a99c47a2 5037 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 5038 return -EFAULT;
5039 }
a99c47a2
TL		 5040 dmar_domain->gaw = addr_width;
5041
5042 /*
5043 * Knock out extra levels of page tables if necessary
5044 */
5045 while (iommu->agaw < dmar_domain->agaw) {
5046 struct dma_pte *pte;
5047
5048 pte = dmar_domain->pgd;
5049 if (dma_pte_present(pte)) {
25cbff16
SY		 5050 dmar_domain->pgd = (struct dma_pte *)
5051 phys_to_virt(dma_pte_addr(pte));
7a661013 5052 free_pgtable_page(pte);
a99c47a2
TL		 5053 }
5054 dmar_domain->agaw--;
5055 }
fe40f1e0 5056
28ccce0d 5057 return domain_add_dev_info(dmar_domain, dev);
38717946 5058}
38717946 5059
4c5478c9
JR		 5060static void intel_iommu_detach_device(struct iommu_domain *domain,
5061 struct device *dev)
38717946 5062{
e6de0f8d 5063 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 5064}
c7151a8d 5065
b146a1c9
JR		 5066static int intel_iommu_map(struct iommu_domain *domain,
5067 unsigned long iova, phys_addr_t hpa,
5009065d 5068 size_t size, int iommu_prot)
faa3d6f5 5069{
00a77deb 5070 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 5071 u64 max_addr;
dde57a21 5072 int prot = 0;
faa3d6f5 5073 int ret;
fe40f1e0 5074
dde57a21
JR		 5075 if (iommu_prot & IOMMU_READ)
5076 prot |= DMA_PTE_READ;
5077 if (iommu_prot & IOMMU_WRITE)
5078 prot |= DMA_PTE_WRITE;
9cf06697
SY		 5079 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5080 prot |= DMA_PTE_SNP;
dde57a21 5081
163cc52c 5082 max_addr = iova + size;
dde57a21 5083 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 5084 u64 end;
5085
5086 /* check if minimum agaw is sufficient for mapped address */
8954da1f 5087 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 5088 if (end < max_addr) {
9f10e5bf 5089 pr_err("%s: iommu width (%d) is not "
fe40f1e0 5090 "sufficient for the mapped address (%llx)\n",
8954da1f 5091 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 5092 return -EFAULT;
5093 }
dde57a21 5094 dmar_domain->max_addr = max_addr;
fe40f1e0 5095 }
ad051221
DW		 5096 /* Round up size to next multiple of PAGE_SIZE, if it and
5097 the low bits of hpa would take us onto the next page */
88cb6a74 5098 size = aligned_nrpages(hpa, size);
ad051221
DW		 5099 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5100 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 5101 return ret;
38717946 5102}
38717946 5103
5009065d 5104static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 5105 unsigned long iova, size_t size)
38717946 5106{
00a77deb 5107 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460 5108 struct page *freelist = NULL;
ea8ea460
DW		 5109 unsigned long start_pfn, last_pfn;
5110 unsigned int npages;
42e8c186 5111 int iommu_id, level = 0;
5cf0a76f
DW		 5112
5113 /* Cope with horrid API which requires us to unmap more than the
5114 size argument if it happens to be a large-page mapping. */
dc02e46e 5115 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5cf0a76f
DW		 5116
5117 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5118 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 5119
ea8ea460
DW		 5120 start_pfn = iova >> VTD_PAGE_SHIFT;
5121 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5122
5123 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5124
5125 npages = last_pfn - start_pfn + 1;
5126
f746a025 5127 for_each_domain_iommu(iommu_id, dmar_domain)
42e8c186
JR		 5128 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5129 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 5130
5131 dma_free_pagelist(freelist);
fe40f1e0 5132
163cc52c
DW		 5133 if (dmar_domain->max_addr == iova + size)
5134 dmar_domain->max_addr = iova;
b146a1c9 5135
5cf0a76f 5136 return size;
38717946 5137}
38717946 5138
d14d6577 5139static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 5140 dma_addr_t iova)
38717946 5141{
00a77deb 5142 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 5143 struct dma_pte *pte;
5cf0a76f 5144 int level = 0;
faa3d6f5 5145 u64 phys = 0;
38717946 5146
5cf0a76f 5147 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 5148 if (pte)
faa3d6f5 5149 phys = dma_pte_addr(pte);
38717946 5150
faa3d6f5 5151 return phys;
38717946 5152}
a8bcbb0d 5153
5d587b8d 5154static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 5155{
dbb9fd86 5156 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 5157 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 5158 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 5159 return irq_remapping_enabled == 1;
dbb9fd86 5160
5d587b8d 5161 return false;
dbb9fd86
SY		 5162}
5163
abdfdde2
AW		 5164static int intel_iommu_add_device(struct device *dev)
5165{
a5459cfe 5166 struct intel_iommu *iommu;
abdfdde2 5167 struct iommu_group *group;
156baca8 5168 u8 bus, devfn;
70ae6f0d 5169
a5459cfe
AW		 5170 iommu = device_to_iommu(dev, &bus, &devfn);
5171 if (!iommu)
70ae6f0d
AW		 5172 return -ENODEV;
5173
e3d10af1 5174 iommu_device_link(&iommu->iommu, dev);
a4ff1fc2 5175
e17f9ff4 5176 group = iommu_group_get_for_dev(dev);
783f157b 5177
e17f9ff4
AW		 5178 if (IS_ERR(group))
5179 return PTR_ERR(group);
bcb71abe 5180
abdfdde2 5181 iommu_group_put(group);
e17f9ff4 5182 return 0;
abdfdde2 5183}
70ae6f0d 5184
abdfdde2
AW		 5185static void intel_iommu_remove_device(struct device *dev)
5186{
a5459cfe
AW		 5187 struct intel_iommu *iommu;
5188 u8 bus, devfn;
5189
5190 iommu = device_to_iommu(dev, &bus, &devfn);
5191 if (!iommu)
5192 return;
5193
abdfdde2 5194 iommu_group_remove_device(dev);
a5459cfe 5195
e3d10af1 5196 iommu_device_unlink(&iommu->iommu, dev);
70ae6f0d
AW		 5197}
5198
0659b8dc
EA		 5199static void intel_iommu_get_resv_regions(struct device *device,
5200 struct list_head *head)
5201{
5202 struct iommu_resv_region *reg;
5203 struct dmar_rmrr_unit *rmrr;
5204 struct device *i_dev;
5205 int i;
5206
5207 rcu_read_lock();
5208 for_each_rmrr_units(rmrr) {
5209 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5210 i, i_dev) {
5211 if (i_dev != device)
5212 continue;
5213
5214 list_add_tail(&rmrr->resv->list, head);
5215 }
5216 }
5217 rcu_read_unlock();
5218
5219 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5220 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
9d3a4de4 5221 0, IOMMU_RESV_MSI);
0659b8dc
EA		 5222 if (!reg)
5223 return;
5224 list_add_tail(&reg->list, head);
5225}
5226
5227static void intel_iommu_put_resv_regions(struct device *dev,
5228 struct list_head *head)
5229{
5230 struct iommu_resv_region *entry, *next;
5231
5232 list_for_each_entry_safe(entry, next, head, list) {
5233 if (entry->type == IOMMU_RESV_RESERVED)
5234 kfree(entry);
5235 }
70ae6f0d
AW		 5236}
5237
2f26e0a9 5238#ifdef CONFIG_INTEL_IOMMU_SVM
65ca7f5f 5239#define MAX_NR_PASID_BITS (20)
4774cc52 5240static inline unsigned long intel_iommu_get_pts(struct device *dev)
65ca7f5f 5241{
4774cc52
LB		 5242 int pts, max_pasid;
5243
5244 max_pasid = intel_pasid_get_dev_max_id(dev);
5245 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5246 if (pts < 5)
65ca7f5f
JP		 5247 return 0;
5248
4774cc52 5249 return pts - 5;
65ca7f5f
JP		 5250}
5251
2f26e0a9
DW		 5252int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5253{
5254 struct device_domain_info *info;
5255 struct context_entry *context;
5256 struct dmar_domain *domain;
5257 unsigned long flags;
5258 u64 ctx_lo;
5259 int ret;
5260
5261 domain = get_valid_domain_for_dev(sdev->dev);
5262 if (!domain)
5263 return -EINVAL;
5264
5265 spin_lock_irqsave(&device_domain_lock, flags);
5266 spin_lock(&iommu->lock);
5267
5268 ret = -EINVAL;
5269 info = sdev->dev->archdata.iommu;
5270 if (!info || !info->pasid_supported)
5271 goto out;
5272
5273 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5274 if (WARN_ON(!context))
5275 goto out;
5276
5277 ctx_lo = context[0].lo;
5278
5279 sdev->did = domain->iommu_did[iommu->seq_id];
5280 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5281
5282 if (!(ctx_lo & CONTEXT_PASIDE)) {
11b93ebf
AR		 5283 if (iommu->pasid_state_table)
5284 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
4774cc52
LB		 5285 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5286 intel_iommu_get_pts(sdev->dev);
65ca7f5f 5287
2f26e0a9
DW		 5288 wmb();
5289 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5290 * extended to permit requests-with-PASID if the PASIDE bit
5291 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5292 * however, the PASIDE bit is ignored and requests-with-PASID
5293 * are unconditionally blocked. Which makes less sense.
5294 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5295 * "guest mode" translation types depending on whether ATS
5296 * is available or not. Annoyingly, we can't use the new
5297 * modes *unless* PASIDE is set. */
5298 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5299 ctx_lo &= ~CONTEXT_TT_MASK;
5300 if (info->ats_supported)
5301 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5302 else
5303 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5304 }
5305 ctx_lo |= CONTEXT_PASIDE;
907fea34
DW		 5306 if (iommu->pasid_state_table)
5307 ctx_lo |= CONTEXT_DINVE;
a222a7f0
DW		 5308 if (info->pri_supported)
5309 ctx_lo |= CONTEXT_PRS;
2f26e0a9
DW		 5310 context[0].lo = ctx_lo;
5311 wmb();
5312 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5313 DMA_CCMD_MASK_NOBIT,
5314 DMA_CCMD_DEVICE_INVL);
5315 }
5316
5317 /* Enable PASID support in the device, if it wasn't already */
5318 if (!info->pasid_enabled)
5319 iommu_enable_dev_iotlb(info);
5320
5321 if (info->ats_enabled) {
5322 sdev->dev_iotlb = 1;
5323 sdev->qdep = info->ats_qdep;
5324 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5325 sdev->qdep = 0;
5326 }
5327 ret = 0;
5328
5329 out:
5330 spin_unlock(&iommu->lock);
5331 spin_unlock_irqrestore(&device_domain_lock, flags);
5332
5333 return ret;
5334}
5335
5336struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5337{
5338 struct intel_iommu *iommu;
5339 u8 bus, devfn;
5340
5341 if (iommu_dummy(dev)) {
5342 dev_warn(dev,
5343 "No IOMMU translation for device; cannot enable SVM\n");
5344 return NULL;
5345 }
5346
5347 iommu = device_to_iommu(dev, &bus, &devfn);
5348 if ((!iommu)) {
b9997e38 5349 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
2f26e0a9
DW		 5350 return NULL;
5351 }
5352
2f26e0a9
DW		 5353 return iommu;
5354}
5355#endif /* CONFIG_INTEL_IOMMU_SVM */
5356
b0119e87 5357const struct iommu_ops intel_iommu_ops = {
0659b8dc
EA		 5358 .capable = intel_iommu_capable,
5359 .domain_alloc = intel_iommu_domain_alloc,
5360 .domain_free = intel_iommu_domain_free,
5361 .attach_dev = intel_iommu_attach_device,
5362 .detach_dev = intel_iommu_detach_device,
5363 .map = intel_iommu_map,
5364 .unmap = intel_iommu_unmap,
0659b8dc
EA		 5365 .iova_to_phys = intel_iommu_iova_to_phys,
5366 .add_device = intel_iommu_add_device,
5367 .remove_device = intel_iommu_remove_device,
5368 .get_resv_regions = intel_iommu_get_resv_regions,
5369 .put_resv_regions = intel_iommu_put_resv_regions,
5370 .device_group = pci_device_group,
5371 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 5372};
9af88143 5373
9452618e
DV		 5374static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5375{
5376 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 5377 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 5378 dmar_map_gfx = 0;
5379}
5380
5381DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5382DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5383DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5384DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5385DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5386DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5387DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5388
d34d6517 5389static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 5390{
5391 /*
5392 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 5393 * but needs it. Same seems to hold for the desktop versions.
9af88143 5394 */
9f10e5bf 5395 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 5396 rwbf_quirk = 1;
5397}
5398
5399DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 5400DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5401DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5402DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5403DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5404DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5405DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 5406
eecfd57f
AJ		 5407#define GGC 0x52
5408#define GGC_MEMORY_SIZE_MASK (0xf << 8)
5409#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5410#define GGC_MEMORY_SIZE_1M (0x1 << 8)
5411#define GGC_MEMORY_SIZE_2M (0x3 << 8)
5412#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5413#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5414#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5415#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5416
d34d6517 5417static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 5418{
5419 unsigned short ggc;
5420
eecfd57f 5421 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 5422 return;
5423
eecfd57f 5424 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 5425 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 5426 dmar_map_gfx = 0;
6fbcfb3e
DW		 5427 } else if (dmar_map_gfx) {
5428 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 5429 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 5430 intel_iommu_strict = 1;
5431 }
9eecabcb
DW		 5432}
5433DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5434DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5435DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5436DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5437
e0fc7e0b
DW		 5438/* On Tylersburg chipsets, some BIOSes have been known to enable the
5439 ISOCH DMAR unit for the Azalia sound device, but not give it any
5440 TLB entries, which causes it to deadlock. Check for that. We do
5441 this in a function called from init_dmars(), instead of in a PCI
5442 quirk, because we don't want to print the obnoxious "BIOS broken"
5443 message if VT-d is actually disabled.
5444*/
5445static void __init check_tylersburg_isoch(void)
5446{
5447 struct pci_dev *pdev;
5448 uint32_t vtisochctrl;
5449
5450 /* If there's no Azalia in the system anyway, forget it. */
5451 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5452 if (!pdev)
5453 return;
5454 pci_dev_put(pdev);
5455
5456 /* System Management Registers. Might be hidden, in which case
5457 we can't do the sanity check. But that's OK, because the
5458 known-broken BIOSes _don't_ actually hide it, so far. */
5459 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5460 if (!pdev)
5461 return;
5462
5463 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5464 pci_dev_put(pdev);
5465 return;
5466 }
5467
5468 pci_dev_put(pdev);
5469
5470 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5471 if (vtisochctrl & 1)
5472 return;
5473
5474 /* Drop all bits other than the number of TLB entries */
5475 vtisochctrl &= 0x1c;
5476
5477 /* If we have the recommended number of TLB entries (16), fine. */
5478 if (vtisochctrl == 0x10)
5479 return;
5480
5481 /* Zero TLB entries? You get to ride the short bus to school. */
5482 if (!vtisochctrl) {
5483 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5484 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5485 dmi_get_system_info(DMI_BIOS_VENDOR),
5486 dmi_get_system_info(DMI_BIOS_VERSION),
5487 dmi_get_system_info(DMI_PRODUCT_VERSION));
5488 iommu_identity_mapping |= IDENTMAP_AZALIA;
5489 return;
5490 }
9f10e5bf
JR		 5491
5492 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5493 vtisochctrl);
5494}
Linux 6.x block layer and io_uring tree(s)