projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'iommu-updates-v4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/joro...
[linux-2.6-block.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
aa473240 36#include <linux/cpu.h>
5e0d2a6f 37#include <linux/timer.h>
dfddb969 38#include <linux/io.h>
38717946 39#include <linux/iova.h>
5d450806 40#include <linux/iommu.h>
38717946 41#include <linux/intel-iommu.h>
134fac3f 42#include <linux/syscore_ops.h>
69575d38 43#include <linux/tboot.h>
adb2fe02 44#include <linux/dmi.h>
5cdede24 45#include <linux/pci-ats.h>
0ee332c1 46#include <linux/memblock.h>
36746436 47#include <linux/dma-contiguous.h>
091d42e4 48#include <linux/crash_dump.h>
8a8f422d 49#include <asm/irq_remapping.h>
ba395927 50#include <asm/cacheflush.h>
46a7fa27 51#include <asm/iommu.h>
ba395927 52
078e1ee2
JR		 53#include "irq_remapping.h"
54
5b6985ce
FY		 55#define ROOT_SIZE VTD_PAGE_SIZE
56#define CONTEXT_SIZE VTD_PAGE_SIZE
57
ba395927 58#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 59#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 60#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 61#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 62
63#define IOAPIC_RANGE_START (0xfee00000)
64#define IOAPIC_RANGE_END (0xfeefffff)
65#define IOVA_START_ADDR (0x1000)
66
67#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
4ed0d3e6 69#define MAX_AGAW_WIDTH 64
5c645b35 70#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 71
2ebe3151
DW		 72#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 80
1b722500
RM		 81/* IO virtual address start page frame number */
82#define IOVA_START_PFN (1)
83
f27be03b 84#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 85#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 86#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 87
df08cdc7
AM		 88/* page table handling */
89#define LEVEL_STRIDE (9)
90#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
6d1c56a9
OBC		 92/*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
df08cdc7
AM		 110static inline int agaw_to_level(int agaw)
111{
112 return agaw + 2;
113}
114
115static inline int agaw_to_width(int agaw)
116{
5c645b35 117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 118}
119
120static inline int width_to_agaw(int width)
121{
5c645b35 122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 123}
124
125static inline unsigned int level_to_offset_bits(int level)
126{
127 return (level - 1) * LEVEL_STRIDE;
128}
129
130static inline int pfn_level_offset(unsigned long pfn, int level)
131{
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133}
134
135static inline unsigned long level_mask(int level)
136{
137 return -1UL << level_to_offset_bits(level);
138}
139
140static inline unsigned long level_size(int level)
141{
142 return 1UL << level_to_offset_bits(level);
143}
144
145static inline unsigned long align_to_level(unsigned long pfn, int level)
146{
147 return (pfn + level_size(level) - 1) & level_mask(level);
148}
fd18de50 149
6dd9a7c7
YS		 150static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151{
5c645b35 152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 153}
154
dd4e8319
DW		 155/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158{
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160}
161
162static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163{
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165}
166static inline unsigned long page_to_dma_pfn(struct page *pg)
167{
168 return mm_to_dma_pfn(page_to_pfn(pg));
169}
170static inline unsigned long virt_to_dma_pfn(void *p)
171{
172 return page_to_dma_pfn(virt_to_page(p));
173}
174
d9630fe9
WH		 175/* global iommu list, set NULL for ignored DMAR units */
176static struct intel_iommu **g_iommus;
177
e0fc7e0b 178static void __init check_tylersburg_isoch(void);
9af88143
DW		 179static int rwbf_quirk;
180
b779260b
JC		 181/*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185static int force_on = 0;
186
46b08e1a
MM		 187/*
188 * 0: Present
189 * 1-11: Reserved
190 * 12-63: Context Ptr (12 - (haw-1))
191 * 64-127: Reserved
192 */
193struct root_entry {
03ecc32c
DW		 194 u64 lo;
195 u64 hi;
46b08e1a
MM		 196};
197#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 198
091d42e4
JR		 199/*
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
201 * if marked present.
202 */
203static phys_addr_t root_entry_lctp(struct root_entry *re)
204{
205 if (!(re->lo & 1))
206 return 0;
207
208 return re->lo & VTD_PAGE_MASK;
209}
210
211/*
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
213 * if marked present.
214 */
215static phys_addr_t root_entry_uctp(struct root_entry *re)
216{
217 if (!(re->hi & 1))
218 return 0;
46b08e1a 219
091d42e4
JR		 220 return re->hi & VTD_PAGE_MASK;
221}
7a8fc25e
MM		 222/*
223 * low 64 bits:
224 * 0: present
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
228 * high 64 bits:
229 * 0-2: address width
230 * 3-6: aval
231 * 8-23: domain id
232 */
233struct context_entry {
234 u64 lo;
235 u64 hi;
236};
c07e7d21 237
cf484d0e
JR		 238static inline void context_clear_pasid_enable(struct context_entry *context)
239{
240 context->lo &= ~(1ULL << 11);
241}
242
243static inline bool context_pasid_enabled(struct context_entry *context)
244{
245 return !!(context->lo & (1ULL << 11));
246}
247
248static inline void context_set_copied(struct context_entry *context)
249{
250 context->hi |= (1ull << 3);
251}
252
253static inline bool context_copied(struct context_entry *context)
254{
255 return !!(context->hi & (1ULL << 3));
256}
257
258static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 259{
260 return (context->lo & 1);
261}
cf484d0e
JR		 262
263static inline bool context_present(struct context_entry *context)
264{
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
268}
269
c07e7d21
MM		 270static inline void context_set_present(struct context_entry *context)
271{
272 context->lo |= 1;
273}
274
275static inline void context_set_fault_enable(struct context_entry *context)
276{
277 context->lo &= (((u64)-1) << 2) | 1;
278}
279
c07e7d21
MM		 280static inline void context_set_translation_type(struct context_entry *context,
281 unsigned long value)
282{
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
285}
286
287static inline void context_set_address_root(struct context_entry *context,
288 unsigned long value)
289{
1a2262f9 290 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 291 context->lo |= value & VTD_PAGE_MASK;
292}
293
294static inline void context_set_address_width(struct context_entry *context,
295 unsigned long value)
296{
297 context->hi |= value & 7;
298}
299
300static inline void context_set_domain_id(struct context_entry *context,
301 unsigned long value)
302{
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
304}
305
dbcd861f
JR		 306static inline int context_domain_id(struct context_entry *c)
307{
308 return((c->hi >> 8) & 0xffff);
309}
310
c07e7d21
MM		 311static inline void context_clear_entry(struct context_entry *context)
312{
313 context->lo = 0;
314 context->hi = 0;
315}
7a8fc25e 316
622ba12a
MM		 317/*
318 * 0: readable
319 * 1: writable
320 * 2-6: reserved
321 * 7: super page
9cf06697
SY		 322 * 8-10: available
323 * 11: snoop behavior
622ba12a
MM		 324 * 12-63: Host physcial address
325 */
326struct dma_pte {
327 u64 val;
328};
622ba12a 329
19c239ce
MM		 330static inline void dma_clear_pte(struct dma_pte *pte)
331{
332 pte->val = 0;
333}
334
19c239ce
MM		 335static inline u64 dma_pte_addr(struct dma_pte *pte)
336{
c85994e4
DW		 337#ifdef CONFIG_64BIT
338 return pte->val & VTD_PAGE_MASK;
339#else
340 /* Must have a full atomic 64-bit read */
1a8bd481 341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 342#endif
19c239ce
MM		 343}
344
19c239ce
MM		 345static inline bool dma_pte_present(struct dma_pte *pte)
346{
347 return (pte->val & 3) != 0;
348}
622ba12a 349
4399c8bf
AK		 350static inline bool dma_pte_superpage(struct dma_pte *pte)
351{
c3c75eb7 352 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 353}
354
75e6bf96
DW		 355static inline int first_pte_in_page(struct dma_pte *pte)
356{
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
358}
359
2c2e2c38
FY		 360/*
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
365 */
19943b0e
DW		 366static struct dmar_domain *si_domain;
367static int hw_pass_through = 1;
2c2e2c38 368
28ccce0d
JR		 369/*
370 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 371 * across iommus may be owned in one domain, e.g. kvm guest.
372 */
ab8dfe25 373#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 374
2c2e2c38 375/* si_domain contains mulitple devices */
ab8dfe25 376#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 377
29a27719
JR		 378#define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
381
99126f7c 382struct dmar_domain {
4c923d47 383 int nid; /* node id */
29a27719
JR		 384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
99126f7c 388
c0e8a6c8
JR		 389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
99126f7c 393
0824c592 394 bool has_iotlb_device;
00a77deb 395 struct list_head devices; /* all devices' list */
99126f7c
MM		 396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
3b5410e7 404 int flags; /* flags to find out type of domain */
8e604097
WH		 405
406 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 407 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 408 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
fe40f1e0 412 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
99126f7c
MM		 416};
417
a647dacb
MM		 418/* PCI domain-device relationship */
419struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
276dbf99 422 u8 bus; /* PCI bus number */
a647dacb 423 u8 devfn; /* PCI devfn number */
b16d0cb9
DW		 424 u8 pasid_supported:3;
425 u8 pasid_enabled:1;
426 u8 pri_supported:1;
427 u8 pri_enabled:1;
428 u8 ats_supported:1;
429 u8 ats_enabled:1;
430 u8 ats_qdep;
0bcb3e28 431 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 432 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 433 struct dmar_domain *domain; /* pointer to domain */
434};
435
b94e4117
JL		 436struct dmar_rmrr_unit {
437 struct list_head list; /* list of rmrr units */
438 struct acpi_dmar_header *hdr; /* ACPI header */
439 u64 base_address; /* reserved base address*/
440 u64 end_address; /* reserved end address */
832bd858 441 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 442 int devices_cnt; /* target device count */
443};
444
445struct dmar_atsr_unit {
446 struct list_head list; /* list of ATSR units */
447 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 448 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 449 int devices_cnt; /* target device count */
450 u8 include_all:1; /* include all ports */
451};
452
453static LIST_HEAD(dmar_atsr_units);
454static LIST_HEAD(dmar_rmrr_units);
455
456#define for_each_rmrr_units(rmrr) \
457 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
5e0d2a6f 459static void flush_unmaps_timeout(unsigned long data);
460
314f1dc1 461struct deferred_flush_entry {
2aac6304 462 unsigned long iova_pfn;
769530e4 463 unsigned long nrpages;
314f1dc1
OP		 464 struct dmar_domain *domain;
465 struct page *freelist;
466};
5e0d2a6f 467
80b20dd8 468#define HIGH_WATER_MARK 250
314f1dc1 469struct deferred_flush_table {
80b20dd8 470 int next;
314f1dc1 471 struct deferred_flush_entry entries[HIGH_WATER_MARK];
80b20dd8 472};
473
aa473240
OP		 474struct deferred_flush_data {
475 spinlock_t lock;
476 int timer_on;
477 struct timer_list timer;
478 long size;
479 struct deferred_flush_table *tables;
80b20dd8 480};
481
aa473240 482DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
80b20dd8 483
5e0d2a6f 484/* bitmap for indexing intel_iommus */
5e0d2a6f 485static int g_num_of_iommus;
486
92d03cc8 487static void domain_exit(struct dmar_domain *domain);
ba395927 488static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 489static void dmar_remove_one_dev_info(struct dmar_domain *domain,
490 struct device *dev);
127c7615 491static void __dmar_remove_one_dev_info(struct device_domain_info *info);
2452d9db
JR		 492static void domain_context_clear(struct intel_iommu *iommu,
493 struct device *dev);
2a46ddf7
JL		 494static int domain_detach_iommu(struct dmar_domain *domain,
495 struct intel_iommu *iommu);
ba395927 496
d3f13810 497#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 498int dmar_disabled = 0;
499#else
500int dmar_disabled = 1;
d3f13810 501#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 502
8bc1f85c
ED		 503int intel_iommu_enabled = 0;
504EXPORT_SYMBOL_GPL(intel_iommu_enabled);
505
2d9e667e 506static int dmar_map_gfx = 1;
7d3b03ce 507static int dmar_forcedac;
5e0d2a6f 508static int intel_iommu_strict;
6dd9a7c7 509static int intel_iommu_superpage = 1;
c83b2f20 510static int intel_iommu_ecs = 1;
ae853ddb
DW		 511static int intel_iommu_pasid28;
512static int iommu_identity_mapping;
c83b2f20 513
ae853ddb
DW		 514#define IDENTMAP_ALL 1
515#define IDENTMAP_GFX 2
516#define IDENTMAP_AZALIA 4
c83b2f20 517
d42fde70
DW		 518/* Broadwell and Skylake have broken ECS support — normal so-called "second
519 * level" translation of DMA requests-without-PASID doesn't actually happen
520 * unless you also set the NESTE bit in an extended context-entry. Which of
521 * course means that SVM doesn't work because it's trying to do nested
522 * translation of the physical addresses it finds in the process page tables,
523 * through the IOVA->phys mapping found in the "second level" page tables.
524 *
525 * The VT-d specification was retroactively changed to change the definition
526 * of the capability bits and pretend that Broadwell/Skylake never happened...
527 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
528 * for some reason it was the PASID capability bit which was redefined (from
529 * bit 28 on BDW/SKL to bit 40 in future).
530 *
531 * So our test for ECS needs to eschew those implementations which set the old
532 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
533 * Unless we are working around the 'pasid28' limitations, that is, by putting
534 * the device into passthrough mode for normal DMA and thus masking the bug.
535 */
c83b2f20 536#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
d42fde70
DW		 537 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
538/* PASID support is thus enabled if ECS is enabled and *either* of the old
539 * or new capability bits are set. */
540#define pasid_enabled(iommu) (ecs_enabled(iommu) && \
541 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
ba395927 542
c0771df8
DW		 543int intel_iommu_gfx_mapped;
544EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
545
ba395927
KA		 546#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
547static DEFINE_SPINLOCK(device_domain_lock);
548static LIST_HEAD(device_domain_list);
549
b22f6434 550static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 551
4158c2ec
JR		 552static bool translation_pre_enabled(struct intel_iommu *iommu)
553{
554 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
555}
556
091d42e4
JR		 557static void clear_translation_pre_enabled(struct intel_iommu *iommu)
558{
559 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
560}
561
4158c2ec
JR		 562static void init_translation_status(struct intel_iommu *iommu)
563{
564 u32 gsts;
565
566 gsts = readl(iommu->reg + DMAR_GSTS_REG);
567 if (gsts & DMA_GSTS_TES)
568 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
569}
570
00a77deb
JR		 571/* Convert generic 'struct iommu_domain to private struct dmar_domain */
572static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
573{
574 return container_of(dom, struct dmar_domain, domain);
575}
576
ba395927
KA		 577static int __init intel_iommu_setup(char *str)
578{
579 if (!str)
580 return -EINVAL;
581 while (*str) {
0cd5c3c8
KM		 582 if (!strncmp(str, "on", 2)) {
583 dmar_disabled = 0;
9f10e5bf 584 pr_info("IOMMU enabled\n");
0cd5c3c8 585 } else if (!strncmp(str, "off", 3)) {
ba395927 586 dmar_disabled = 1;
9f10e5bf 587 pr_info("IOMMU disabled\n");
ba395927
KA		 588 } else if (!strncmp(str, "igfx_off", 8)) {
589 dmar_map_gfx = 0;
9f10e5bf 590 pr_info("Disable GFX device mapping\n");
7d3b03ce 591 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 592 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 593 dmar_forcedac = 1;
5e0d2a6f 594 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 595 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 596 intel_iommu_strict = 1;
6dd9a7c7 597 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 598 pr_info("Disable supported super page\n");
6dd9a7c7 599 intel_iommu_superpage = 0;
c83b2f20
DW		 600 } else if (!strncmp(str, "ecs_off", 7)) {
601 printk(KERN_INFO
602 "Intel-IOMMU: disable extended context table support\n");
603 intel_iommu_ecs = 0;
ae853ddb
DW		 604 } else if (!strncmp(str, "pasid28", 7)) {
605 printk(KERN_INFO
606 "Intel-IOMMU: enable pre-production PASID support\n");
607 intel_iommu_pasid28 = 1;
608 iommu_identity_mapping |= IDENTMAP_GFX;
ba395927
KA		 609 }
610
611 str += strcspn(str, ",");
612 while (*str == ',')
613 str++;
614 }
615 return 0;
616}
617__setup("intel_iommu=", intel_iommu_setup);
618
619static struct kmem_cache *iommu_domain_cache;
620static struct kmem_cache *iommu_devinfo_cache;
ba395927 621
9452d5bf
JR		 622static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
623{
8bf47816
JR		 624 struct dmar_domain **domains;
625 int idx = did >> 8;
626
627 domains = iommu->domains[idx];
628 if (!domains)
629 return NULL;
630
631 return domains[did & 0xff];
9452d5bf
JR		 632}
633
634static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
635 struct dmar_domain *domain)
636{
8bf47816
JR		 637 struct dmar_domain **domains;
638 int idx = did >> 8;
639
640 if (!iommu->domains[idx]) {
641 size_t size = 256 * sizeof(struct dmar_domain *);
642 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
643 }
644
645 domains = iommu->domains[idx];
646 if (WARN_ON(!domains))
647 return;
648 else
649 domains[did & 0xff] = domain;
9452d5bf
JR		 650}
651
4c923d47 652static inline void *alloc_pgtable_page(int node)
eb3fa7cb 653{
4c923d47
SS		 654 struct page *page;
655 void *vaddr = NULL;
eb3fa7cb 656
4c923d47
SS		 657 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
658 if (page)
659 vaddr = page_address(page);
eb3fa7cb 660 return vaddr;
ba395927
KA		 661}
662
663static inline void free_pgtable_page(void *vaddr)
664{
665 free_page((unsigned long)vaddr);
666}
667
668static inline void *alloc_domain_mem(void)
669{
354bb65e 670 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 671}
672
38717946 673static void free_domain_mem(void *vaddr)
ba395927
KA		 674{
675 kmem_cache_free(iommu_domain_cache, vaddr);
676}
677
678static inline void * alloc_devinfo_mem(void)
679{
354bb65e 680 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 681}
682
683static inline void free_devinfo_mem(void *vaddr)
684{
685 kmem_cache_free(iommu_devinfo_cache, vaddr);
686}
687
ab8dfe25
JL		 688static inline int domain_type_is_vm(struct dmar_domain *domain)
689{
690 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
691}
692
28ccce0d
JR		 693static inline int domain_type_is_si(struct dmar_domain *domain)
694{
695 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
696}
697
ab8dfe25
JL		 698static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
699{
700 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
701 DOMAIN_FLAG_STATIC_IDENTITY);
702}
1b573683 703
162d1b10
JL		 704static inline int domain_pfn_supported(struct dmar_domain *domain,
705 unsigned long pfn)
706{
707 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
708
709 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
710}
711
4ed0d3e6 712static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 713{
714 unsigned long sagaw;
715 int agaw = -1;
716
717 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 718 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 719 agaw >= 0; agaw--) {
720 if (test_bit(agaw, &sagaw))
721 break;
722 }
723
724 return agaw;
725}
726
4ed0d3e6
FY		 727/*
728 * Calculate max SAGAW for each iommu.
729 */
730int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
731{
732 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
733}
734
735/*
736 * calculate agaw for each iommu.
737 * "SAGAW" may be different across iommus, use a default agaw, and
738 * get a supported less agaw for iommus that don't support the default agaw.
739 */
740int iommu_calculate_agaw(struct intel_iommu *iommu)
741{
742 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
743}
744
2c2e2c38 745/* This functionin only returns single iommu in a domain */
8c11e798
WH		 746static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
747{
748 int iommu_id;
749
2c2e2c38 750 /* si_domain and vm domain should not get here. */
ab8dfe25 751 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 752 for_each_domain_iommu(iommu_id, domain)
753 break;
754
8c11e798
WH		 755 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
756 return NULL;
757
758 return g_iommus[iommu_id];
759}
760
8e604097
WH		 761static void domain_update_iommu_coherency(struct dmar_domain *domain)
762{
d0501960
DW		 763 struct dmar_drhd_unit *drhd;
764 struct intel_iommu *iommu;
2f119c78
QL		 765 bool found = false;
766 int i;
2e12bc29 767
d0501960 768 domain->iommu_coherency = 1;
8e604097 769
29a27719 770 for_each_domain_iommu(i, domain) {
2f119c78 771 found = true;
8e604097
WH		 772 if (!ecap_coherent(g_iommus[i]->ecap)) {
773 domain->iommu_coherency = 0;
774 break;
775 }
8e604097 776 }
d0501960
DW		 777 if (found)
778 return;
779
780 /* No hardware attached; use lowest common denominator */
781 rcu_read_lock();
782 for_each_active_iommu(iommu, drhd) {
783 if (!ecap_coherent(iommu->ecap)) {
784 domain->iommu_coherency = 0;
785 break;
786 }
787 }
788 rcu_read_unlock();
8e604097
WH		 789}
790
161f6934 791static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 792{
161f6934
JL		 793 struct dmar_drhd_unit *drhd;
794 struct intel_iommu *iommu;
795 int ret = 1;
58c610bd 796
161f6934
JL		 797 rcu_read_lock();
798 for_each_active_iommu(iommu, drhd) {
799 if (iommu != skip) {
800 if (!ecap_sc_support(iommu->ecap)) {
801 ret = 0;
802 break;
803 }
58c610bd 804 }
58c610bd 805 }
161f6934
JL		 806 rcu_read_unlock();
807
808 return ret;
58c610bd
SY		 809}
810
161f6934 811static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 812{
8140a95d 813 struct dmar_drhd_unit *drhd;
161f6934 814 struct intel_iommu *iommu;
8140a95d 815 int mask = 0xf;
6dd9a7c7
YS		 816
817 if (!intel_iommu_superpage) {
161f6934 818 return 0;
6dd9a7c7
YS		 819 }
820
8140a95d 821 /* set iommu_superpage to the smallest common denominator */
0e242612 822 rcu_read_lock();
8140a95d 823 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 824 if (iommu != skip) {
825 mask &= cap_super_page_val(iommu->cap);
826 if (!mask)
827 break;
6dd9a7c7
YS		 828 }
829 }
0e242612
JL		 830 rcu_read_unlock();
831
161f6934 832 return fls(mask);
6dd9a7c7
YS		 833}
834
58c610bd
SY		 835/* Some capabilities may be different across iommus */
836static void domain_update_iommu_cap(struct dmar_domain *domain)
837{
838 domain_update_iommu_coherency(domain);
161f6934
JL		 839 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
840 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 841}
842
03ecc32c
DW		 843static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
844 u8 bus, u8 devfn, int alloc)
845{
846 struct root_entry *root = &iommu->root_entry[bus];
847 struct context_entry *context;
848 u64 *entry;
849
4df4eab1 850 entry = &root->lo;
c83b2f20 851 if (ecs_enabled(iommu)) {
03ecc32c
DW		 852 if (devfn >= 0x80) {
853 devfn -= 0x80;
854 entry = &root->hi;
855 }
856 devfn *= 2;
857 }
03ecc32c
DW		 858 if (*entry & 1)
859 context = phys_to_virt(*entry & VTD_PAGE_MASK);
860 else {
861 unsigned long phy_addr;
862 if (!alloc)
863 return NULL;
864
865 context = alloc_pgtable_page(iommu->node);
866 if (!context)
867 return NULL;
868
869 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
870 phy_addr = virt_to_phys((void *)context);
871 *entry = phy_addr | 1;
872 __iommu_flush_cache(iommu, entry, sizeof(*entry));
873 }
874 return &context[devfn];
875}
876
4ed6a540
DW		 877static int iommu_dummy(struct device *dev)
878{
879 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
880}
881
156baca8 882static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 883{
884 struct dmar_drhd_unit *drhd = NULL;
b683b230 885 struct intel_iommu *iommu;
156baca8
DW		 886 struct device *tmp;
887 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 888 u16 segment = 0;
c7151a8d
WH		 889 int i;
890
4ed6a540
DW		 891 if (iommu_dummy(dev))
892 return NULL;
893
156baca8
DW		 894 if (dev_is_pci(dev)) {
895 pdev = to_pci_dev(dev);
896 segment = pci_domain_nr(pdev->bus);
ca5b74d2 897 } else if (has_acpi_companion(dev))
156baca8
DW		 898 dev = &ACPI_COMPANION(dev)->dev;
899
0e242612 900 rcu_read_lock();
b683b230 901 for_each_active_iommu(iommu, drhd) {
156baca8 902 if (pdev && segment != drhd->segment)
276dbf99 903 continue;
c7151a8d 904
b683b230 905 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 906 drhd->devices_cnt, i, tmp) {
907 if (tmp == dev) {
908 *bus = drhd->devices[i].bus;
909 *devfn = drhd->devices[i].devfn;
b683b230 910 goto out;
156baca8
DW		 911 }
912
913 if (!pdev || !dev_is_pci(tmp))
914 continue;
915
916 ptmp = to_pci_dev(tmp);
917 if (ptmp->subordinate &&
918 ptmp->subordinate->number <= pdev->bus->number &&
919 ptmp->subordinate->busn_res.end >= pdev->bus->number)
920 goto got_pdev;
924b6231 921 }
c7151a8d 922
156baca8
DW		 923 if (pdev && drhd->include_all) {
924 got_pdev:
925 *bus = pdev->bus->number;
926 *devfn = pdev->devfn;
b683b230 927 goto out;
156baca8 928 }
c7151a8d 929 }
b683b230 930 iommu = NULL;
156baca8 931 out:
0e242612 932 rcu_read_unlock();
c7151a8d 933
b683b230 934 return iommu;
c7151a8d
WH		 935}
936
5331fe6f
WH		 937static void domain_flush_cache(struct dmar_domain *domain,
938 void *addr, int size)
939{
940 if (!domain->iommu_coherency)
941 clflush_cache_range(addr, size);
942}
943
ba395927
KA		 944static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
945{
ba395927 946 struct context_entry *context;
03ecc32c 947 int ret = 0;
ba395927
KA		 948 unsigned long flags;
949
950 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 951 context = iommu_context_addr(iommu, bus, devfn, 0);
952 if (context)
953 ret = context_present(context);
ba395927
KA		 954 spin_unlock_irqrestore(&iommu->lock, flags);
955 return ret;
956}
957
958static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
959{
ba395927
KA		 960 struct context_entry *context;
961 unsigned long flags;
962
963 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 964 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 965 if (context) {
03ecc32c
DW		 966 context_clear_entry(context);
967 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 968 }
969 spin_unlock_irqrestore(&iommu->lock, flags);
970}
971
972static void free_context_table(struct intel_iommu *iommu)
973{
ba395927
KA		 974 int i;
975 unsigned long flags;
976 struct context_entry *context;
977
978 spin_lock_irqsave(&iommu->lock, flags);
979 if (!iommu->root_entry) {
980 goto out;
981 }
982 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 983 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 984 if (context)
985 free_pgtable_page(context);
03ecc32c 986
c83b2f20 987 if (!ecs_enabled(iommu))
03ecc32c
DW		 988 continue;
989
990 context = iommu_context_addr(iommu, i, 0x80, 0);
991 if (context)
992 free_pgtable_page(context);
993
ba395927
KA		 994 }
995 free_pgtable_page(iommu->root_entry);
996 iommu->root_entry = NULL;
997out:
998 spin_unlock_irqrestore(&iommu->lock, flags);
999}
1000
b026fd28 1001static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 1002 unsigned long pfn, int *target_level)
ba395927 1003{
ba395927
KA		 1004 struct dma_pte *parent, *pte = NULL;
1005 int level = agaw_to_level(domain->agaw);
4399c8bf 1006 int offset;
ba395927
KA		 1007
1008 BUG_ON(!domain->pgd);
f9423606 1009
162d1b10 1010 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 1011 /* Address beyond IOMMU's addressing capabilities. */
1012 return NULL;
1013
ba395927
KA		 1014 parent = domain->pgd;
1015
5cf0a76f 1016 while (1) {
ba395927
KA		 1017 void *tmp_page;
1018
b026fd28 1019 offset = pfn_level_offset(pfn, level);
ba395927 1020 pte = &parent[offset];
5cf0a76f 1021 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 1022 break;
5cf0a76f 1023 if (level == *target_level)
ba395927
KA		 1024 break;
1025
19c239ce 1026 if (!dma_pte_present(pte)) {
c85994e4
DW		 1027 uint64_t pteval;
1028
4c923d47 1029 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 1030
206a73c1 1031 if (!tmp_page)
ba395927 1032 return NULL;
206a73c1 1033
c85994e4 1034 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 1035 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 1036 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 1037 /* Someone else set it while we were thinking; use theirs. */
1038 free_pgtable_page(tmp_page);
effad4b5 1039 else
c85994e4 1040 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 1041 }
5cf0a76f
DW		 1042 if (level == 1)
1043 break;
1044
19c239ce 1045 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1046 level--;
1047 }
1048
5cf0a76f
DW		 1049 if (!*target_level)
1050 *target_level = level;
1051
ba395927
KA		 1052 return pte;
1053}
1054
6dd9a7c7 1055
ba395927 1056/* return address's pte at specific level */
90dcfb5e
DW		 1057static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1058 unsigned long pfn,
6dd9a7c7 1059 int level, int *large_page)
ba395927
KA		 1060{
1061 struct dma_pte *parent, *pte = NULL;
1062 int total = agaw_to_level(domain->agaw);
1063 int offset;
1064
1065 parent = domain->pgd;
1066 while (level <= total) {
90dcfb5e 1067 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1068 pte = &parent[offset];
1069 if (level == total)
1070 return pte;
1071
6dd9a7c7
YS		 1072 if (!dma_pte_present(pte)) {
1073 *large_page = total;
ba395927 1074 break;
6dd9a7c7
YS		 1075 }
1076
e16922af 1077 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1078 *large_page = total;
1079 return pte;
1080 }
1081
19c239ce 1082 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1083 total--;
1084 }
1085 return NULL;
1086}
1087
ba395927 1088/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1089static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1090 unsigned long start_pfn,
1091 unsigned long last_pfn)
ba395927 1092{
6dd9a7c7 1093 unsigned int large_page = 1;
310a5ab9 1094 struct dma_pte *first_pte, *pte;
66eae846 1095
162d1b10
JL		 1096 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1098 BUG_ON(start_pfn > last_pfn);
ba395927 1099
04b18e65 1100 /* we don't need lock here; nobody else touches the iova range */
59c36286 1101 do {
6dd9a7c7
YS		 1102 large_page = 1;
1103 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1104 if (!pte) {
6dd9a7c7 1105 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1106 continue;
1107 }
6dd9a7c7 1108 do {
310a5ab9 1109 dma_clear_pte(pte);
6dd9a7c7 1110 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1111 pte++;
75e6bf96
DW		 1112 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1113
310a5ab9
DW		 1114 domain_flush_cache(domain, first_pte,
1115 (void *)pte - (void *)first_pte);
59c36286
DW		 1116
1117 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1118}
1119
3269ee0b
AW		 1120static void dma_pte_free_level(struct dmar_domain *domain, int level,
1121 struct dma_pte *pte, unsigned long pfn,
1122 unsigned long start_pfn, unsigned long last_pfn)
1123{
1124 pfn = max(start_pfn, pfn);
1125 pte = &pte[pfn_level_offset(pfn, level)];
1126
1127 do {
1128 unsigned long level_pfn;
1129 struct dma_pte *level_pte;
1130
1131 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1132 goto next;
1133
1134 level_pfn = pfn & level_mask(level - 1);
1135 level_pte = phys_to_virt(dma_pte_addr(pte));
1136
1137 if (level > 2)
1138 dma_pte_free_level(domain, level - 1, level_pte,
1139 level_pfn, start_pfn, last_pfn);
1140
1141 /* If range covers entire pagetable, free it */
1142 if (!(start_pfn > level_pfn ||
08336fd2 1143 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1144 dma_clear_pte(pte);
1145 domain_flush_cache(domain, pte, sizeof(*pte));
1146 free_pgtable_page(level_pte);
1147 }
1148next:
1149 pfn += level_size(level);
1150 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1151}
1152
3d1a2442 1153/* clear last level (leaf) ptes and free page table pages. */
ba395927 1154static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1155 unsigned long start_pfn,
1156 unsigned long last_pfn)
ba395927 1157{
162d1b10
JL		 1158 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1159 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1160 BUG_ON(start_pfn > last_pfn);
ba395927 1161
d41a4adb
JL		 1162 dma_pte_clear_range(domain, start_pfn, last_pfn);
1163
f3a0a52f 1164 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1165 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1166 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1167
ba395927 1168 /* free pgd */
d794dc9b 1169 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1170 free_pgtable_page(domain->pgd);
1171 domain->pgd = NULL;
1172 }
1173}
1174
ea8ea460
DW		 1175/* When a page at a given level is being unlinked from its parent, we don't
1176 need to *modify* it at all. All we need to do is make a list of all the
1177 pages which can be freed just as soon as we've flushed the IOTLB and we
1178 know the hardware page-walk will no longer touch them.
1179 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1180 be freed. */
1181static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1182 int level, struct dma_pte *pte,
1183 struct page *freelist)
1184{
1185 struct page *pg;
1186
1187 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1188 pg->freelist = freelist;
1189 freelist = pg;
1190
1191 if (level == 1)
1192 return freelist;
1193
adeb2590
JL		 1194 pte = page_address(pg);
1195 do {
ea8ea460
DW		 1196 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1197 freelist = dma_pte_list_pagetables(domain, level - 1,
1198 pte, freelist);
adeb2590
JL		 1199 pte++;
1200 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1201
1202 return freelist;
1203}
1204
1205static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1206 struct dma_pte *pte, unsigned long pfn,
1207 unsigned long start_pfn,
1208 unsigned long last_pfn,
1209 struct page *freelist)
1210{
1211 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1212
1213 pfn = max(start_pfn, pfn);
1214 pte = &pte[pfn_level_offset(pfn, level)];
1215
1216 do {
1217 unsigned long level_pfn;
1218
1219 if (!dma_pte_present(pte))
1220 goto next;
1221
1222 level_pfn = pfn & level_mask(level);
1223
1224 /* If range covers entire pagetable, free it */
1225 if (start_pfn <= level_pfn &&
1226 last_pfn >= level_pfn + level_size(level) - 1) {
1227 /* These suborbinate page tables are going away entirely. Don't
1228 bother to clear them; we're just going to *free* them. */
1229 if (level > 1 && !dma_pte_superpage(pte))
1230 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1231
1232 dma_clear_pte(pte);
1233 if (!first_pte)
1234 first_pte = pte;
1235 last_pte = pte;
1236 } else if (level > 1) {
1237 /* Recurse down into a level that isn't *entirely* obsolete */
1238 freelist = dma_pte_clear_level(domain, level - 1,
1239 phys_to_virt(dma_pte_addr(pte)),
1240 level_pfn, start_pfn, last_pfn,
1241 freelist);
1242 }
1243next:
1244 pfn += level_size(level);
1245 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1246
1247 if (first_pte)
1248 domain_flush_cache(domain, first_pte,
1249 (void *)++last_pte - (void *)first_pte);
1250
1251 return freelist;
1252}
1253
1254/* We can't just free the pages because the IOMMU may still be walking
1255 the page tables, and may have cached the intermediate levels. The
1256 pages can only be freed after the IOTLB flush has been done. */
b690420a
JR		 1257static struct page *domain_unmap(struct dmar_domain *domain,
1258 unsigned long start_pfn,
1259 unsigned long last_pfn)
ea8ea460 1260{
ea8ea460
DW		 1261 struct page *freelist = NULL;
1262
162d1b10
JL		 1263 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1264 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1265 BUG_ON(start_pfn > last_pfn);
1266
1267 /* we don't need lock here; nobody else touches the iova range */
1268 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1269 domain->pgd, 0, start_pfn, last_pfn, NULL);
1270
1271 /* free pgd */
1272 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1273 struct page *pgd_page = virt_to_page(domain->pgd);
1274 pgd_page->freelist = freelist;
1275 freelist = pgd_page;
1276
1277 domain->pgd = NULL;
1278 }
1279
1280 return freelist;
1281}
1282
b690420a 1283static void dma_free_pagelist(struct page *freelist)
ea8ea460
DW		 1284{
1285 struct page *pg;
1286
1287 while ((pg = freelist)) {
1288 freelist = pg->freelist;
1289 free_pgtable_page(page_address(pg));
1290 }
1291}
1292
ba395927
KA		 1293/* iommu handling */
1294static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1295{
1296 struct root_entry *root;
1297 unsigned long flags;
1298
4c923d47 1299 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1300 if (!root) {
9f10e5bf 1301 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1302 iommu->name);
ba395927 1303 return -ENOMEM;
ffebeb46 1304 }
ba395927 1305
5b6985ce 1306 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1307
1308 spin_lock_irqsave(&iommu->lock, flags);
1309 iommu->root_entry = root;
1310 spin_unlock_irqrestore(&iommu->lock, flags);
1311
1312 return 0;
1313}
1314
ba395927
KA		 1315static void iommu_set_root_entry(struct intel_iommu *iommu)
1316{
03ecc32c 1317 u64 addr;
c416daa9 1318 u32 sts;
ba395927
KA		 1319 unsigned long flag;
1320
03ecc32c 1321 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1322 if (ecs_enabled(iommu))
03ecc32c 1323 addr |= DMA_RTADDR_RTT;
ba395927 1324
1f5b3c3f 1325 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1326 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1327
c416daa9 1328 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1329
1330 /* Make sure hardware complete it */
1331 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1332 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1333
1f5b3c3f 1334 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1335}
1336
1337static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1338{
1339 u32 val;
1340 unsigned long flag;
1341
9af88143 1342 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1343 return;
ba395927 1344
1f5b3c3f 1345 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1346 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1347
1348 /* Make sure hardware complete it */
1349 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1350 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1351
1f5b3c3f 1352 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1353}
1354
1355/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1356static void __iommu_flush_context(struct intel_iommu *iommu,
1357 u16 did, u16 source_id, u8 function_mask,
1358 u64 type)
ba395927
KA		 1359{
1360 u64 val = 0;
1361 unsigned long flag;
1362
ba395927
KA		 1363 switch (type) {
1364 case DMA_CCMD_GLOBAL_INVL:
1365 val = DMA_CCMD_GLOBAL_INVL;
1366 break;
1367 case DMA_CCMD_DOMAIN_INVL:
1368 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1369 break;
1370 case DMA_CCMD_DEVICE_INVL:
1371 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1372 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1373 break;
1374 default:
1375 BUG();
1376 }
1377 val |= DMA_CCMD_ICC;
1378
1f5b3c3f 1379 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1380 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1381
1382 /* Make sure hardware complete it */
1383 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1384 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1385
1f5b3c3f 1386 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1387}
1388
ba395927 1389/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1390static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1391 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1392{
1393 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1394 u64 val = 0, val_iva = 0;
1395 unsigned long flag;
1396
ba395927
KA		 1397 switch (type) {
1398 case DMA_TLB_GLOBAL_FLUSH:
1399 /* global flush doesn't need set IVA_REG */
1400 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1401 break;
1402 case DMA_TLB_DSI_FLUSH:
1403 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1404 break;
1405 case DMA_TLB_PSI_FLUSH:
1406 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1407 /* IH bit is passed in as part of address */
ba395927
KA		 1408 val_iva = size_order | addr;
1409 break;
1410 default:
1411 BUG();
1412 }
1413 /* Note: set drain read/write */
1414#if 0
1415 /*
1416 * This is probably to be super secure.. Looks like we can
1417 * ignore it without any impact.
1418 */
1419 if (cap_read_drain(iommu->cap))
1420 val |= DMA_TLB_READ_DRAIN;
1421#endif
1422 if (cap_write_drain(iommu->cap))
1423 val |= DMA_TLB_WRITE_DRAIN;
1424
1f5b3c3f 1425 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1426 /* Note: Only uses first TLB reg currently */
1427 if (val_iva)
1428 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1429 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1430
1431 /* Make sure hardware complete it */
1432 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1433 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1434
1f5b3c3f 1435 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1436
1437 /* check IOTLB invalidation granularity */
1438 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1439 pr_err("Flush IOTLB failed\n");
ba395927 1440 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1441 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1442 (unsigned long long)DMA_TLB_IIRG(type),
1443 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1444}
1445
64ae892b
DW		 1446static struct device_domain_info *
1447iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1448 u8 bus, u8 devfn)
93a23a72 1449{
93a23a72 1450 struct device_domain_info *info;
93a23a72 1451
55d94043
JR		 1452 assert_spin_locked(&device_domain_lock);
1453
93a23a72
YZ		 1454 if (!iommu->qi)
1455 return NULL;
1456
93a23a72 1457 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1458 if (info->iommu == iommu && info->bus == bus &&
1459 info->devfn == devfn) {
b16d0cb9
DW		 1460 if (info->ats_supported && info->dev)
1461 return info;
93a23a72
YZ		 1462 break;
1463 }
93a23a72 1464
b16d0cb9 1465 return NULL;
93a23a72
YZ		 1466}
1467
0824c592
OP		 1468static void domain_update_iotlb(struct dmar_domain *domain)
1469{
1470 struct device_domain_info *info;
1471 bool has_iotlb_device = false;
1472
1473 assert_spin_locked(&device_domain_lock);
1474
1475 list_for_each_entry(info, &domain->devices, link) {
1476 struct pci_dev *pdev;
1477
1478 if (!info->dev || !dev_is_pci(info->dev))
1479 continue;
1480
1481 pdev = to_pci_dev(info->dev);
1482 if (pdev->ats_enabled) {
1483 has_iotlb_device = true;
1484 break;
1485 }
1486 }
1487
1488 domain->has_iotlb_device = has_iotlb_device;
1489}
1490
93a23a72 1491static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1492{
fb0cc3aa
BH		 1493 struct pci_dev *pdev;
1494
0824c592
OP		 1495 assert_spin_locked(&device_domain_lock);
1496
0bcb3e28 1497 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1498 return;
1499
fb0cc3aa 1500 pdev = to_pci_dev(info->dev);
fb0cc3aa 1501
b16d0cb9
DW		 1502#ifdef CONFIG_INTEL_IOMMU_SVM
1503 /* The PCIe spec, in its wisdom, declares that the behaviour of
1504 the device if you enable PASID support after ATS support is
1505 undefined. So always enable PASID support on devices which
1506 have it, even if we can't yet know if we're ever going to
1507 use it. */
1508 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1509 info->pasid_enabled = 1;
1510
1511 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1512 info->pri_enabled = 1;
1513#endif
1514 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1515 info->ats_enabled = 1;
0824c592 1516 domain_update_iotlb(info->domain);
b16d0cb9
DW		 1517 info->ats_qdep = pci_ats_queue_depth(pdev);
1518 }
93a23a72
YZ		 1519}
1520
1521static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1522{
b16d0cb9
DW		 1523 struct pci_dev *pdev;
1524
0824c592
OP		 1525 assert_spin_locked(&device_domain_lock);
1526
da972fb1 1527 if (!dev_is_pci(info->dev))
93a23a72
YZ		 1528 return;
1529
b16d0cb9
DW		 1530 pdev = to_pci_dev(info->dev);
1531
1532 if (info->ats_enabled) {
1533 pci_disable_ats(pdev);
1534 info->ats_enabled = 0;
0824c592 1535 domain_update_iotlb(info->domain);
b16d0cb9
DW		 1536 }
1537#ifdef CONFIG_INTEL_IOMMU_SVM
1538 if (info->pri_enabled) {
1539 pci_disable_pri(pdev);
1540 info->pri_enabled = 0;
1541 }
1542 if (info->pasid_enabled) {
1543 pci_disable_pasid(pdev);
1544 info->pasid_enabled = 0;
1545 }
1546#endif
93a23a72
YZ		 1547}
1548
1549static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1550 u64 addr, unsigned mask)
1551{
1552 u16 sid, qdep;
1553 unsigned long flags;
1554 struct device_domain_info *info;
1555
0824c592
OP		 1556 if (!domain->has_iotlb_device)
1557 return;
1558
93a23a72
YZ		 1559 spin_lock_irqsave(&device_domain_lock, flags);
1560 list_for_each_entry(info, &domain->devices, link) {
b16d0cb9 1561 if (!info->ats_enabled)
93a23a72
YZ		 1562 continue;
1563
1564 sid = info->bus << 8 | info->devfn;
b16d0cb9 1565 qdep = info->ats_qdep;
93a23a72
YZ		 1566 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1567 }
1568 spin_unlock_irqrestore(&device_domain_lock, flags);
1569}
1570
a1ddcbe9
JR		 1571static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1572 struct dmar_domain *domain,
1573 unsigned long pfn, unsigned int pages,
1574 int ih, int map)
ba395927 1575{
9dd2fe89 1576 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1577 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1578 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1579
ba395927
KA		 1580 BUG_ON(pages == 0);
1581
ea8ea460
DW		 1582 if (ih)
1583 ih = 1 << 6;
ba395927 1584 /*
9dd2fe89
YZ		 1585 * Fallback to domain selective flush if no PSI support or the size is
1586 * too big.
ba395927
KA		 1587 * PSI requires page size to be 2 ^ x, and the base address is naturally
1588 * aligned to the size
1589 */
9dd2fe89
YZ		 1590 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1591 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1592 DMA_TLB_DSI_FLUSH);
9dd2fe89 1593 else
ea8ea460 1594 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1595 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1596
1597 /*
82653633
NA		 1598 * In caching mode, changes of pages from non-present to present require
1599 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1600 */
82653633 1601 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1602 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1603 addr, mask);
ba395927
KA		 1604}
1605
f8bab735 1606static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1607{
1608 u32 pmen;
1609 unsigned long flags;
1610
1f5b3c3f 1611 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1612 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1613 pmen &= ~DMA_PMEN_EPM;
1614 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1615
1616 /* wait for the protected region status bit to clear */
1617 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1618 readl, !(pmen & DMA_PMEN_PRS), pmen);
1619
1f5b3c3f 1620 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1621}
1622
2a41ccee 1623static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1624{
1625 u32 sts;
1626 unsigned long flags;
1627
1f5b3c3f 1628 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1629 iommu->gcmd |= DMA_GCMD_TE;
1630 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1631
1632 /* Make sure hardware complete it */
1633 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1634 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1635
1f5b3c3f 1636 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1637}
1638
2a41ccee 1639static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1640{
1641 u32 sts;
1642 unsigned long flag;
1643
1f5b3c3f 1644 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1645 iommu->gcmd &= ~DMA_GCMD_TE;
1646 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648 /* Make sure hardware complete it */
1649 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1650 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1651
1f5b3c3f 1652 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1653}
1654
3460a6d9 1655
ba395927
KA		 1656static int iommu_init_domains(struct intel_iommu *iommu)
1657{
8bf47816
JR		 1658 u32 ndomains, nlongs;
1659 size_t size;
ba395927
KA		 1660
1661 ndomains = cap_ndoms(iommu->cap);
8bf47816 1662 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1663 iommu->name, ndomains);
ba395927
KA		 1664 nlongs = BITS_TO_LONGS(ndomains);
1665
94a91b50
DD		 1666 spin_lock_init(&iommu->lock);
1667
ba395927
KA		 1668 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1669 if (!iommu->domain_ids) {
9f10e5bf
JR		 1670 pr_err("%s: Allocating domain id array failed\n",
1671 iommu->name);
ba395927
KA		 1672 return -ENOMEM;
1673 }
8bf47816 1674
86f004c7 1675 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
8bf47816
JR		 1676 iommu->domains = kzalloc(size, GFP_KERNEL);
1677
1678 if (iommu->domains) {
1679 size = 256 * sizeof(struct dmar_domain *);
1680 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1681 }
1682
1683 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1684 pr_err("%s: Allocating domain array failed\n",
1685 iommu->name);
852bdb04 1686 kfree(iommu->domain_ids);
8bf47816 1687 kfree(iommu->domains);
852bdb04 1688 iommu->domain_ids = NULL;
8bf47816 1689 iommu->domains = NULL;
ba395927
KA		 1690 return -ENOMEM;
1691 }
1692
8bf47816
JR		 1693
1694
ba395927 1695 /*
c0e8a6c8
JR		 1696 * If Caching mode is set, then invalid translations are tagged
1697 * with domain-id 0, hence we need to pre-allocate it. We also
1698 * use domain-id 0 as a marker for non-allocated domain-id, so
1699 * make sure it is not used for a real domain.
ba395927 1700 */
c0e8a6c8
JR		 1701 set_bit(0, iommu->domain_ids);
1702
ba395927
KA		 1703 return 0;
1704}
ba395927 1705
ffebeb46 1706static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1707{
29a27719 1708 struct device_domain_info *info, *tmp;
55d94043 1709 unsigned long flags;
ba395927 1710
29a27719
JR		 1711 if (!iommu->domains || !iommu->domain_ids)
1712 return;
a4eaa86c 1713
55d94043 1714 spin_lock_irqsave(&device_domain_lock, flags);
29a27719
JR		 1715 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1716 struct dmar_domain *domain;
1717
1718 if (info->iommu != iommu)
1719 continue;
1720
1721 if (!info->dev || !info->domain)
1722 continue;
1723
1724 domain = info->domain;
1725
e6de0f8d 1726 dmar_remove_one_dev_info(domain, info->dev);
29a27719
JR		 1727
1728 if (!domain_type_is_vm_or_si(domain))
1729 domain_exit(domain);
ba395927 1730 }
55d94043 1731 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1732
1733 if (iommu->gcmd & DMA_GCMD_TE)
1734 iommu_disable_translation(iommu);
ffebeb46 1735}
ba395927 1736
ffebeb46
JL		 1737static void free_dmar_iommu(struct intel_iommu *iommu)
1738{
1739 if ((iommu->domains) && (iommu->domain_ids)) {
86f004c7 1740 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
8bf47816
JR		 1741 int i;
1742
1743 for (i = 0; i < elems; i++)
1744 kfree(iommu->domains[i]);
ffebeb46
JL		 1745 kfree(iommu->domains);
1746 kfree(iommu->domain_ids);
1747 iommu->domains = NULL;
1748 iommu->domain_ids = NULL;
1749 }
ba395927 1750
d9630fe9
WH		 1751 g_iommus[iommu->seq_id] = NULL;
1752
ba395927
KA		 1753 /* free context mapping */
1754 free_context_table(iommu);
8a94ade4
DW		 1755
1756#ifdef CONFIG_INTEL_IOMMU_SVM
a222a7f0
DW		 1757 if (pasid_enabled(iommu)) {
1758 if (ecap_prs(iommu->ecap))
1759 intel_svm_finish_prq(iommu);
8a94ade4 1760 intel_svm_free_pasid_tables(iommu);
a222a7f0 1761 }
8a94ade4 1762#endif
ba395927
KA		 1763}
1764
ab8dfe25 1765static struct dmar_domain *alloc_domain(int flags)
ba395927 1766{
ba395927 1767 struct dmar_domain *domain;
ba395927
KA		 1768
1769 domain = alloc_domain_mem();
1770 if (!domain)
1771 return NULL;
1772
ab8dfe25 1773 memset(domain, 0, sizeof(*domain));
4c923d47 1774 domain->nid = -1;
ab8dfe25 1775 domain->flags = flags;
0824c592 1776 domain->has_iotlb_device = false;
92d03cc8 1777 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1778
1779 return domain;
1780}
1781
d160aca5
JR		 1782/* Must be called with iommu->lock */
1783static int domain_attach_iommu(struct dmar_domain *domain,
fb170fb4
JL		 1784 struct intel_iommu *iommu)
1785{
44bde614 1786 unsigned long ndomains;
55d94043 1787 int num;
44bde614 1788
55d94043 1789 assert_spin_locked(&device_domain_lock);
d160aca5 1790 assert_spin_locked(&iommu->lock);
ba395927 1791
29a27719
JR		 1792 domain->iommu_refcnt[iommu->seq_id] += 1;
1793 domain->iommu_count += 1;
1794 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
fb170fb4 1795 ndomains = cap_ndoms(iommu->cap);
d160aca5
JR		 1796 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1797
1798 if (num >= ndomains) {
1799 pr_err("%s: No free domain ids\n", iommu->name);
1800 domain->iommu_refcnt[iommu->seq_id] -= 1;
1801 domain->iommu_count -= 1;
55d94043 1802 return -ENOSPC;
2c2e2c38 1803 }
ba395927 1804
d160aca5
JR		 1805 set_bit(num, iommu->domain_ids);
1806 set_iommu_domain(iommu, num, domain);
1807
1808 domain->iommu_did[iommu->seq_id] = num;
1809 domain->nid = iommu->node;
fb170fb4 1810
fb170fb4
JL		 1811 domain_update_iommu_cap(domain);
1812 }
d160aca5 1813
55d94043 1814 return 0;
fb170fb4
JL		 1815}
1816
1817static int domain_detach_iommu(struct dmar_domain *domain,
1818 struct intel_iommu *iommu)
1819{
d160aca5 1820 int num, count = INT_MAX;
d160aca5 1821
55d94043 1822 assert_spin_locked(&device_domain_lock);
d160aca5 1823 assert_spin_locked(&iommu->lock);
fb170fb4 1824
29a27719
JR		 1825 domain->iommu_refcnt[iommu->seq_id] -= 1;
1826 count = --domain->iommu_count;
1827 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
d160aca5
JR		 1828 num = domain->iommu_did[iommu->seq_id];
1829 clear_bit(num, iommu->domain_ids);
1830 set_iommu_domain(iommu, num, NULL);
fb170fb4 1831
fb170fb4 1832 domain_update_iommu_cap(domain);
c0e8a6c8 1833 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4 1834 }
fb170fb4
JL		 1835
1836 return count;
1837}
1838
ba395927 1839static struct iova_domain reserved_iova_list;
8a443df4 1840static struct lock_class_key reserved_rbtree_key;
ba395927 1841
51a63e67 1842static int dmar_init_reserved_ranges(void)
ba395927
KA		 1843{
1844 struct pci_dev *pdev = NULL;
1845 struct iova *iova;
1846 int i;
ba395927 1847
0fb5fe87
RM		 1848 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1849 DMA_32BIT_PFN);
ba395927 1850
8a443df4
MG		 1851 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852 &reserved_rbtree_key);
1853
ba395927
KA		 1854 /* IOAPIC ranges shouldn't be accessed by DMA */
1855 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1857 if (!iova) {
9f10e5bf 1858 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1859 return -ENODEV;
1860 }
ba395927
KA		 1861
1862 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863 for_each_pci_dev(pdev) {
1864 struct resource *r;
1865
1866 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867 r = &pdev->resource[i];
1868 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869 continue;
1a4a4551
DW		 1870 iova = reserve_iova(&reserved_iova_list,
1871 IOVA_PFN(r->start),
1872 IOVA_PFN(r->end));
51a63e67 1873 if (!iova) {
9f10e5bf 1874 pr_err("Reserve iova failed\n");
51a63e67
JC		 1875 return -ENODEV;
1876 }
ba395927
KA		 1877 }
1878 }
51a63e67 1879 return 0;
ba395927
KA		 1880}
1881
1882static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883{
1884 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885}
1886
1887static inline int guestwidth_to_adjustwidth(int gaw)
1888{
1889 int agaw;
1890 int r = (gaw - 12) % 9;
1891
1892 if (r == 0)
1893 agaw = gaw;
1894 else
1895 agaw = gaw + 9 - r;
1896 if (agaw > 64)
1897 agaw = 64;
1898 return agaw;
1899}
1900
dc534b25
JR		 1901static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902 int guest_width)
ba395927 1903{
ba395927
KA		 1904 int adjust_width, agaw;
1905 unsigned long sagaw;
1906
0fb5fe87
RM		 1907 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1908 DMA_32BIT_PFN);
ba395927
KA		 1909 domain_reserve_special_ranges(domain);
1910
1911 /* calculate AGAW */
ba395927
KA		 1912 if (guest_width > cap_mgaw(iommu->cap))
1913 guest_width = cap_mgaw(iommu->cap);
1914 domain->gaw = guest_width;
1915 adjust_width = guestwidth_to_adjustwidth(guest_width);
1916 agaw = width_to_agaw(adjust_width);
1917 sagaw = cap_sagaw(iommu->cap);
1918 if (!test_bit(agaw, &sagaw)) {
1919 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1920 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1921 agaw = find_next_bit(&sagaw, 5, agaw);
1922 if (agaw >= 5)
1923 return -ENODEV;
1924 }
1925 domain->agaw = agaw;
ba395927 1926
8e604097
WH		 1927 if (ecap_coherent(iommu->ecap))
1928 domain->iommu_coherency = 1;
1929 else
1930 domain->iommu_coherency = 0;
1931
58c610bd
SY		 1932 if (ecap_sc_support(iommu->ecap))
1933 domain->iommu_snooping = 1;
1934 else
1935 domain->iommu_snooping = 0;
1936
214e39aa
DW		 1937 if (intel_iommu_superpage)
1938 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1939 else
1940 domain->iommu_superpage = 0;
1941
4c923d47 1942 domain->nid = iommu->node;
c7151a8d 1943
ba395927 1944 /* always allocate the top pgd */
4c923d47 1945 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1946 if (!domain->pgd)
1947 return -ENOMEM;
5b6985ce 1948 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1949 return 0;
1950}
1951
1952static void domain_exit(struct dmar_domain *domain)
1953{
ea8ea460 1954 struct page *freelist = NULL;
ba395927
KA		 1955
1956 /* Domain 0 is reserved, so dont process it */
1957 if (!domain)
1958 return;
1959
7b668357 1960 /* Flush any lazy unmaps that may reference this domain */
aa473240
OP		 1961 if (!intel_iommu_strict) {
1962 int cpu;
1963
1964 for_each_possible_cpu(cpu)
1965 flush_unmaps_timeout(cpu);
1966 }
7b668357 1967
d160aca5
JR		 1968 /* Remove associated devices and clear attached or cached domains */
1969 rcu_read_lock();
ba395927 1970 domain_remove_dev_info(domain);
d160aca5 1971 rcu_read_unlock();
92d03cc8 1972
ba395927
KA		 1973 /* destroy iovas */
1974 put_iova_domain(&domain->iovad);
ba395927 1975
ea8ea460 1976 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1977
ea8ea460
DW		 1978 dma_free_pagelist(freelist);
1979
ba395927
KA		 1980 free_domain_mem(domain);
1981}
1982
64ae892b
DW		 1983static int domain_context_mapping_one(struct dmar_domain *domain,
1984 struct intel_iommu *iommu,
28ccce0d 1985 u8 bus, u8 devfn)
ba395927 1986{
c6c2cebd 1987 u16 did = domain->iommu_did[iommu->seq_id];
28ccce0d
JR		 1988 int translation = CONTEXT_TT_MULTI_LEVEL;
1989 struct device_domain_info *info = NULL;
ba395927 1990 struct context_entry *context;
ba395927 1991 unsigned long flags;
ea6606b0 1992 struct dma_pte *pgd;
55d94043 1993 int ret, agaw;
28ccce0d 1994
c6c2cebd
JR		 1995 WARN_ON(did == 0);
1996
28ccce0d
JR		 1997 if (hw_pass_through && domain_type_is_si(domain))
1998 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1999
2000 pr_debug("Set context mapping for %02x:%02x.%d\n",
2001 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 2002
ba395927 2003 BUG_ON(!domain->pgd);
5331fe6f 2004
55d94043
JR		 2005 spin_lock_irqsave(&device_domain_lock, flags);
2006 spin_lock(&iommu->lock);
2007
2008 ret = -ENOMEM;
03ecc32c 2009 context = iommu_context_addr(iommu, bus, devfn, 1);
ba395927 2010 if (!context)
55d94043 2011 goto out_unlock;
ba395927 2012
55d94043
JR		 2013 ret = 0;
2014 if (context_present(context))
2015 goto out_unlock;
cf484d0e 2016
ea6606b0
WH		 2017 pgd = domain->pgd;
2018
de24e553 2019 context_clear_entry(context);
c6c2cebd 2020 context_set_domain_id(context, did);
ea6606b0 2021
de24e553
JR		 2022 /*
2023 * Skip top levels of page tables for iommu which has less agaw
2024 * than default. Unnecessary for PT mode.
2025 */
93a23a72 2026 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553 2027 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
55d94043 2028 ret = -ENOMEM;
de24e553 2029 pgd = phys_to_virt(dma_pte_addr(pgd));
55d94043
JR		 2030 if (!dma_pte_present(pgd))
2031 goto out_unlock;
ea6606b0 2032 }
4ed0d3e6 2033
64ae892b 2034 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
b16d0cb9
DW		 2035 if (info && info->ats_supported)
2036 translation = CONTEXT_TT_DEV_IOTLB;
2037 else
2038 translation = CONTEXT_TT_MULTI_LEVEL;
de24e553 2039
93a23a72
YZ		 2040 context_set_address_root(context, virt_to_phys(pgd));
2041 context_set_address_width(context, iommu->agaw);
de24e553
JR		 2042 } else {
2043 /*
2044 * In pass through mode, AW must be programmed to
2045 * indicate the largest AGAW value supported by
2046 * hardware. And ASR is ignored by hardware.
2047 */
2048 context_set_address_width(context, iommu->msagaw);
93a23a72 2049 }
4ed0d3e6
FY		 2050
2051 context_set_translation_type(context, translation);
c07e7d21
MM		 2052 context_set_fault_enable(context);
2053 context_set_present(context);
5331fe6f 2054 domain_flush_cache(domain, context, sizeof(*context));
ba395927 2055
4c25a2c1
DW		 2056 /*
2057 * It's a non-present to present mapping. If hardware doesn't cache
2058 * non-present entry we only need to flush the write-buffer. If the
2059 * _does_ cache non-present entries, then it does so in the special
2060 * domain #0, which we have to flush:
2061 */
2062 if (cap_caching_mode(iommu->cap)) {
2063 iommu->flush.flush_context(iommu, 0,
2064 (((u16)bus) << 8) | devfn,
2065 DMA_CCMD_MASK_NOBIT,
2066 DMA_CCMD_DEVICE_INVL);
c6c2cebd 2067 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 2068 } else {
ba395927 2069 iommu_flush_write_buffer(iommu);
4c25a2c1 2070 }
93a23a72 2071 iommu_enable_dev_iotlb(info);
c7151a8d 2072
55d94043
JR		 2073 ret = 0;
2074
2075out_unlock:
2076 spin_unlock(&iommu->lock);
2077 spin_unlock_irqrestore(&device_domain_lock, flags);
fb170fb4 2078
5c365d18 2079 return ret;
ba395927
KA		 2080}
2081
579305f7
AW		 2082struct domain_context_mapping_data {
2083 struct dmar_domain *domain;
2084 struct intel_iommu *iommu;
579305f7
AW		 2085};
2086
2087static int domain_context_mapping_cb(struct pci_dev *pdev,
2088 u16 alias, void *opaque)
2089{
2090 struct domain_context_mapping_data *data = opaque;
2091
2092 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2093 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2094}
2095
ba395927 2096static int
28ccce0d 2097domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2098{
64ae892b 2099 struct intel_iommu *iommu;
156baca8 2100 u8 bus, devfn;
579305f7 2101 struct domain_context_mapping_data data;
64ae892b 2102
e1f167f3 2103 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2104 if (!iommu)
2105 return -ENODEV;
ba395927 2106
579305f7 2107 if (!dev_is_pci(dev))
28ccce0d 2108 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2109
2110 data.domain = domain;
2111 data.iommu = iommu;
579305f7
AW		 2112
2113 return pci_for_each_dma_alias(to_pci_dev(dev),
2114 &domain_context_mapping_cb, &data);
2115}
2116
2117static int domain_context_mapped_cb(struct pci_dev *pdev,
2118 u16 alias, void *opaque)
2119{
2120 struct intel_iommu *iommu = opaque;
2121
2122 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2123}
2124
e1f167f3 2125static int domain_context_mapped(struct device *dev)
ba395927 2126{
5331fe6f 2127 struct intel_iommu *iommu;
156baca8 2128 u8 bus, devfn;
5331fe6f 2129
e1f167f3 2130 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2131 if (!iommu)
2132 return -ENODEV;
ba395927 2133
579305f7
AW		 2134 if (!dev_is_pci(dev))
2135 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2136
579305f7
AW		 2137 return !pci_for_each_dma_alias(to_pci_dev(dev),
2138 domain_context_mapped_cb, iommu);
ba395927
KA		 2139}
2140
f532959b
FY		 2141/* Returns a number of VTD pages, but aligned to MM page size */
2142static inline unsigned long aligned_nrpages(unsigned long host_addr,
2143 size_t size)
2144{
2145 host_addr &= ~PAGE_MASK;
2146 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2147}
2148
6dd9a7c7
YS		 2149/* Return largest possible superpage level for a given mapping */
2150static inline int hardware_largepage_caps(struct dmar_domain *domain,
2151 unsigned long iov_pfn,
2152 unsigned long phy_pfn,
2153 unsigned long pages)
2154{
2155 int support, level = 1;
2156 unsigned long pfnmerge;
2157
2158 support = domain->iommu_superpage;
2159
2160 /* To use a large page, the virtual *and* physical addresses
2161 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2162 of them will mean we have to use smaller pages. So just
2163 merge them and check both at once. */
2164 pfnmerge = iov_pfn | phy_pfn;
2165
2166 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2167 pages >>= VTD_STRIDE_SHIFT;
2168 if (!pages)
2169 break;
2170 pfnmerge >>= VTD_STRIDE_SHIFT;
2171 level++;
2172 support--;
2173 }
2174 return level;
2175}
2176
9051aa02
DW		 2177static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178 struct scatterlist *sg, unsigned long phys_pfn,
2179 unsigned long nr_pages, int prot)
e1605495
DW		 2180{
2181 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2182 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2183 unsigned long sg_res = 0;
6dd9a7c7
YS		 2184 unsigned int largepage_lvl = 0;
2185 unsigned long lvl_pages = 0;
e1605495 2186
162d1b10 2187 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2188
2189 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2190 return -EINVAL;
2191
2192 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2193
cc4f14aa
JL		 2194 if (!sg) {
2195 sg_res = nr_pages;
9051aa02
DW		 2196 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2197 }
2198
6dd9a7c7 2199 while (nr_pages > 0) {
c85994e4
DW		 2200 uint64_t tmp;
2201
e1605495 2202 if (!sg_res) {
f532959b 2203 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2204 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2205 sg->dma_length = sg->length;
3e6110fd 2206 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2207 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2208 }
6dd9a7c7 2209
e1605495 2210 if (!pte) {
6dd9a7c7
YS		 2211 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2212
5cf0a76f 2213 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2214 if (!pte)
2215 return -ENOMEM;
6dd9a7c7 2216 /* It is large page*/
6491d4d0 2217 if (largepage_lvl > 1) {
ba2374fd
CZ		 2218 unsigned long nr_superpages, end_pfn;
2219
6dd9a7c7 2220 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb 2221 lvl_pages = lvl_to_nr_pages(largepage_lvl);
ba2374fd
CZ		 2222
2223 nr_superpages = sg_res / lvl_pages;
2224 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2225
d41a4adb
JL		 2226 /*
2227 * Ensure that old small page tables are
ba2374fd 2228 * removed to make room for superpage(s).
d41a4adb 2229 */
ba2374fd 2230 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
6491d4d0 2231 } else {
6dd9a7c7 2232 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2233 }
6dd9a7c7 2234
e1605495
DW		 2235 }
2236 /* We don't need lock here, nobody else
2237 * touches the iova range
2238 */
7766a3fb 2239 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2240 if (tmp) {
1bf20f0d 2241 static int dumps = 5;
9f10e5bf
JR		 2242 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2243 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2244 if (dumps) {
2245 dumps--;
2246 debug_dma_dump_mappings(NULL);
2247 }
2248 WARN_ON(1);
2249 }
6dd9a7c7
YS		 2250
2251 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2252
2253 BUG_ON(nr_pages < lvl_pages);
2254 BUG_ON(sg_res < lvl_pages);
2255
2256 nr_pages -= lvl_pages;
2257 iov_pfn += lvl_pages;
2258 phys_pfn += lvl_pages;
2259 pteval += lvl_pages * VTD_PAGE_SIZE;
2260 sg_res -= lvl_pages;
2261
2262 /* If the next PTE would be the first in a new page, then we
2263 need to flush the cache on the entries we've just written.
2264 And then we'll need to recalculate 'pte', so clear it and
2265 let it get set again in the if (!pte) block above.
2266
2267 If we're done (!nr_pages) we need to flush the cache too.
2268
2269 Also if we've been setting superpages, we may need to
2270 recalculate 'pte' and switch back to smaller pages for the
2271 end of the mapping, if the trailing size is not enough to
2272 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2273 pte++;
6dd9a7c7
YS		 2274 if (!nr_pages || first_pte_in_page(pte) ||
2275 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2276 domain_flush_cache(domain, first_pte,
2277 (void *)pte - (void *)first_pte);
2278 pte = NULL;
2279 }
6dd9a7c7
YS		 2280
2281 if (!sg_res && nr_pages)
e1605495
DW		 2282 sg = sg_next(sg);
2283 }
2284 return 0;
2285}
2286
9051aa02
DW		 2287static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2288 struct scatterlist *sg, unsigned long nr_pages,
2289 int prot)
ba395927 2290{
9051aa02
DW		 2291 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2292}
6f6a00e4 2293
9051aa02
DW		 2294static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2295 unsigned long phys_pfn, unsigned long nr_pages,
2296 int prot)
2297{
2298 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2299}
2300
2452d9db 2301static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2302{
c7151a8d
WH		 2303 if (!iommu)
2304 return;
8c11e798
WH		 2305
2306 clear_context_table(iommu, bus, devfn);
2307 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2308 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2309 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2310}
2311
109b9b04
DW		 2312static inline void unlink_domain_info(struct device_domain_info *info)
2313{
2314 assert_spin_locked(&device_domain_lock);
2315 list_del(&info->link);
2316 list_del(&info->global);
2317 if (info->dev)
0bcb3e28 2318 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2319}
2320
ba395927
KA		 2321static void domain_remove_dev_info(struct dmar_domain *domain)
2322{
3a74ca01 2323 struct device_domain_info *info, *tmp;
fb170fb4 2324 unsigned long flags;
ba395927
KA		 2325
2326 spin_lock_irqsave(&device_domain_lock, flags);
76f45fe3 2327 list_for_each_entry_safe(info, tmp, &domain->devices, link)
127c7615 2328 __dmar_remove_one_dev_info(info);
ba395927
KA		 2329 spin_unlock_irqrestore(&device_domain_lock, flags);
2330}
2331
2332/*
2333 * find_domain
1525a29a 2334 * Note: we use struct device->archdata.iommu stores the info
ba395927 2335 */
1525a29a 2336static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2337{
2338 struct device_domain_info *info;
2339
2340 /* No lock here, assumes no domain exit in normal case */
1525a29a 2341 info = dev->archdata.iommu;
ba395927
KA		 2342 if (info)
2343 return info->domain;
2344 return NULL;
2345}
2346
5a8f40e8 2347static inline struct device_domain_info *
745f2586
JL		 2348dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2349{
2350 struct device_domain_info *info;
2351
2352 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2353 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2354 info->devfn == devfn)
5a8f40e8 2355 return info;
745f2586
JL		 2356
2357 return NULL;
2358}
2359
5db31569
JR		 2360static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2361 int bus, int devfn,
2362 struct device *dev,
2363 struct dmar_domain *domain)
745f2586 2364{
5a8f40e8 2365 struct dmar_domain *found = NULL;
745f2586
JL		 2366 struct device_domain_info *info;
2367 unsigned long flags;
d160aca5 2368 int ret;
745f2586
JL		 2369
2370 info = alloc_devinfo_mem();
2371 if (!info)
b718cd3d 2372 return NULL;
745f2586 2373
745f2586
JL		 2374 info->bus = bus;
2375 info->devfn = devfn;
b16d0cb9
DW		 2376 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2377 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2378 info->ats_qdep = 0;
745f2586
JL		 2379 info->dev = dev;
2380 info->domain = domain;
5a8f40e8 2381 info->iommu = iommu;
745f2586 2382
b16d0cb9
DW		 2383 if (dev && dev_is_pci(dev)) {
2384 struct pci_dev *pdev = to_pci_dev(info->dev);
2385
2386 if (ecap_dev_iotlb_support(iommu->ecap) &&
2387 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2388 dmar_find_matched_atsr_unit(pdev))
2389 info->ats_supported = 1;
2390
2391 if (ecs_enabled(iommu)) {
2392 if (pasid_enabled(iommu)) {
2393 int features = pci_pasid_features(pdev);
2394 if (features >= 0)
2395 info->pasid_supported = features | 1;
2396 }
2397
2398 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2399 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2400 info->pri_supported = 1;
2401 }
2402 }
2403
745f2586
JL		 2404 spin_lock_irqsave(&device_domain_lock, flags);
2405 if (dev)
0bcb3e28 2406 found = find_domain(dev);
f303e507
JR		 2407
2408 if (!found) {
5a8f40e8 2409 struct device_domain_info *info2;
41e80dca 2410 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
f303e507
JR		 2411 if (info2) {
2412 found = info2->domain;
2413 info2->dev = dev;
2414 }
5a8f40e8 2415 }
f303e507 2416
745f2586
JL		 2417 if (found) {
2418 spin_unlock_irqrestore(&device_domain_lock, flags);
2419 free_devinfo_mem(info);
b718cd3d
DW		 2420 /* Caller must free the original domain */
2421 return found;
745f2586
JL		 2422 }
2423
d160aca5
JR		 2424 spin_lock(&iommu->lock);
2425 ret = domain_attach_iommu(domain, iommu);
2426 spin_unlock(&iommu->lock);
2427
2428 if (ret) {
c6c2cebd 2429 spin_unlock_irqrestore(&device_domain_lock, flags);
499f3aa4 2430 free_devinfo_mem(info);
c6c2cebd
JR		 2431 return NULL;
2432 }
c6c2cebd 2433
b718cd3d
DW		 2434 list_add(&info->link, &domain->devices);
2435 list_add(&info->global, &device_domain_list);
2436 if (dev)
2437 dev->archdata.iommu = info;
2438 spin_unlock_irqrestore(&device_domain_lock, flags);
2439
cc4e2575
JR		 2440 if (dev && domain_context_mapping(domain, dev)) {
2441 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2442 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2443 return NULL;
2444 }
2445
b718cd3d 2446 return domain;
745f2586
JL		 2447}
2448
579305f7
AW		 2449static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2450{
2451 *(u16 *)opaque = alias;
2452 return 0;
2453}
2454
ba395927 2455/* domain is initialized */
146922ec 2456static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2457{
cc4e2575 2458 struct device_domain_info *info = NULL;
579305f7
AW		 2459 struct dmar_domain *domain, *tmp;
2460 struct intel_iommu *iommu;
08a7f456 2461 u16 req_id, dma_alias;
ba395927 2462 unsigned long flags;
aa4d066a 2463 u8 bus, devfn;
ba395927 2464
146922ec 2465 domain = find_domain(dev);
ba395927
KA		 2466 if (domain)
2467 return domain;
2468
579305f7
AW		 2469 iommu = device_to_iommu(dev, &bus, &devfn);
2470 if (!iommu)
2471 return NULL;
2472
08a7f456
JR		 2473 req_id = ((u16)bus << 8) | devfn;
2474
146922ec
DW		 2475 if (dev_is_pci(dev)) {
2476 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2477
579305f7
AW		 2478 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2479
2480 spin_lock_irqsave(&device_domain_lock, flags);
2481 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2482 PCI_BUS_NUM(dma_alias),
2483 dma_alias & 0xff);
2484 if (info) {
2485 iommu = info->iommu;
2486 domain = info->domain;
5a8f40e8 2487 }
579305f7 2488 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2489
579305f7
AW		 2490 /* DMA alias already has a domain, uses it */
2491 if (info)
2492 goto found_domain;
2493 }
ba395927 2494
146922ec 2495 /* Allocate and initialize new domain for the device */
ab8dfe25 2496 domain = alloc_domain(0);
745f2586 2497 if (!domain)
579305f7 2498 return NULL;
dc534b25 2499 if (domain_init(domain, iommu, gaw)) {
579305f7
AW		 2500 domain_exit(domain);
2501 return NULL;
2c2e2c38 2502 }
ba395927 2503
579305f7 2504 /* register PCI DMA alias device */
0b74ecdf 2505 if (dev_is_pci(dev) && req_id != dma_alias) {
5db31569
JR		 2506 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2507 dma_alias & 0xff, NULL, domain);
579305f7
AW		 2508
2509 if (!tmp || tmp != domain) {
2510 domain_exit(domain);
2511 domain = tmp;
2512 }
2513
b718cd3d 2514 if (!domain)
579305f7 2515 return NULL;
ba395927
KA		 2516 }
2517
2518found_domain:
5db31569 2519 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
579305f7
AW		 2520
2521 if (!tmp || tmp != domain) {
2522 domain_exit(domain);
2523 domain = tmp;
2524 }
b718cd3d
DW		 2525
2526 return domain;
ba395927
KA		 2527}
2528
b213203e
DW		 2529static int iommu_domain_identity_map(struct dmar_domain *domain,
2530 unsigned long long start,
2531 unsigned long long end)
ba395927 2532{
c5395d5c
DW		 2533 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2534 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2535
2536 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2537 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2538 pr_err("Reserving iova failed\n");
b213203e 2539 return -ENOMEM;
ba395927
KA		 2540 }
2541
af1089ce 2542 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2543 /*
2544 * RMRR range might have overlap with physical memory range,
2545 * clear it first
2546 */
c5395d5c 2547 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2548
c5395d5c
DW		 2549 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2550 last_vpfn - first_vpfn + 1,
61df7443 2551 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2552}
2553
d66ce54b
JR		 2554static int domain_prepare_identity_map(struct device *dev,
2555 struct dmar_domain *domain,
2556 unsigned long long start,
2557 unsigned long long end)
b213203e 2558{
19943b0e
DW		 2559 /* For _hardware_ passthrough, don't bother. But for software
2560 passthrough, we do it anyway -- it may indicate a memory
2561 range which is reserved in E820, so which didn't get set
2562 up to start with in si_domain */
2563 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2564 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2565 dev_name(dev), start, end);
19943b0e
DW		 2566 return 0;
2567 }
2568
9f10e5bf
JR		 2569 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2570 dev_name(dev), start, end);
2571
5595b528
DW		 2572 if (end < start) {
2573 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2574 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2575 dmi_get_system_info(DMI_BIOS_VENDOR),
2576 dmi_get_system_info(DMI_BIOS_VERSION),
2577 dmi_get_system_info(DMI_PRODUCT_VERSION));
d66ce54b 2578 return -EIO;
5595b528
DW		 2579 }
2580
2ff729f5
DW		 2581 if (end >> agaw_to_width(domain->agaw)) {
2582 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2583 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2584 agaw_to_width(domain->agaw),
2585 dmi_get_system_info(DMI_BIOS_VENDOR),
2586 dmi_get_system_info(DMI_BIOS_VERSION),
2587 dmi_get_system_info(DMI_PRODUCT_VERSION));
d66ce54b 2588 return -EIO;
2ff729f5 2589 }
19943b0e 2590
d66ce54b
JR		 2591 return iommu_domain_identity_map(domain, start, end);
2592}
ba395927 2593
d66ce54b
JR		 2594static int iommu_prepare_identity_map(struct device *dev,
2595 unsigned long long start,
2596 unsigned long long end)
2597{
2598 struct dmar_domain *domain;
2599 int ret;
2600
2601 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2602 if (!domain)
2603 return -ENOMEM;
2604
2605 ret = domain_prepare_identity_map(dev, domain, start, end);
2606 if (ret)
2607 domain_exit(domain);
b213203e 2608
ba395927 2609 return ret;
ba395927
KA		 2610}
2611
2612static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2613 struct device *dev)
ba395927 2614{
0b9d9753 2615 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2616 return 0;
0b9d9753
DW		 2617 return iommu_prepare_identity_map(dev, rmrr->base_address,
2618 rmrr->end_address);
ba395927
KA		 2619}
2620
d3f13810 2621#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2622static inline void iommu_prepare_isa(void)
2623{
2624 struct pci_dev *pdev;
2625 int ret;
2626
2627 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2628 if (!pdev)
2629 return;
2630
9f10e5bf 2631 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2632 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2633
2634 if (ret)
9f10e5bf 2635 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2636
9b27e82d 2637 pci_dev_put(pdev);
49a0429e
KA		 2638}
2639#else
2640static inline void iommu_prepare_isa(void)
2641{
2642 return;
2643}
d3f13810 2644#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2645
2c2e2c38 2646static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2647
071e1374 2648static int __init si_domain_init(int hw)
2c2e2c38 2649{
c7ab48d2 2650 int nid, ret = 0;
2c2e2c38 2651
ab8dfe25 2652 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2653 if (!si_domain)
2654 return -EFAULT;
2655
2c2e2c38
FY		 2656 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2657 domain_exit(si_domain);
2658 return -EFAULT;
2659 }
2660
0dc79715 2661 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2662
19943b0e
DW		 2663 if (hw)
2664 return 0;
2665
c7ab48d2 2666 for_each_online_node(nid) {
5dfe8660
TH		 2667 unsigned long start_pfn, end_pfn;
2668 int i;
2669
2670 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2671 ret = iommu_domain_identity_map(si_domain,
2672 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2673 if (ret)
2674 return ret;
2675 }
c7ab48d2
DW		 2676 }
2677
2c2e2c38
FY		 2678 return 0;
2679}
2680
9b226624 2681static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2682{
2683 struct device_domain_info *info;
2684
2685 if (likely(!iommu_identity_mapping))
2686 return 0;
2687
9b226624 2688 info = dev->archdata.iommu;
cb452a40
MT		 2689 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2690 return (info->domain == si_domain);
2c2e2c38 2691
2c2e2c38
FY		 2692 return 0;
2693}
2694
28ccce0d 2695static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2696{
0ac72664 2697 struct dmar_domain *ndomain;
5a8f40e8 2698 struct intel_iommu *iommu;
156baca8 2699 u8 bus, devfn;
2c2e2c38 2700
5913c9bf 2701 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2702 if (!iommu)
2703 return -ENODEV;
2704
5db31569 2705 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2706 if (ndomain != domain)
2707 return -EBUSY;
2c2e2c38
FY		 2708
2709 return 0;
2710}
2711
0b9d9753 2712static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2713{
2714 struct dmar_rmrr_unit *rmrr;
832bd858 2715 struct device *tmp;
ea2447f7
TM		 2716 int i;
2717
0e242612 2718 rcu_read_lock();
ea2447f7 2719 for_each_rmrr_units(rmrr) {
b683b230
JL		 2720 /*
2721 * Return TRUE if this RMRR contains the device that
2722 * is passed in.
2723 */
2724 for_each_active_dev_scope(rmrr->devices,
2725 rmrr->devices_cnt, i, tmp)
0b9d9753 2726 if (tmp == dev) {
0e242612 2727 rcu_read_unlock();
ea2447f7 2728 return true;
b683b230 2729 }
ea2447f7 2730 }
0e242612 2731 rcu_read_unlock();
ea2447f7
TM		 2732 return false;
2733}
2734
c875d2c1
AW		 2735/*
2736 * There are a couple cases where we need to restrict the functionality of
2737 * devices associated with RMRRs. The first is when evaluating a device for
2738 * identity mapping because problems exist when devices are moved in and out
2739 * of domains and their respective RMRR information is lost. This means that
2740 * a device with associated RMRRs will never be in a "passthrough" domain.
2741 * The second is use of the device through the IOMMU API. This interface
2742 * expects to have full control of the IOVA space for the device. We cannot
2743 * satisfy both the requirement that RMRR access is maintained and have an
2744 * unencumbered IOVA space. We also have no ability to quiesce the device's
2745 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2746 * We therefore prevent devices associated with an RMRR from participating in
2747 * the IOMMU API, which eliminates them from device assignment.
2748 *
2749 * In both cases we assume that PCI USB devices with RMRRs have them largely
2750 * for historical reasons and that the RMRR space is not actively used post
2751 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2752 *
2753 * The same exception is made for graphics devices, with the requirement that
2754 * any use of the RMRR regions will be torn down before assigning the device
2755 * to a guest.
c875d2c1
AW		 2756 */
2757static bool device_is_rmrr_locked(struct device *dev)
2758{
2759 if (!device_has_rmrr(dev))
2760 return false;
2761
2762 if (dev_is_pci(dev)) {
2763 struct pci_dev *pdev = to_pci_dev(dev);
2764
18436afd 2765 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2766 return false;
2767 }
2768
2769 return true;
2770}
2771
3bdb2591 2772static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2773{
ea2447f7 2774
3bdb2591
DW		 2775 if (dev_is_pci(dev)) {
2776 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2777
c875d2c1 2778 if (device_is_rmrr_locked(dev))
3bdb2591 2779 return 0;
e0fc7e0b 2780
3bdb2591
DW		 2781 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2782 return 1;
e0fc7e0b 2783
3bdb2591
DW		 2784 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2785 return 1;
6941af28 2786
3bdb2591 2787 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2788 return 0;
3bdb2591
DW		 2789
2790 /*
2791 * We want to start off with all devices in the 1:1 domain, and
2792 * take them out later if we find they can't access all of memory.
2793 *
2794 * However, we can't do this for PCI devices behind bridges,
2795 * because all PCI devices behind the same bridge will end up
2796 * with the same source-id on their transactions.
2797 *
2798 * Practically speaking, we can't change things around for these
2799 * devices at run-time, because we can't be sure there'll be no
2800 * DMA transactions in flight for any of their siblings.
2801 *
2802 * So PCI devices (unless they're on the root bus) as well as
2803 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2804 * the 1:1 domain, just in _case_ one of their siblings turns out
2805 * not to be able to map all of memory.
2806 */
2807 if (!pci_is_pcie(pdev)) {
2808 if (!pci_is_root_bus(pdev->bus))
2809 return 0;
2810 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2811 return 0;
2812 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2813 return 0;
3bdb2591
DW		 2814 } else {
2815 if (device_has_rmrr(dev))
2816 return 0;
2817 }
3dfc813d 2818
3bdb2591 2819 /*
3dfc813d 2820 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2821 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2822 * take them out of the 1:1 domain later.
2823 */
8fcc5372
CW		 2824 if (!startup) {
2825 /*
2826 * If the device's dma_mask is less than the system's memory
2827 * size then this is not a candidate for identity mapping.
2828 */
3bdb2591 2829 u64 dma_mask = *dev->dma_mask;
8fcc5372 2830
3bdb2591
DW		 2831 if (dev->coherent_dma_mask &&
2832 dev->coherent_dma_mask < dma_mask)
2833 dma_mask = dev->coherent_dma_mask;
8fcc5372 2834
3bdb2591 2835 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2836 }
6941af28
DW		 2837
2838 return 1;
2839}
2840
cf04eee8
DW		 2841static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2842{
2843 int ret;
2844
2845 if (!iommu_should_identity_map(dev, 1))
2846 return 0;
2847
28ccce0d 2848 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2849 if (!ret)
9f10e5bf
JR		 2850 pr_info("%s identity mapping for device %s\n",
2851 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2852 else if (ret == -ENODEV)
2853 /* device not associated with an iommu */
2854 ret = 0;
2855
2856 return ret;
2857}
2858
2859
071e1374 2860static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2861{
2c2e2c38 2862 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2863 struct dmar_drhd_unit *drhd;
2864 struct intel_iommu *iommu;
2865 struct device *dev;
2866 int i;
2867 int ret = 0;
2c2e2c38 2868
2c2e2c38 2869 for_each_pci_dev(pdev) {
cf04eee8
DW		 2870 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2871 if (ret)
2872 return ret;
2873 }
2874
2875 for_each_active_iommu(iommu, drhd)
2876 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2877 struct acpi_device_physical_node *pn;
2878 struct acpi_device *adev;
2879
2880 if (dev->bus != &acpi_bus_type)
2881 continue;
86080ccc 2882
cf04eee8
DW		 2883 adev= to_acpi_device(dev);
2884 mutex_lock(&adev->physical_node_lock);
2885 list_for_each_entry(pn, &adev->physical_node_list, node) {
2886 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2887 if (ret)
2888 break;
eae460b6 2889 }
cf04eee8
DW		 2890 mutex_unlock(&adev->physical_node_lock);
2891 if (ret)
2892 return ret;
62edf5dc 2893 }
2c2e2c38
FY		 2894
2895 return 0;
2896}
2897
ffebeb46
JL		 2898static void intel_iommu_init_qi(struct intel_iommu *iommu)
2899{
2900 /*
2901 * Start from the sane iommu hardware state.
2902 * If the queued invalidation is already initialized by us
2903 * (for example, while enabling interrupt-remapping) then
2904 * we got the things already rolling from a sane state.
2905 */
2906 if (!iommu->qi) {
2907 /*
2908 * Clear any previous faults.
2909 */
2910 dmar_fault(-1, iommu);
2911 /*
2912 * Disable queued invalidation if supported and already enabled
2913 * before OS handover.
2914 */
2915 dmar_disable_qi(iommu);
2916 }
2917
2918 if (dmar_enable_qi(iommu)) {
2919 /*
2920 * Queued Invalidate not enabled, use Register Based Invalidate
2921 */
2922 iommu->flush.flush_context = __iommu_flush_context;
2923 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2924 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2925 iommu->name);
2926 } else {
2927 iommu->flush.flush_context = qi_flush_context;
2928 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2929 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2930 }
2931}
2932
091d42e4 2933static int copy_context_table(struct intel_iommu *iommu,
dfddb969 2934 struct root_entry *old_re,
091d42e4
JR		 2935 struct context_entry **tbl,
2936 int bus, bool ext)
2937{
dbcd861f 2938 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
543c8dcf 2939 struct context_entry *new_ce = NULL, ce;
dfddb969 2940 struct context_entry *old_ce = NULL;
543c8dcf 2941 struct root_entry re;
091d42e4
JR		 2942 phys_addr_t old_ce_phys;
2943
2944 tbl_idx = ext ? bus * 2 : bus;
dfddb969 2945 memcpy(&re, old_re, sizeof(re));
091d42e4
JR		 2946
2947 for (devfn = 0; devfn < 256; devfn++) {
2948 /* First calculate the correct index */
2949 idx = (ext ? devfn * 2 : devfn) % 256;
2950
2951 if (idx == 0) {
2952 /* First save what we may have and clean up */
2953 if (new_ce) {
2954 tbl[tbl_idx] = new_ce;
2955 __iommu_flush_cache(iommu, new_ce,
2956 VTD_PAGE_SIZE);
2957 pos = 1;
2958 }
2959
2960 if (old_ce)
2961 iounmap(old_ce);
2962
2963 ret = 0;
2964 if (devfn < 0x80)
543c8dcf 2965 old_ce_phys = root_entry_lctp(&re);
091d42e4 2966 else
543c8dcf 2967 old_ce_phys = root_entry_uctp(&re);
091d42e4
JR		 2968
2969 if (!old_ce_phys) {
2970 if (ext && devfn == 0) {
2971 /* No LCTP, try UCTP */
2972 devfn = 0x7f;
2973 continue;
2974 } else {
2975 goto out;
2976 }
2977 }
2978
2979 ret = -ENOMEM;
dfddb969
DW		 2980 old_ce = memremap(old_ce_phys, PAGE_SIZE,
2981 MEMREMAP_WB);
091d42e4
JR		 2982 if (!old_ce)
2983 goto out;
2984
2985 new_ce = alloc_pgtable_page(iommu->node);
2986 if (!new_ce)
2987 goto out_unmap;
2988
2989 ret = 0;
2990 }
2991
2992 /* Now copy the context entry */
dfddb969 2993 memcpy(&ce, old_ce + idx, sizeof(ce));
091d42e4 2994
cf484d0e 2995 if (!__context_present(&ce))
091d42e4
JR		 2996 continue;
2997
dbcd861f
JR		 2998 did = context_domain_id(&ce);
2999 if (did >= 0 && did < cap_ndoms(iommu->cap))
3000 set_bit(did, iommu->domain_ids);
3001
cf484d0e
JR		 3002 /*
3003 * We need a marker for copied context entries. This
3004 * marker needs to work for the old format as well as
3005 * for extended context entries.
3006 *
3007 * Bit 67 of the context entry is used. In the old
3008 * format this bit is available to software, in the
3009 * extended format it is the PGE bit, but PGE is ignored
3010 * by HW if PASIDs are disabled (and thus still
3011 * available).
3012 *
3013 * So disable PASIDs first and then mark the entry
3014 * copied. This means that we don't copy PASID
3015 * translations from the old kernel, but this is fine as
3016 * faults there are not fatal.
3017 */
3018 context_clear_pasid_enable(&ce);
3019 context_set_copied(&ce);
3020
091d42e4
JR		 3021 new_ce[idx] = ce;
3022 }
3023
3024 tbl[tbl_idx + pos] = new_ce;
3025
3026 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3027
3028out_unmap:
dfddb969 3029 memunmap(old_ce);
091d42e4
JR		 3030
3031out:
3032 return ret;
3033}
3034
3035static int copy_translation_tables(struct intel_iommu *iommu)
3036{
3037 struct context_entry **ctxt_tbls;
dfddb969 3038 struct root_entry *old_rt;
091d42e4
JR		 3039 phys_addr_t old_rt_phys;
3040 int ctxt_table_entries;
3041 unsigned long flags;
3042 u64 rtaddr_reg;
3043 int bus, ret;
c3361f2f 3044 bool new_ext, ext;
091d42e4
JR		 3045
3046 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3047 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 3048 new_ext = !!ecap_ecs(iommu->ecap);
3049
3050 /*
3051 * The RTT bit can only be changed when translation is disabled,
3052 * but disabling translation means to open a window for data
3053 * corruption. So bail out and don't copy anything if we would
3054 * have to change the bit.
3055 */
3056 if (new_ext != ext)
3057 return -EINVAL;
091d42e4
JR		 3058
3059 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3060 if (!old_rt_phys)
3061 return -EINVAL;
3062
dfddb969 3063 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
091d42e4
JR		 3064 if (!old_rt)
3065 return -ENOMEM;
3066
3067 /* This is too big for the stack - allocate it from slab */
3068 ctxt_table_entries = ext ? 512 : 256;
3069 ret = -ENOMEM;
3070 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3071 if (!ctxt_tbls)
3072 goto out_unmap;
3073
3074 for (bus = 0; bus < 256; bus++) {
3075 ret = copy_context_table(iommu, &old_rt[bus],
3076 ctxt_tbls, bus, ext);
3077 if (ret) {
3078 pr_err("%s: Failed to copy context table for bus %d\n",
3079 iommu->name, bus);
3080 continue;
3081 }
3082 }
3083
3084 spin_lock_irqsave(&iommu->lock, flags);
3085
3086 /* Context tables are copied, now write them to the root_entry table */
3087 for (bus = 0; bus < 256; bus++) {
3088 int idx = ext ? bus * 2 : bus;
3089 u64 val;
3090
3091 if (ctxt_tbls[idx]) {
3092 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3093 iommu->root_entry[bus].lo = val;
3094 }
3095
3096 if (!ext || !ctxt_tbls[idx + 1])
3097 continue;
3098
3099 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3100 iommu->root_entry[bus].hi = val;
3101 }
3102
3103 spin_unlock_irqrestore(&iommu->lock, flags);
3104
3105 kfree(ctxt_tbls);
3106
3107 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3108
3109 ret = 0;
3110
3111out_unmap:
dfddb969 3112 memunmap(old_rt);
091d42e4
JR		 3113
3114 return ret;
3115}
3116
b779260b 3117static int __init init_dmars(void)
ba395927
KA		 3118{
3119 struct dmar_drhd_unit *drhd;
3120 struct dmar_rmrr_unit *rmrr;
a87f4918 3121 bool copied_tables = false;
832bd858 3122 struct device *dev;
ba395927 3123 struct intel_iommu *iommu;
aa473240 3124 int i, ret, cpu;
2c2e2c38 3125
ba395927
KA		 3126 /*
3127 * for each drhd
3128 * allocate root
3129 * initialize and program root entry to not present
3130 * endfor
3131 */
3132 for_each_drhd_unit(drhd) {
5e0d2a6f 3133 /*
3134 * lock not needed as this is only incremented in the single
3135 * threaded kernel __init code path all other access are read
3136 * only
3137 */
78d8e704 3138 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3139 g_num_of_iommus++;
3140 continue;
3141 }
9f10e5bf 3142 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3143 }
3144
ffebeb46
JL		 3145 /* Preallocate enough resources for IOMMU hot-addition */
3146 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3147 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3148
d9630fe9
WH		 3149 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3150 GFP_KERNEL);
3151 if (!g_iommus) {
9f10e5bf 3152 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3153 ret = -ENOMEM;
3154 goto error;
3155 }
3156
aa473240
OP		 3157 for_each_possible_cpu(cpu) {
3158 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3159 cpu);
3160
3161 dfd->tables = kzalloc(g_num_of_iommus *
3162 sizeof(struct deferred_flush_table),
3163 GFP_KERNEL);
3164 if (!dfd->tables) {
3165 ret = -ENOMEM;
3166 goto free_g_iommus;
3167 }
3168
3169 spin_lock_init(&dfd->lock);
3170 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
5e0d2a6f 3171 }
3172
7c919779 3173 for_each_active_iommu(iommu, drhd) {
d9630fe9 3174 g_iommus[iommu->seq_id] = iommu;
ba395927 3175
b63d80d1
JR		 3176 intel_iommu_init_qi(iommu);
3177
e61d98d8
SS		 3178 ret = iommu_init_domains(iommu);
3179 if (ret)
989d51fc 3180 goto free_iommu;
e61d98d8 3181
4158c2ec
JR		 3182 init_translation_status(iommu);
3183
091d42e4
JR		 3184 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3185 iommu_disable_translation(iommu);
3186 clear_translation_pre_enabled(iommu);
3187 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3188 iommu->name);
3189 }
4158c2ec 3190
ba395927
KA		 3191 /*
3192 * TBD:
3193 * we could share the same root & context tables
25985edc 3194 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3195 */
3196 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3197 if (ret)
989d51fc 3198 goto free_iommu;
5f0a7f76 3199
091d42e4
JR		 3200 if (translation_pre_enabled(iommu)) {
3201 pr_info("Translation already enabled - trying to copy translation structures\n");
3202
3203 ret = copy_translation_tables(iommu);
3204 if (ret) {
3205 /*
3206 * We found the IOMMU with translation
3207 * enabled - but failed to copy over the
3208 * old root-entry table. Try to proceed
3209 * by disabling translation now and
3210 * allocating a clean root-entry table.
3211 * This might cause DMAR faults, but
3212 * probably the dump will still succeed.
3213 */
3214 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3215 iommu->name);
3216 iommu_disable_translation(iommu);
3217 clear_translation_pre_enabled(iommu);
3218 } else {
3219 pr_info("Copied translation tables from previous kernel for %s\n",
3220 iommu->name);
a87f4918 3221 copied_tables = true;
091d42e4
JR		 3222 }
3223 }
3224
4ed0d3e6 3225 if (!ecap_pass_through(iommu->ecap))
19943b0e 3226 hw_pass_through = 0;
8a94ade4
DW		 3227#ifdef CONFIG_INTEL_IOMMU_SVM
3228 if (pasid_enabled(iommu))
3229 intel_svm_alloc_pasid_tables(iommu);
3230#endif
ba395927
KA		 3231 }
3232
a4c34ff1
JR		 3233 /*
3234 * Now that qi is enabled on all iommus, set the root entry and flush
3235 * caches. This is required on some Intel X58 chipsets, otherwise the
3236 * flush_context function will loop forever and the boot hangs.
3237 */
3238 for_each_active_iommu(iommu, drhd) {
3239 iommu_flush_write_buffer(iommu);
3240 iommu_set_root_entry(iommu);
3241 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3242 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3243 }
3244
19943b0e 3245 if (iommu_pass_through)
e0fc7e0b
DW		 3246 iommu_identity_mapping |= IDENTMAP_ALL;
3247
d3f13810 3248#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3249 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3250#endif
e0fc7e0b 3251
86080ccc
JR		 3252 if (iommu_identity_mapping) {
3253 ret = si_domain_init(hw_pass_through);
3254 if (ret)
3255 goto free_iommu;
3256 }
3257
e0fc7e0b
DW		 3258 check_tylersburg_isoch();
3259
a87f4918
JR		 3260 /*
3261 * If we copied translations from a previous kernel in the kdump
3262 * case, we can not assign the devices to domains now, as that
3263 * would eliminate the old mappings. So skip this part and defer
3264 * the assignment to device driver initialization time.
3265 */
3266 if (copied_tables)
3267 goto domains_done;
3268
ba395927 3269 /*
19943b0e
DW		 3270 * If pass through is not set or not enabled, setup context entries for
3271 * identity mappings for rmrr, gfx, and isa and may fall back to static
3272 * identity mapping if iommu_identity_mapping is set.
ba395927 3273 */
19943b0e
DW		 3274 if (iommu_identity_mapping) {
3275 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3276 if (ret) {
9f10e5bf 3277 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3278 goto free_iommu;
ba395927
KA		 3279 }
3280 }
ba395927 3281 /*
19943b0e
DW		 3282 * For each rmrr
3283 * for each dev attached to rmrr
3284 * do
3285 * locate drhd for dev, alloc domain for dev
3286 * allocate free domain
3287 * allocate page table entries for rmrr
3288 * if context not allocated for bus
3289 * allocate and init context
3290 * set present in root table for this bus
3291 * init context with domain, translation etc
3292 * endfor
3293 * endfor
ba395927 3294 */
9f10e5bf 3295 pr_info("Setting RMRR:\n");
19943b0e 3296 for_each_rmrr_units(rmrr) {
b683b230
JL		 3297 /* some BIOS lists non-exist devices in DMAR table. */
3298 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3299 i, dev) {
0b9d9753 3300 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3301 if (ret)
9f10e5bf 3302 pr_err("Mapping reserved region failed\n");
ba395927 3303 }
4ed0d3e6 3304 }
49a0429e 3305
19943b0e
DW		 3306 iommu_prepare_isa();
3307
a87f4918
JR		 3308domains_done:
3309
ba395927
KA		 3310 /*
3311 * for each drhd
3312 * enable fault log
3313 * global invalidate context cache
3314 * global invalidate iotlb
3315 * enable translation
3316 */
7c919779 3317 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3318 if (drhd->ignored) {
3319 /*
3320 * we always have to disable PMRs or DMA may fail on
3321 * this device
3322 */
3323 if (force_on)
7c919779 3324 iommu_disable_protect_mem_regions(iommu);
ba395927 3325 continue;
51a63e67 3326 }
ba395927
KA		 3327
3328 iommu_flush_write_buffer(iommu);
3329
a222a7f0
DW		 3330#ifdef CONFIG_INTEL_IOMMU_SVM
3331 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3332 ret = intel_svm_enable_prq(iommu);
3333 if (ret)
3334 goto free_iommu;
3335 }
3336#endif
3460a6d9
KA		 3337 ret = dmar_set_interrupt(iommu);
3338 if (ret)
989d51fc 3339 goto free_iommu;
3460a6d9 3340
8939ddf6
JR		 3341 if (!translation_pre_enabled(iommu))
3342 iommu_enable_translation(iommu);
3343
b94996c9 3344 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3345 }
3346
3347 return 0;
989d51fc
JL		 3348
3349free_iommu:
ffebeb46
JL		 3350 for_each_active_iommu(iommu, drhd) {
3351 disable_dmar_iommu(iommu);
a868e6b7 3352 free_dmar_iommu(iommu);
ffebeb46 3353 }
989d51fc 3354free_g_iommus:
aa473240
OP		 3355 for_each_possible_cpu(cpu)
3356 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
d9630fe9 3357 kfree(g_iommus);
989d51fc 3358error:
ba395927
KA		 3359 return ret;
3360}
3361
5a5e02a6 3362/* This takes a number of _MM_ pages, not VTD pages */
2aac6304 3363static unsigned long intel_alloc_iova(struct device *dev,
875764de
DW		 3364 struct dmar_domain *domain,
3365 unsigned long nrpages, uint64_t dma_mask)
ba395927 3366{
22e2f9fa 3367 unsigned long iova_pfn = 0;
ba395927 3368
875764de
DW		 3369 /* Restrict dma_mask to the width that the iommu can handle */
3370 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
8f6429c7
RM		 3371 /* Ensure we reserve the whole size-aligned region */
3372 nrpages = __roundup_pow_of_two(nrpages);
875764de
DW		 3373
3374 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3375 /*
3376 * First try to allocate an io virtual address in
284901a9 3377 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3378 * from higher range
ba395927 3379 */
22e2f9fa
OP		 3380 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3381 IOVA_PFN(DMA_BIT_MASK(32)));
3382 if (iova_pfn)
3383 return iova_pfn;
875764de 3384 }
22e2f9fa
OP		 3385 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3386 if (unlikely(!iova_pfn)) {
9f10e5bf 3387 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3388 nrpages, dev_name(dev));
2aac6304 3389 return 0;
f76aec76
KA		 3390 }
3391
22e2f9fa 3392 return iova_pfn;
f76aec76
KA		 3393}
3394
d4b709f4 3395static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76 3396{
b1ce5b79 3397 struct dmar_rmrr_unit *rmrr;
f76aec76 3398 struct dmar_domain *domain;
b1ce5b79
JR		 3399 struct device *i_dev;
3400 int i, ret;
f76aec76 3401
d4b709f4 3402 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3403 if (!domain) {
9f10e5bf 3404 pr_err("Allocating domain for %s failed\n",
d4b709f4 3405 dev_name(dev));
4fe05bbc 3406 return NULL;
ba395927
KA		 3407 }
3408
b1ce5b79
JR		 3409 /* We have a new domain - setup possible RMRRs for the device */
3410 rcu_read_lock();
3411 for_each_rmrr_units(rmrr) {
3412 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3413 i, i_dev) {
3414 if (i_dev != dev)
3415 continue;
3416
3417 ret = domain_prepare_identity_map(dev, domain,
3418 rmrr->base_address,
3419 rmrr->end_address);
3420 if (ret)
3421 dev_err(dev, "Mapping reserved region failed\n");
3422 }
3423 }
3424 rcu_read_unlock();
3425
f76aec76
KA		 3426 return domain;
3427}
3428
d4b709f4 3429static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3430{
3431 struct device_domain_info *info;
3432
3433 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3434 info = dev->archdata.iommu;
147202aa
DW		 3435 if (likely(info))
3436 return info->domain;
3437
3438 return __get_valid_domain_for_dev(dev);
3439}
3440
ecb509ec 3441/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3442static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3443{
3444 int found;
3445
3d89194a 3446 if (iommu_dummy(dev))
1e4c64c4
DW		 3447 return 1;
3448
2c2e2c38 3449 if (!iommu_identity_mapping)
1e4c64c4 3450 return 0;
2c2e2c38 3451
9b226624 3452 found = identity_mapping(dev);
2c2e2c38 3453 if (found) {
ecb509ec 3454 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3455 return 1;
3456 else {
3457 /*
3458 * 32 bit DMA is removed from si_domain and fall back
3459 * to non-identity mapping.
3460 */
e6de0f8d 3461 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3462 pr_info("32bit %s uses non-identity mapping\n",
3463 dev_name(dev));
2c2e2c38
FY		 3464 return 0;
3465 }
3466 } else {
3467 /*
3468 * In case of a detached 64 bit DMA device from vm, the device
3469 * is put into si_domain for identity mapping.
3470 */
ecb509ec 3471 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3472 int ret;
28ccce0d 3473 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3474 if (!ret) {
9f10e5bf
JR		 3475 pr_info("64bit %s uses identity mapping\n",
3476 dev_name(dev));
2c2e2c38
FY		 3477 return 1;
3478 }
3479 }
3480 }
3481
1e4c64c4 3482 return 0;
2c2e2c38
FY		 3483}
3484
5040a918 3485static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3486 size_t size, int dir, u64 dma_mask)
f76aec76 3487{
f76aec76 3488 struct dmar_domain *domain;
5b6985ce 3489 phys_addr_t start_paddr;
2aac6304 3490 unsigned long iova_pfn;
f76aec76 3491 int prot = 0;
6865f0d1 3492 int ret;
8c11e798 3493 struct intel_iommu *iommu;
33041ec0 3494 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3495
3496 BUG_ON(dir == DMA_NONE);
2c2e2c38 3497
5040a918 3498 if (iommu_no_mapping(dev))
6865f0d1 3499 return paddr;
f76aec76 3500
5040a918 3501 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3502 if (!domain)
3503 return 0;
3504
8c11e798 3505 iommu = domain_get_iommu(domain);
88cb6a74 3506 size = aligned_nrpages(paddr, size);
f76aec76 3507
2aac6304
OP		 3508 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3509 if (!iova_pfn)
f76aec76
KA		 3510 goto error;
3511
ba395927
KA		 3512 /*
3513 * Check if DMAR supports zero-length reads on write only
3514 * mappings..
3515 */
3516 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3517 !cap_zlr(iommu->cap))
ba395927
KA		 3518 prot |= DMA_PTE_READ;
3519 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3520 prot |= DMA_PTE_WRITE;
3521 /*
6865f0d1 3522 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3523 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3524 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3525 * is not a big problem
3526 */
2aac6304 3527 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
33041ec0 3528 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3529 if (ret)
3530 goto error;
3531
1f0ef2aa
DW		 3532 /* it's a non-present to present mapping. Only flush if caching mode */
3533 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3534 iommu_flush_iotlb_psi(iommu, domain,
2aac6304 3535 mm_to_dma_pfn(iova_pfn),
a1ddcbe9 3536 size, 0, 1);
1f0ef2aa 3537 else
8c11e798 3538 iommu_flush_write_buffer(iommu);
f76aec76 3539
2aac6304 3540 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
03d6a246
DW		 3541 start_paddr += paddr & ~PAGE_MASK;
3542 return start_paddr;
ba395927 3543
ba395927 3544error:
2aac6304 3545 if (iova_pfn)
22e2f9fa 3546 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
9f10e5bf 3547 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3548 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3549 return 0;
3550}
3551
ffbbef5c
FT		 3552static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3553 unsigned long offset, size_t size,
3554 enum dma_data_direction dir,
3555 struct dma_attrs *attrs)
bb9e6d65 3556{
ffbbef5c 3557 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3558 dir, *dev->dma_mask);
bb9e6d65
FT		 3559}
3560
aa473240 3561static void flush_unmaps(struct deferred_flush_data *flush_data)
5e0d2a6f 3562{
80b20dd8 3563 int i, j;
5e0d2a6f 3564
aa473240 3565 flush_data->timer_on = 0;
5e0d2a6f 3566
3567 /* just flush them all */
3568 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459 3569 struct intel_iommu *iommu = g_iommus[i];
aa473240
OP		 3570 struct deferred_flush_table *flush_table =
3571 &flush_data->tables[i];
a2bb8459
WH		 3572 if (!iommu)
3573 continue;
c42d9f32 3574
aa473240 3575 if (!flush_table->next)
9dd2fe89
YZ		 3576 continue;
3577
78d5f0f5
NA		 3578 /* In caching mode, global flushes turn emulation expensive */
3579 if (!cap_caching_mode(iommu->cap))
3580 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3581 DMA_TLB_GLOBAL_FLUSH);
aa473240 3582 for (j = 0; j < flush_table->next; j++) {
93a23a72 3583 unsigned long mask;
314f1dc1 3584 struct deferred_flush_entry *entry =
aa473240 3585 &flush_table->entries[j];
2aac6304 3586 unsigned long iova_pfn = entry->iova_pfn;
769530e4 3587 unsigned long nrpages = entry->nrpages;
314f1dc1
OP		 3588 struct dmar_domain *domain = entry->domain;
3589 struct page *freelist = entry->freelist;
78d5f0f5
NA		 3590
3591 /* On real hardware multiple invalidations are expensive */
3592 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3593 iommu_flush_iotlb_psi(iommu, domain,
2aac6304 3594 mm_to_dma_pfn(iova_pfn),
769530e4 3595 nrpages, !freelist, 0);
78d5f0f5 3596 else {
769530e4 3597 mask = ilog2(nrpages);
314f1dc1 3598 iommu_flush_dev_iotlb(domain,
2aac6304 3599 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
78d5f0f5 3600 }
22e2f9fa 3601 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
314f1dc1
OP		 3602 if (freelist)
3603 dma_free_pagelist(freelist);
80b20dd8 3604 }
aa473240 3605 flush_table->next = 0;
5e0d2a6f 3606 }
3607
aa473240 3608 flush_data->size = 0;
5e0d2a6f 3609}
3610
aa473240 3611static void flush_unmaps_timeout(unsigned long cpuid)
5e0d2a6f 3612{
aa473240 3613 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
80b20dd8 3614 unsigned long flags;
3615
aa473240
OP		 3616 spin_lock_irqsave(&flush_data->lock, flags);
3617 flush_unmaps(flush_data);
3618 spin_unlock_irqrestore(&flush_data->lock, flags);
5e0d2a6f 3619}
3620
2aac6304 3621static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
769530e4 3622 unsigned long nrpages, struct page *freelist)
5e0d2a6f 3623{
3624 unsigned long flags;
314f1dc1 3625 int entry_id, iommu_id;
8c11e798 3626 struct intel_iommu *iommu;
314f1dc1 3627 struct deferred_flush_entry *entry;
aa473240
OP		 3628 struct deferred_flush_data *flush_data;
3629 unsigned int cpuid;
5e0d2a6f 3630
aa473240
OP		 3631 cpuid = get_cpu();
3632 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3633
3634 /* Flush all CPUs' entries to avoid deferring too much. If
3635 * this becomes a bottleneck, can just flush us, and rely on
3636 * flush timer for the rest.
3637 */
3638 if (flush_data->size == HIGH_WATER_MARK) {
3639 int cpu;
3640
3641 for_each_online_cpu(cpu)
3642 flush_unmaps_timeout(cpu);
3643 }
3644
3645 spin_lock_irqsave(&flush_data->lock, flags);
80b20dd8 3646
8c11e798
WH		 3647 iommu = domain_get_iommu(dom);
3648 iommu_id = iommu->seq_id;
c42d9f32 3649
aa473240
OP		 3650 entry_id = flush_data->tables[iommu_id].next;
3651 ++(flush_data->tables[iommu_id].next);
5e0d2a6f 3652
aa473240 3653 entry = &flush_data->tables[iommu_id].entries[entry_id];
314f1dc1 3654 entry->domain = dom;
2aac6304 3655 entry->iova_pfn = iova_pfn;
769530e4 3656 entry->nrpages = nrpages;
314f1dc1 3657 entry->freelist = freelist;
5e0d2a6f 3658
aa473240
OP		 3659 if (!flush_data->timer_on) {
3660 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3661 flush_data->timer_on = 1;
5e0d2a6f 3662 }
aa473240
OP		 3663 flush_data->size++;
3664 spin_unlock_irqrestore(&flush_data->lock, flags);
3665
3666 put_cpu();
5e0d2a6f 3667}
3668
769530e4 3669static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
ba395927 3670{
f76aec76 3671 struct dmar_domain *domain;
d794dc9b 3672 unsigned long start_pfn, last_pfn;
769530e4 3673 unsigned long nrpages;
2aac6304 3674 unsigned long iova_pfn;
8c11e798 3675 struct intel_iommu *iommu;
ea8ea460 3676 struct page *freelist;
ba395927 3677
73676832 3678 if (iommu_no_mapping(dev))
f76aec76 3679 return;
2c2e2c38 3680
1525a29a 3681 domain = find_domain(dev);
ba395927
KA		 3682 BUG_ON(!domain);
3683
8c11e798
WH		 3684 iommu = domain_get_iommu(domain);
3685
2aac6304 3686 iova_pfn = IOVA_PFN(dev_addr);
ba395927 3687
769530e4 3688 nrpages = aligned_nrpages(dev_addr, size);
2aac6304 3689 start_pfn = mm_to_dma_pfn(iova_pfn);
769530e4 3690 last_pfn = start_pfn + nrpages - 1;
ba395927 3691
d794dc9b 3692 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3693 dev_name(dev), start_pfn, last_pfn);
ba395927 3694
ea8ea460 3695 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3696
5e0d2a6f 3697 if (intel_iommu_strict) {
a1ddcbe9 3698 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
769530e4 3699 nrpages, !freelist, 0);
5e0d2a6f 3700 /* free iova */
22e2f9fa 3701 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
ea8ea460 3702 dma_free_pagelist(freelist);
5e0d2a6f 3703 } else {
2aac6304 3704 add_unmap(domain, iova_pfn, nrpages, freelist);
5e0d2a6f 3705 /*
3706 * queue up the release of the unmap to save the 1/6th of the
3707 * cpu used up by the iotlb flush operation...
3708 */
5e0d2a6f 3709 }
ba395927
KA		 3710}
3711
d41a4adb
JL		 3712static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3713 size_t size, enum dma_data_direction dir,
3714 struct dma_attrs *attrs)
3715{
769530e4 3716 intel_unmap(dev, dev_addr, size);
d41a4adb
JL		 3717}
3718
5040a918 3719static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3720 dma_addr_t *dma_handle, gfp_t flags,
3721 struct dma_attrs *attrs)
ba395927 3722{
36746436 3723 struct page *page = NULL;
ba395927
KA		 3724 int order;
3725
5b6985ce 3726 size = PAGE_ALIGN(size);
ba395927 3727 order = get_order(size);
e8bb910d 3728
5040a918 3729 if (!iommu_no_mapping(dev))
e8bb910d 3730 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3731 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3732 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3733 flags |= GFP_DMA;
3734 else
3735 flags |= GFP_DMA32;
3736 }
ba395927 3737
d0164adc 3738 if (gfpflags_allow_blocking(flags)) {
36746436
AM		 3739 unsigned int count = size >> PAGE_SHIFT;
3740
3741 page = dma_alloc_from_contiguous(dev, count, order);
3742 if (page && iommu_no_mapping(dev) &&
3743 page_to_phys(page) + size > dev->coherent_dma_mask) {
3744 dma_release_from_contiguous(dev, page, count);
3745 page = NULL;
3746 }
3747 }
3748
3749 if (!page)
3750 page = alloc_pages(flags, order);
3751 if (!page)
ba395927 3752 return NULL;
36746436 3753 memset(page_address(page), 0, size);
ba395927 3754
36746436 3755 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3756 DMA_BIDIRECTIONAL,
5040a918 3757 dev->coherent_dma_mask);
ba395927 3758 if (*dma_handle)
36746436
AM		 3759 return page_address(page);
3760 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3761 __free_pages(page, order);
3762
ba395927
KA		 3763 return NULL;
3764}
3765
5040a918 3766static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3767 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3768{
3769 int order;
36746436 3770 struct page *page = virt_to_page(vaddr);
ba395927 3771
5b6985ce 3772 size = PAGE_ALIGN(size);
ba395927
KA		 3773 order = get_order(size);
3774
769530e4 3775 intel_unmap(dev, dma_handle, size);
36746436
AM		 3776 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3777 __free_pages(page, order);
ba395927
KA		 3778}
3779
5040a918 3780static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3781 int nelems, enum dma_data_direction dir,
3782 struct dma_attrs *attrs)
ba395927 3783{
769530e4
OP		 3784 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3785 unsigned long nrpages = 0;
3786 struct scatterlist *sg;
3787 int i;
3788
3789 for_each_sg(sglist, sg, nelems, i) {
3790 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3791 }
3792
3793 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
ba395927
KA		 3794}
3795
ba395927 3796static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3797 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3798{
3799 int i;
c03ab37c 3800 struct scatterlist *sg;
ba395927 3801
c03ab37c 3802 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3803 BUG_ON(!sg_page(sg));
3e6110fd 3804 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3805 sg->dma_length = sg->length;
ba395927
KA		 3806 }
3807 return nelems;
3808}
3809
5040a918 3810static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3811 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3812{
ba395927 3813 int i;
ba395927 3814 struct dmar_domain *domain;
f76aec76
KA		 3815 size_t size = 0;
3816 int prot = 0;
2aac6304 3817 unsigned long iova_pfn;
f76aec76 3818 int ret;
c03ab37c 3819 struct scatterlist *sg;
b536d24d 3820 unsigned long start_vpfn;
8c11e798 3821 struct intel_iommu *iommu;
ba395927
KA		 3822
3823 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3824 if (iommu_no_mapping(dev))
3825 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3826
5040a918 3827 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3828 if (!domain)
3829 return 0;
3830
8c11e798
WH		 3831 iommu = domain_get_iommu(domain);
3832
b536d24d 3833 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3834 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3835
2aac6304 3836 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
5040a918 3837 *dev->dma_mask);
2aac6304 3838 if (!iova_pfn) {
c03ab37c 3839 sglist->dma_length = 0;
f76aec76
KA		 3840 return 0;
3841 }
3842
3843 /*
3844 * Check if DMAR supports zero-length reads on write only
3845 * mappings..
3846 */
3847 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3848 !cap_zlr(iommu->cap))
f76aec76
KA		 3849 prot |= DMA_PTE_READ;
3850 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3851 prot |= DMA_PTE_WRITE;
3852
2aac6304 3853 start_vpfn = mm_to_dma_pfn(iova_pfn);
e1605495 3854
f532959b 3855 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3856 if (unlikely(ret)) {
e1605495
DW		 3857 dma_pte_free_pagetable(domain, start_vpfn,
3858 start_vpfn + size - 1);
22e2f9fa 3859 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
e1605495 3860 return 0;
ba395927
KA		 3861 }
3862
1f0ef2aa
DW		 3863 /* it's a non-present to present mapping. Only flush if caching mode */
3864 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3865 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3866 else
8c11e798 3867 iommu_flush_write_buffer(iommu);
1f0ef2aa 3868
ba395927
KA		 3869 return nelems;
3870}
3871
dfb805e8
FT		 3872static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3873{
3874 return !dma_addr;
3875}
3876
160c1d8e 3877struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3878 .alloc = intel_alloc_coherent,
3879 .free = intel_free_coherent,
ba395927
KA		 3880 .map_sg = intel_map_sg,
3881 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3882 .map_page = intel_map_page,
3883 .unmap_page = intel_unmap_page,
dfb805e8 3884 .mapping_error = intel_mapping_error,
ba395927
KA		 3885};
3886
3887static inline int iommu_domain_cache_init(void)
3888{
3889 int ret = 0;
3890
3891 iommu_domain_cache = kmem_cache_create("iommu_domain",
3892 sizeof(struct dmar_domain),
3893 0,
3894 SLAB_HWCACHE_ALIGN,
3895
3896 NULL);
3897 if (!iommu_domain_cache) {
9f10e5bf 3898 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3899 ret = -ENOMEM;
3900 }
3901
3902 return ret;
3903}
3904
3905static inline int iommu_devinfo_cache_init(void)
3906{
3907 int ret = 0;
3908
3909 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3910 sizeof(struct device_domain_info),
3911 0,
3912 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3913 NULL);
3914 if (!iommu_devinfo_cache) {
9f10e5bf 3915 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3916 ret = -ENOMEM;
3917 }
3918
3919 return ret;
3920}
3921
ba395927
KA		 3922static int __init iommu_init_mempool(void)
3923{
3924 int ret;
ae1ff3d6 3925 ret = iova_cache_get();
ba395927
KA		 3926 if (ret)
3927 return ret;
3928
3929 ret = iommu_domain_cache_init();
3930 if (ret)
3931 goto domain_error;
3932
3933 ret = iommu_devinfo_cache_init();
3934 if (!ret)
3935 return ret;
3936
3937 kmem_cache_destroy(iommu_domain_cache);
3938domain_error:
ae1ff3d6 3939 iova_cache_put();
ba395927
KA		 3940
3941 return -ENOMEM;
3942}
3943
3944static void __init iommu_exit_mempool(void)
3945{
3946 kmem_cache_destroy(iommu_devinfo_cache);
3947 kmem_cache_destroy(iommu_domain_cache);
ae1ff3d6 3948 iova_cache_put();
ba395927
KA		 3949}
3950
556ab45f
DW		 3951static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3952{
3953 struct dmar_drhd_unit *drhd;
3954 u32 vtbar;
3955 int rc;
3956
3957 /* We know that this device on this chipset has its own IOMMU.
3958 * If we find it under a different IOMMU, then the BIOS is lying
3959 * to us. Hope that the IOMMU for this device is actually
3960 * disabled, and it needs no translation...
3961 */
3962 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3963 if (rc) {
3964 /* "can't" happen */
3965 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3966 return;
3967 }
3968 vtbar &= 0xffff0000;
3969
3970 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3971 drhd = dmar_find_matched_drhd_unit(pdev);
3972 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3973 TAINT_FIRMWARE_WORKAROUND,
3974 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3975 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3976}
3977DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3978
ba395927
KA		 3979static void __init init_no_remapping_devices(void)
3980{
3981 struct dmar_drhd_unit *drhd;
832bd858 3982 struct device *dev;
b683b230 3983 int i;
ba395927
KA		 3984
3985 for_each_drhd_unit(drhd) {
3986 if (!drhd->include_all) {
b683b230
JL		 3987 for_each_active_dev_scope(drhd->devices,
3988 drhd->devices_cnt, i, dev)
3989 break;
832bd858 3990 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3991 if (i == drhd->devices_cnt)
3992 drhd->ignored = 1;
3993 }
3994 }
3995
7c919779 3996 for_each_active_drhd_unit(drhd) {
7c919779 3997 if (drhd->include_all)
ba395927
KA		 3998 continue;
3999
b683b230
JL		 4000 for_each_active_dev_scope(drhd->devices,
4001 drhd->devices_cnt, i, dev)
832bd858 4002 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 4003 break;
ba395927
KA		 4004 if (i < drhd->devices_cnt)
4005 continue;
4006
c0771df8
DW		 4007 /* This IOMMU has *only* gfx devices. Either bypass it or
4008 set the gfx_mapped flag, as appropriate */
4009 if (dmar_map_gfx) {
4010 intel_iommu_gfx_mapped = 1;
4011 } else {
4012 drhd->ignored = 1;
b683b230
JL		 4013 for_each_active_dev_scope(drhd->devices,
4014 drhd->devices_cnt, i, dev)
832bd858 4015 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 4016 }
4017 }
4018}
4019
f59c7b69
FY		 4020#ifdef CONFIG_SUSPEND
4021static int init_iommu_hw(void)
4022{
4023 struct dmar_drhd_unit *drhd;
4024 struct intel_iommu *iommu = NULL;
4025
4026 for_each_active_iommu(iommu, drhd)
4027 if (iommu->qi)
4028 dmar_reenable_qi(iommu);
4029
b779260b
JC		 4030 for_each_iommu(iommu, drhd) {
4031 if (drhd->ignored) {
4032 /*
4033 * we always have to disable PMRs or DMA may fail on
4034 * this device
4035 */
4036 if (force_on)
4037 iommu_disable_protect_mem_regions(iommu);
4038 continue;
4039 }
4040
f59c7b69
FY		 4041 iommu_flush_write_buffer(iommu);
4042
4043 iommu_set_root_entry(iommu);
4044
4045 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 4046 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 4047 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4048 iommu_enable_translation(iommu);
b94996c9 4049 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 4050 }
4051
4052 return 0;
4053}
4054
4055static void iommu_flush_all(void)
4056{
4057 struct dmar_drhd_unit *drhd;
4058 struct intel_iommu *iommu;
4059
4060 for_each_active_iommu(iommu, drhd) {
4061 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 4062 DMA_CCMD_GLOBAL_INVL);
f59c7b69 4063 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 4064 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 4065 }
4066}
4067
134fac3f 4068static int iommu_suspend(void)
f59c7b69
FY		 4069{
4070 struct dmar_drhd_unit *drhd;
4071 struct intel_iommu *iommu = NULL;
4072 unsigned long flag;
4073
4074 for_each_active_iommu(iommu, drhd) {
4075 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4076 GFP_ATOMIC);
4077 if (!iommu->iommu_state)
4078 goto nomem;
4079 }
4080
4081 iommu_flush_all();
4082
4083 for_each_active_iommu(iommu, drhd) {
4084 iommu_disable_translation(iommu);
4085
1f5b3c3f 4086 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 4087
4088 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4089 readl(iommu->reg + DMAR_FECTL_REG);
4090 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4091 readl(iommu->reg + DMAR_FEDATA_REG);
4092 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4093 readl(iommu->reg + DMAR_FEADDR_REG);
4094 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4095 readl(iommu->reg + DMAR_FEUADDR_REG);
4096
1f5b3c3f 4097 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 4098 }
4099 return 0;
4100
4101nomem:
4102 for_each_active_iommu(iommu, drhd)
4103 kfree(iommu->iommu_state);
4104
4105 return -ENOMEM;
4106}
4107
134fac3f 4108static void iommu_resume(void)
f59c7b69
FY		 4109{
4110 struct dmar_drhd_unit *drhd;
4111 struct intel_iommu *iommu = NULL;
4112 unsigned long flag;
4113
4114 if (init_iommu_hw()) {
b779260b
JC		 4115 if (force_on)
4116 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4117 else
4118 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 4119 return;
f59c7b69
FY		 4120 }
4121
4122 for_each_active_iommu(iommu, drhd) {
4123
1f5b3c3f 4124 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 4125
4126 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4127 iommu->reg + DMAR_FECTL_REG);
4128 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4129 iommu->reg + DMAR_FEDATA_REG);
4130 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4131 iommu->reg + DMAR_FEADDR_REG);
4132 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4133 iommu->reg + DMAR_FEUADDR_REG);
4134
1f5b3c3f 4135 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 4136 }
4137
4138 for_each_active_iommu(iommu, drhd)
4139 kfree(iommu->iommu_state);
f59c7b69
FY		 4140}
4141
134fac3f 4142static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 4143 .resume = iommu_resume,
4144 .suspend = iommu_suspend,
4145};
4146
134fac3f 4147static void __init init_iommu_pm_ops(void)
f59c7b69 4148{
134fac3f 4149 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 4150}
4151
4152#else
99592ba4 4153static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 4154#endif /* CONFIG_PM */
4155
318fe7df 4156
c2a0b538 4157int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 4158{
4159 struct acpi_dmar_reserved_memory *rmrr;
4160 struct dmar_rmrr_unit *rmrru;
4161
4162 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4163 if (!rmrru)
4164 return -ENOMEM;
4165
4166 rmrru->hdr = header;
4167 rmrr = (struct acpi_dmar_reserved_memory *)header;
4168 rmrru->base_address = rmrr->base_address;
4169 rmrru->end_address = rmrr->end_address;
2e455289
JL		 4170 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4171 ((void *)rmrr) + rmrr->header.length,
4172 &rmrru->devices_cnt);
4173 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4174 kfree(rmrru);
4175 return -ENOMEM;
4176 }
318fe7df 4177
2e455289 4178 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4179
2e455289 4180 return 0;
318fe7df
SS		 4181}
4182
6b197249
JL		 4183static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4184{
4185 struct dmar_atsr_unit *atsru;
4186 struct acpi_dmar_atsr *tmp;
4187
4188 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4189 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4190 if (atsr->segment != tmp->segment)
4191 continue;
4192 if (atsr->header.length != tmp->header.length)
4193 continue;
4194 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4195 return atsru;
4196 }
4197
4198 return NULL;
4199}
4200
4201int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4202{
4203 struct acpi_dmar_atsr *atsr;
4204 struct dmar_atsr_unit *atsru;
4205
6b197249
JL		 4206 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4207 return 0;
4208
318fe7df 4209 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4210 atsru = dmar_find_atsr(atsr);
4211 if (atsru)
4212 return 0;
4213
4214 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4215 if (!atsru)
4216 return -ENOMEM;
4217
6b197249
JL		 4218 /*
4219 * If memory is allocated from slab by ACPI _DSM method, we need to
4220 * copy the memory content because the memory buffer will be freed
4221 * on return.
4222 */
4223 atsru->hdr = (void *)(atsru + 1);
4224 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4225 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4226 if (!atsru->include_all) {
4227 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4228 (void *)atsr + atsr->header.length,
4229 &atsru->devices_cnt);
4230 if (atsru->devices_cnt && atsru->devices == NULL) {
4231 kfree(atsru);
4232 return -ENOMEM;
4233 }
4234 }
318fe7df 4235
0e242612 4236 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4237
4238 return 0;
4239}
4240
9bdc531e
JL		 4241static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4242{
4243 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4244 kfree(atsru);
4245}
4246
6b197249
JL		 4247int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4248{
4249 struct acpi_dmar_atsr *atsr;
4250 struct dmar_atsr_unit *atsru;
4251
4252 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4253 atsru = dmar_find_atsr(atsr);
4254 if (atsru) {
4255 list_del_rcu(&atsru->list);
4256 synchronize_rcu();
4257 intel_iommu_free_atsr(atsru);
4258 }
4259
4260 return 0;
4261}
4262
4263int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4264{
4265 int i;
4266 struct device *dev;
4267 struct acpi_dmar_atsr *atsr;
4268 struct dmar_atsr_unit *atsru;
4269
4270 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4271 atsru = dmar_find_atsr(atsr);
4272 if (!atsru)
4273 return 0;
4274
194dc870 4275 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
6b197249
JL		 4276 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4277 i, dev)
4278 return -EBUSY;
194dc870 4279 }
6b197249
JL		 4280
4281 return 0;
4282}
4283
ffebeb46
JL		 4284static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4285{
4286 int sp, ret = 0;
4287 struct intel_iommu *iommu = dmaru->iommu;
4288
4289 if (g_iommus[iommu->seq_id])
4290 return 0;
4291
4292 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4293 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4294 iommu->name);
4295 return -ENXIO;
4296 }
4297 if (!ecap_sc_support(iommu->ecap) &&
4298 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4299 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4300 iommu->name);
4301 return -ENXIO;
4302 }
4303 sp = domain_update_iommu_superpage(iommu) - 1;
4304 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4305 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4306 iommu->name);
4307 return -ENXIO;
4308 }
4309
4310 /*
4311 * Disable translation if already enabled prior to OS handover.
4312 */
4313 if (iommu->gcmd & DMA_GCMD_TE)
4314 iommu_disable_translation(iommu);
4315
4316 g_iommus[iommu->seq_id] = iommu;
4317 ret = iommu_init_domains(iommu);
4318 if (ret == 0)
4319 ret = iommu_alloc_root_entry(iommu);
4320 if (ret)
4321 goto out;
4322
8a94ade4
DW		 4323#ifdef CONFIG_INTEL_IOMMU_SVM
4324 if (pasid_enabled(iommu))
4325 intel_svm_alloc_pasid_tables(iommu);
4326#endif
4327
ffebeb46
JL		 4328 if (dmaru->ignored) {
4329 /*
4330 * we always have to disable PMRs or DMA may fail on this device
4331 */
4332 if (force_on)
4333 iommu_disable_protect_mem_regions(iommu);
4334 return 0;
4335 }
4336
4337 intel_iommu_init_qi(iommu);
4338 iommu_flush_write_buffer(iommu);
a222a7f0
DW		 4339
4340#ifdef CONFIG_INTEL_IOMMU_SVM
4341 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4342 ret = intel_svm_enable_prq(iommu);
4343 if (ret)
4344 goto disable_iommu;
4345 }
4346#endif
ffebeb46
JL		 4347 ret = dmar_set_interrupt(iommu);
4348 if (ret)
4349 goto disable_iommu;
4350
4351 iommu_set_root_entry(iommu);
4352 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4353 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4354 iommu_enable_translation(iommu);
4355
ffebeb46
JL		 4356 iommu_disable_protect_mem_regions(iommu);
4357 return 0;
4358
4359disable_iommu:
4360 disable_dmar_iommu(iommu);
4361out:
4362 free_dmar_iommu(iommu);
4363 return ret;
4364}
4365
6b197249
JL		 4366int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4367{
ffebeb46
JL		 4368 int ret = 0;
4369 struct intel_iommu *iommu = dmaru->iommu;
4370
4371 if (!intel_iommu_enabled)
4372 return 0;
4373 if (iommu == NULL)
4374 return -EINVAL;
4375
4376 if (insert) {
4377 ret = intel_iommu_add(dmaru);
4378 } else {
4379 disable_dmar_iommu(iommu);
4380 free_dmar_iommu(iommu);
4381 }
4382
4383 return ret;
6b197249
JL		 4384}
4385
9bdc531e
JL		 4386static void intel_iommu_free_dmars(void)
4387{
4388 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4389 struct dmar_atsr_unit *atsru, *atsr_n;
4390
4391 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4392 list_del(&rmrru->list);
4393 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4394 kfree(rmrru);
318fe7df
SS		 4395 }
4396
9bdc531e
JL		 4397 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4398 list_del(&atsru->list);
4399 intel_iommu_free_atsr(atsru);
4400 }
318fe7df
SS		 4401}
4402
4403int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4404{
b683b230 4405 int i, ret = 1;
318fe7df 4406 struct pci_bus *bus;
832bd858
DW		 4407 struct pci_dev *bridge = NULL;
4408 struct device *tmp;
318fe7df
SS		 4409 struct acpi_dmar_atsr *atsr;
4410 struct dmar_atsr_unit *atsru;
4411
4412 dev = pci_physfn(dev);
318fe7df 4413 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4414 bridge = bus->self;
d14053b3
DW		 4415 /* If it's an integrated device, allow ATS */
4416 if (!bridge)
4417 return 1;
4418 /* Connected via non-PCIe: no ATS */
4419 if (!pci_is_pcie(bridge) ||
62f87c0e 4420 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4421 return 0;
d14053b3 4422 /* If we found the root port, look it up in the ATSR */
b5f82ddf 4423 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4424 break;
318fe7df
SS		 4425 }
4426
0e242612 4427 rcu_read_lock();
b5f82ddf
JL		 4428 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4429 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4430 if (atsr->segment != pci_domain_nr(dev->bus))
4431 continue;
4432
b683b230 4433 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4434 if (tmp == &bridge->dev)
b683b230 4435 goto out;
b5f82ddf
JL		 4436
4437 if (atsru->include_all)
b683b230 4438 goto out;
b5f82ddf 4439 }
b683b230
JL		 4440 ret = 0;
4441out:
0e242612 4442 rcu_read_unlock();
318fe7df 4443
b683b230 4444 return ret;
318fe7df
SS		 4445}
4446
59ce0515
JL		 4447int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4448{
4449 int ret = 0;
4450 struct dmar_rmrr_unit *rmrru;
4451 struct dmar_atsr_unit *atsru;
4452 struct acpi_dmar_atsr *atsr;
4453 struct acpi_dmar_reserved_memory *rmrr;
4454
4455 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4456 return 0;
4457
4458 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4459 rmrr = container_of(rmrru->hdr,
4460 struct acpi_dmar_reserved_memory, header);
4461 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4462 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4463 ((void *)rmrr) + rmrr->header.length,
4464 rmrr->segment, rmrru->devices,
4465 rmrru->devices_cnt);
27e24950 4466 if(ret < 0)
59ce0515 4467 return ret;
e6a8c9b3 4468 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
27e24950
JL		 4469 dmar_remove_dev_scope(info, rmrr->segment,
4470 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4471 }
4472 }
4473
4474 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4475 if (atsru->include_all)
4476 continue;
4477
4478 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4479 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4480 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4481 (void *)atsr + atsr->header.length,
4482 atsr->segment, atsru->devices,
4483 atsru->devices_cnt);
4484 if (ret > 0)
4485 break;
4486 else if(ret < 0)
4487 return ret;
e6a8c9b3 4488 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
59ce0515
JL		 4489 if (dmar_remove_dev_scope(info, atsr->segment,
4490 atsru->devices, atsru->devices_cnt))
4491 break;
4492 }
4493 }
4494
4495 return 0;
4496}
4497
99dcaded
FY		 4498/*
4499 * Here we only respond to action of unbound device from driver.
4500 *
4501 * Added device is not attached to its DMAR domain here yet. That will happen
4502 * when mapping the device to iova.
4503 */
4504static int device_notifier(struct notifier_block *nb,
4505 unsigned long action, void *data)
4506{
4507 struct device *dev = data;
99dcaded
FY		 4508 struct dmar_domain *domain;
4509
3d89194a 4510 if (iommu_dummy(dev))
44cd613c
DW		 4511 return 0;
4512
1196c2fb 4513 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4514 return 0;
4515
1525a29a 4516 domain = find_domain(dev);
99dcaded
FY		 4517 if (!domain)
4518 return 0;
4519
e6de0f8d 4520 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4521 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4522 domain_exit(domain);
a97590e5 4523
99dcaded
FY		 4524 return 0;
4525}
4526
4527static struct notifier_block device_nb = {
4528 .notifier_call = device_notifier,
4529};
4530
75f05569
JL		 4531static int intel_iommu_memory_notifier(struct notifier_block *nb,
4532 unsigned long val, void *v)
4533{
4534 struct memory_notify *mhp = v;
4535 unsigned long long start, end;
4536 unsigned long start_vpfn, last_vpfn;
4537
4538 switch (val) {
4539 case MEM_GOING_ONLINE:
4540 start = mhp->start_pfn << PAGE_SHIFT;
4541 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4542 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4543 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4544 start, end);
4545 return NOTIFY_BAD;
4546 }
4547 break;
4548
4549 case MEM_OFFLINE:
4550 case MEM_CANCEL_ONLINE:
4551 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4552 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4553 while (start_vpfn <= last_vpfn) {
4554 struct iova *iova;
4555 struct dmar_drhd_unit *drhd;
4556 struct intel_iommu *iommu;
ea8ea460 4557 struct page *freelist;
75f05569
JL		 4558
4559 iova = find_iova(&si_domain->iovad, start_vpfn);
4560 if (iova == NULL) {
9f10e5bf 4561 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4562 start_vpfn);
4563 break;
4564 }
4565
4566 iova = split_and_remove_iova(&si_domain->iovad, iova,
4567 start_vpfn, last_vpfn);
4568 if (iova == NULL) {
9f10e5bf 4569 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4570 start_vpfn, last_vpfn);
4571 return NOTIFY_BAD;
4572 }
4573
ea8ea460
DW		 4574 freelist = domain_unmap(si_domain, iova->pfn_lo,
4575 iova->pfn_hi);
4576
75f05569
JL		 4577 rcu_read_lock();
4578 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4579 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4580 iova->pfn_lo, iova_size(iova),
ea8ea460 4581 !freelist, 0);
75f05569 4582 rcu_read_unlock();
ea8ea460 4583 dma_free_pagelist(freelist);
75f05569
JL		 4584
4585 start_vpfn = iova->pfn_hi + 1;
4586 free_iova_mem(iova);
4587 }
4588 break;
4589 }
4590
4591 return NOTIFY_OK;
4592}
4593
4594static struct notifier_block intel_iommu_memory_nb = {
4595 .notifier_call = intel_iommu_memory_notifier,
4596 .priority = 0
4597};
4598
22e2f9fa
OP		 4599static void free_all_cpu_cached_iovas(unsigned int cpu)
4600{
4601 int i;
4602
4603 for (i = 0; i < g_num_of_iommus; i++) {
4604 struct intel_iommu *iommu = g_iommus[i];
4605 struct dmar_domain *domain;
0caa7616 4606 int did;
22e2f9fa
OP		 4607
4608 if (!iommu)
4609 continue;
4610
3bd4f911 4611 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
0caa7616 4612 domain = get_iommu_domain(iommu, (u16)did);
22e2f9fa
OP		 4613
4614 if (!domain)
4615 continue;
4616 free_cpu_cached_iovas(cpu, &domain->iovad);
4617 }
4618 }
4619}
4620
aa473240
OP		 4621static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4622 unsigned long action, void *v)
4623{
4624 unsigned int cpu = (unsigned long)v;
4625
4626 switch (action) {
4627 case CPU_DEAD:
4628 case CPU_DEAD_FROZEN:
22e2f9fa 4629 free_all_cpu_cached_iovas(cpu);
aa473240
OP		 4630 flush_unmaps_timeout(cpu);
4631 break;
4632 }
4633 return NOTIFY_OK;
4634}
4635
4636static struct notifier_block intel_iommu_cpu_nb = {
4637 .notifier_call = intel_iommu_cpu_notifier,
4638};
a5459cfe
AW		 4639
4640static ssize_t intel_iommu_show_version(struct device *dev,
4641 struct device_attribute *attr,
4642 char *buf)
4643{
4644 struct intel_iommu *iommu = dev_get_drvdata(dev);
4645 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4646 return sprintf(buf, "%d:%d\n",
4647 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4648}
4649static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4650
4651static ssize_t intel_iommu_show_address(struct device *dev,
4652 struct device_attribute *attr,
4653 char *buf)
4654{
4655 struct intel_iommu *iommu = dev_get_drvdata(dev);
4656 return sprintf(buf, "%llx\n", iommu->reg_phys);
4657}
4658static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4659
4660static ssize_t intel_iommu_show_cap(struct device *dev,
4661 struct device_attribute *attr,
4662 char *buf)
4663{
4664 struct intel_iommu *iommu = dev_get_drvdata(dev);
4665 return sprintf(buf, "%llx\n", iommu->cap);
4666}
4667static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4668
4669static ssize_t intel_iommu_show_ecap(struct device *dev,
4670 struct device_attribute *attr,
4671 char *buf)
4672{
4673 struct intel_iommu *iommu = dev_get_drvdata(dev);
4674 return sprintf(buf, "%llx\n", iommu->ecap);
4675}
4676static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4677
2238c082
AW		 4678static ssize_t intel_iommu_show_ndoms(struct device *dev,
4679 struct device_attribute *attr,
4680 char *buf)
4681{
4682 struct intel_iommu *iommu = dev_get_drvdata(dev);
4683 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4684}
4685static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4686
4687static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4688 struct device_attribute *attr,
4689 char *buf)
4690{
4691 struct intel_iommu *iommu = dev_get_drvdata(dev);
4692 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4693 cap_ndoms(iommu->cap)));
4694}
4695static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4696
a5459cfe
AW		 4697static struct attribute *intel_iommu_attrs[] = {
4698 &dev_attr_version.attr,
4699 &dev_attr_address.attr,
4700 &dev_attr_cap.attr,
4701 &dev_attr_ecap.attr,
2238c082
AW		 4702 &dev_attr_domains_supported.attr,
4703 &dev_attr_domains_used.attr,
a5459cfe
AW		 4704 NULL,
4705};
4706
4707static struct attribute_group intel_iommu_group = {
4708 .name = "intel-iommu",
4709 .attrs = intel_iommu_attrs,
4710};
4711
4712const struct attribute_group *intel_iommu_groups[] = {
4713 &intel_iommu_group,
4714 NULL,
4715};
4716
ba395927
KA		 4717int __init intel_iommu_init(void)
4718{
9bdc531e 4719 int ret = -ENODEV;
3a93c841 4720 struct dmar_drhd_unit *drhd;
7c919779 4721 struct intel_iommu *iommu;
ba395927 4722
a59b50e9
JC		 4723 /* VT-d is required for a TXT/tboot launch, so enforce that */
4724 force_on = tboot_force_iommu();
4725
3a5670e8
JL		 4726 if (iommu_init_mempool()) {
4727 if (force_on)
4728 panic("tboot: Failed to initialize iommu memory\n");
4729 return -ENOMEM;
4730 }
4731
4732 down_write(&dmar_global_lock);
a59b50e9
JC		 4733 if (dmar_table_init()) {
4734 if (force_on)
4735 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4736 goto out_free_dmar;
a59b50e9 4737 }
ba395927 4738
c2c7286a 4739 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4740 if (force_on)
4741 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4742 goto out_free_dmar;
a59b50e9 4743 }
1886e8a9 4744
75f1cdf1 4745 if (no_iommu || dmar_disabled)
9bdc531e 4746 goto out_free_dmar;
2ae21010 4747
318fe7df 4748 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4749 pr_info("No RMRR found\n");
318fe7df
SS		 4750
4751 if (list_empty(&dmar_atsr_units))
9f10e5bf 4752 pr_info("No ATSR found\n");
318fe7df 4753
51a63e67
JC		 4754 if (dmar_init_reserved_ranges()) {
4755 if (force_on)
4756 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4757 goto out_free_reserved_range;
51a63e67 4758 }
ba395927
KA		 4759
4760 init_no_remapping_devices();
4761
b779260b 4762 ret = init_dmars();
ba395927 4763 if (ret) {
a59b50e9
JC		 4764 if (force_on)
4765 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4766 pr_err("Initialization failed\n");
9bdc531e 4767 goto out_free_reserved_range;
ba395927 4768 }
3a5670e8 4769 up_write(&dmar_global_lock);
9f10e5bf 4770 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4771
75f1cdf1
FT		 4772#ifdef CONFIG_SWIOTLB
4773 swiotlb = 0;
4774#endif
19943b0e 4775 dma_ops = &intel_dma_ops;
4ed0d3e6 4776
134fac3f 4777 init_iommu_pm_ops();
a8bcbb0d 4778
a5459cfe
AW		 4779 for_each_active_iommu(iommu, drhd)
4780 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4781 intel_iommu_groups,
2439d4aa 4782 "%s", iommu->name);
a5459cfe 4783
4236d97d 4784 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4785 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4786 if (si_domain && !hw_pass_through)
4787 register_memory_notifier(&intel_iommu_memory_nb);
aa473240 4788 register_hotcpu_notifier(&intel_iommu_cpu_nb);
99dcaded 4789
8bc1f85c
ED		 4790 intel_iommu_enabled = 1;
4791
ba395927 4792 return 0;
9bdc531e
JL		 4793
4794out_free_reserved_range:
4795 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4796out_free_dmar:
4797 intel_iommu_free_dmars();
3a5670e8
JL		 4798 up_write(&dmar_global_lock);
4799 iommu_exit_mempool();
9bdc531e 4800 return ret;
ba395927 4801}
e820482c 4802
2452d9db 4803static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
579305f7
AW		 4804{
4805 struct intel_iommu *iommu = opaque;
4806
2452d9db 4807 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 4808 return 0;
4809}
4810
4811/*
4812 * NB - intel-iommu lacks any sort of reference counting for the users of
4813 * dependent devices. If multiple endpoints have intersecting dependent
4814 * devices, unbinding the driver from any one of them will possibly leave
4815 * the others unable to operate.
4816 */
2452d9db 4817static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
3199aa6b 4818{
0bcb3e28 4819 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4820 return;
4821
2452d9db 4822 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
3199aa6b
HW		 4823}
4824
127c7615 4825static void __dmar_remove_one_dev_info(struct device_domain_info *info)
c7151a8d 4826{
c7151a8d
WH		 4827 struct intel_iommu *iommu;
4828 unsigned long flags;
c7151a8d 4829
55d94043
JR		 4830 assert_spin_locked(&device_domain_lock);
4831
127c7615 4832 if (WARN_ON(!info))
c7151a8d
WH		 4833 return;
4834
127c7615 4835 iommu = info->iommu;
c7151a8d 4836
127c7615
JR		 4837 if (info->dev) {
4838 iommu_disable_dev_iotlb(info);
4839 domain_context_clear(iommu, info->dev);
4840 }
c7151a8d 4841
b608ac3b 4842 unlink_domain_info(info);
c7151a8d 4843
d160aca5 4844 spin_lock_irqsave(&iommu->lock, flags);
127c7615 4845 domain_detach_iommu(info->domain, iommu);
d160aca5 4846 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 4847
127c7615 4848 free_devinfo_mem(info);
c7151a8d 4849}
c7151a8d 4850
55d94043
JR		 4851static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4852 struct device *dev)
4853{
127c7615 4854 struct device_domain_info *info;
55d94043 4855 unsigned long flags;
3e7abe25 4856
55d94043 4857 spin_lock_irqsave(&device_domain_lock, flags);
127c7615
JR		 4858 info = dev->archdata.iommu;
4859 __dmar_remove_one_dev_info(info);
55d94043 4860 spin_unlock_irqrestore(&device_domain_lock, flags);
c7151a8d
WH		 4861}
4862
2c2e2c38 4863static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4864{
4865 int adjust_width;
4866
0fb5fe87
RM		 4867 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4868 DMA_32BIT_PFN);
5e98c4b1
WH		 4869 domain_reserve_special_ranges(domain);
4870
4871 /* calculate AGAW */
4872 domain->gaw = guest_width;
4873 adjust_width = guestwidth_to_adjustwidth(guest_width);
4874 domain->agaw = width_to_agaw(adjust_width);
4875
5e98c4b1 4876 domain->iommu_coherency = 0;
c5b15255 4877 domain->iommu_snooping = 0;
6dd9a7c7 4878 domain->iommu_superpage = 0;
fe40f1e0 4879 domain->max_addr = 0;
5e98c4b1
WH		 4880
4881 /* always allocate the top pgd */
4c923d47 4882 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4883 if (!domain->pgd)
4884 return -ENOMEM;
4885 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4886 return 0;
4887}
4888
00a77deb 4889static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4890{
5d450806 4891 struct dmar_domain *dmar_domain;
00a77deb
JR		 4892 struct iommu_domain *domain;
4893
4894 if (type != IOMMU_DOMAIN_UNMANAGED)
4895 return NULL;
38717946 4896
ab8dfe25 4897 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4898 if (!dmar_domain) {
9f10e5bf 4899 pr_err("Can't allocate dmar_domain\n");
00a77deb 4900 return NULL;
38717946 4901 }
2c2e2c38 4902 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4903 pr_err("Domain initialization failed\n");
92d03cc8 4904 domain_exit(dmar_domain);
00a77deb 4905 return NULL;
38717946 4906 }
8140a95d 4907 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4908
00a77deb 4909 domain = &dmar_domain->domain;
8a0e715b
JR		 4910 domain->geometry.aperture_start = 0;
4911 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4912 domain->geometry.force_aperture = true;
4913
00a77deb 4914 return domain;
38717946 4915}
38717946 4916
00a77deb 4917static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4918{
00a77deb 4919 domain_exit(to_dmar_domain(domain));
38717946 4920}
38717946 4921
4c5478c9
JR		 4922static int intel_iommu_attach_device(struct iommu_domain *domain,
4923 struct device *dev)
38717946 4924{
00a77deb 4925 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4926 struct intel_iommu *iommu;
4927 int addr_width;
156baca8 4928 u8 bus, devfn;
faa3d6f5 4929
c875d2c1
AW		 4930 if (device_is_rmrr_locked(dev)) {
4931 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4932 return -EPERM;
4933 }
4934
7207d8f9
DW		 4935 /* normally dev is not mapped */
4936 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4937 struct dmar_domain *old_domain;
4938
1525a29a 4939 old_domain = find_domain(dev);
faa3d6f5 4940 if (old_domain) {
d160aca5 4941 rcu_read_lock();
de7e8886 4942 dmar_remove_one_dev_info(old_domain, dev);
d160aca5 4943 rcu_read_unlock();
62c22167
JR		 4944
4945 if (!domain_type_is_vm_or_si(old_domain) &&
4946 list_empty(&old_domain->devices))
4947 domain_exit(old_domain);
faa3d6f5
WH		 4948 }
4949 }
4950
156baca8 4951 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4952 if (!iommu)
4953 return -ENODEV;
4954
4955 /* check if this iommu agaw is sufficient for max mapped address */
4956 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4957 if (addr_width > cap_mgaw(iommu->cap))
4958 addr_width = cap_mgaw(iommu->cap);
4959
4960 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4961 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4962 "sufficient for the mapped address (%llx)\n",
a99c47a2 4963 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4964 return -EFAULT;
4965 }
a99c47a2
TL		 4966 dmar_domain->gaw = addr_width;
4967
4968 /*
4969 * Knock out extra levels of page tables if necessary
4970 */
4971 while (iommu->agaw < dmar_domain->agaw) {
4972 struct dma_pte *pte;
4973
4974 pte = dmar_domain->pgd;
4975 if (dma_pte_present(pte)) {
25cbff16
SY		 4976 dmar_domain->pgd = (struct dma_pte *)
4977 phys_to_virt(dma_pte_addr(pte));
7a661013 4978 free_pgtable_page(pte);
a99c47a2
TL		 4979 }
4980 dmar_domain->agaw--;
4981 }
fe40f1e0 4982
28ccce0d 4983 return domain_add_dev_info(dmar_domain, dev);
38717946 4984}
38717946 4985
4c5478c9
JR		 4986static void intel_iommu_detach_device(struct iommu_domain *domain,
4987 struct device *dev)
38717946 4988{
e6de0f8d 4989 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4990}
c7151a8d 4991
b146a1c9
JR		 4992static int intel_iommu_map(struct iommu_domain *domain,
4993 unsigned long iova, phys_addr_t hpa,
5009065d 4994 size_t size, int iommu_prot)
faa3d6f5 4995{
00a77deb 4996 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4997 u64 max_addr;
dde57a21 4998 int prot = 0;
faa3d6f5 4999 int ret;
fe40f1e0 5000
dde57a21
JR		 5001 if (iommu_prot & IOMMU_READ)
5002 prot |= DMA_PTE_READ;
5003 if (iommu_prot & IOMMU_WRITE)
5004 prot |= DMA_PTE_WRITE;
9cf06697
SY		 5005 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5006 prot |= DMA_PTE_SNP;
dde57a21 5007
163cc52c 5008 max_addr = iova + size;
dde57a21 5009 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 5010 u64 end;
5011
5012 /* check if minimum agaw is sufficient for mapped address */
8954da1f 5013 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 5014 if (end < max_addr) {
9f10e5bf 5015 pr_err("%s: iommu width (%d) is not "
fe40f1e0 5016 "sufficient for the mapped address (%llx)\n",
8954da1f 5017 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 5018 return -EFAULT;
5019 }
dde57a21 5020 dmar_domain->max_addr = max_addr;
fe40f1e0 5021 }
ad051221
DW		 5022 /* Round up size to next multiple of PAGE_SIZE, if it and
5023 the low bits of hpa would take us onto the next page */
88cb6a74 5024 size = aligned_nrpages(hpa, size);
ad051221
DW		 5025 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5026 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 5027 return ret;
38717946 5028}
38717946 5029
5009065d 5030static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 5031 unsigned long iova, size_t size)
38717946 5032{
00a77deb 5033 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 5034 struct page *freelist = NULL;
5035 struct intel_iommu *iommu;
5036 unsigned long start_pfn, last_pfn;
5037 unsigned int npages;
42e8c186 5038 int iommu_id, level = 0;
5cf0a76f
DW		 5039
5040 /* Cope with horrid API which requires us to unmap more than the
5041 size argument if it happens to be a large-page mapping. */
dc02e46e 5042 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5cf0a76f
DW		 5043
5044 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5045 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 5046
ea8ea460
DW		 5047 start_pfn = iova >> VTD_PAGE_SHIFT;
5048 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5049
5050 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5051
5052 npages = last_pfn - start_pfn + 1;
5053
29a27719 5054 for_each_domain_iommu(iommu_id, dmar_domain) {
a1ddcbe9 5055 iommu = g_iommus[iommu_id];
ea8ea460 5056
42e8c186
JR		 5057 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5058 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 5059 }
5060
5061 dma_free_pagelist(freelist);
fe40f1e0 5062
163cc52c
DW		 5063 if (dmar_domain->max_addr == iova + size)
5064 dmar_domain->max_addr = iova;
b146a1c9 5065
5cf0a76f 5066 return size;
38717946 5067}
38717946 5068
d14d6577 5069static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 5070 dma_addr_t iova)
38717946 5071{
00a77deb 5072 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 5073 struct dma_pte *pte;
5cf0a76f 5074 int level = 0;
faa3d6f5 5075 u64 phys = 0;
38717946 5076
5cf0a76f 5077 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 5078 if (pte)
faa3d6f5 5079 phys = dma_pte_addr(pte);
38717946 5080
faa3d6f5 5081 return phys;
38717946 5082}
a8bcbb0d 5083
5d587b8d 5084static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 5085{
dbb9fd86 5086 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 5087 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 5088 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 5089 return irq_remapping_enabled == 1;
dbb9fd86 5090
5d587b8d 5091 return false;
dbb9fd86
SY		 5092}
5093
abdfdde2
AW		 5094static int intel_iommu_add_device(struct device *dev)
5095{
a5459cfe 5096 struct intel_iommu *iommu;
abdfdde2 5097 struct iommu_group *group;
156baca8 5098 u8 bus, devfn;
70ae6f0d 5099
a5459cfe
AW		 5100 iommu = device_to_iommu(dev, &bus, &devfn);
5101 if (!iommu)
70ae6f0d
AW		 5102 return -ENODEV;
5103
a5459cfe 5104 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 5105
e17f9ff4 5106 group = iommu_group_get_for_dev(dev);
783f157b 5107
e17f9ff4
AW		 5108 if (IS_ERR(group))
5109 return PTR_ERR(group);
bcb71abe 5110
abdfdde2 5111 iommu_group_put(group);
e17f9ff4 5112 return 0;
abdfdde2 5113}
70ae6f0d 5114
abdfdde2
AW		 5115static void intel_iommu_remove_device(struct device *dev)
5116{
a5459cfe
AW		 5117 struct intel_iommu *iommu;
5118 u8 bus, devfn;
5119
5120 iommu = device_to_iommu(dev, &bus, &devfn);
5121 if (!iommu)
5122 return;
5123
abdfdde2 5124 iommu_group_remove_device(dev);
a5459cfe
AW		 5125
5126 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 5127}
5128
2f26e0a9
DW		 5129#ifdef CONFIG_INTEL_IOMMU_SVM
5130int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5131{
5132 struct device_domain_info *info;
5133 struct context_entry *context;
5134 struct dmar_domain *domain;
5135 unsigned long flags;
5136 u64 ctx_lo;
5137 int ret;
5138
5139 domain = get_valid_domain_for_dev(sdev->dev);
5140 if (!domain)
5141 return -EINVAL;
5142
5143 spin_lock_irqsave(&device_domain_lock, flags);
5144 spin_lock(&iommu->lock);
5145
5146 ret = -EINVAL;
5147 info = sdev->dev->archdata.iommu;
5148 if (!info || !info->pasid_supported)
5149 goto out;
5150
5151 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5152 if (WARN_ON(!context))
5153 goto out;
5154
5155 ctx_lo = context[0].lo;
5156
5157 sdev->did = domain->iommu_did[iommu->seq_id];
5158 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5159
5160 if (!(ctx_lo & CONTEXT_PASIDE)) {
5161 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5162 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap);
5163 wmb();
5164 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5165 * extended to permit requests-with-PASID if the PASIDE bit
5166 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5167 * however, the PASIDE bit is ignored and requests-with-PASID
5168 * are unconditionally blocked. Which makes less sense.
5169 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5170 * "guest mode" translation types depending on whether ATS
5171 * is available or not. Annoyingly, we can't use the new
5172 * modes *unless* PASIDE is set. */
5173 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5174 ctx_lo &= ~CONTEXT_TT_MASK;
5175 if (info->ats_supported)
5176 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5177 else
5178 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5179 }
5180 ctx_lo |= CONTEXT_PASIDE;
907fea34
DW		 5181 if (iommu->pasid_state_table)
5182 ctx_lo |= CONTEXT_DINVE;
a222a7f0
DW		 5183 if (info->pri_supported)
5184 ctx_lo |= CONTEXT_PRS;
2f26e0a9
DW		 5185 context[0].lo = ctx_lo;
5186 wmb();
5187 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5188 DMA_CCMD_MASK_NOBIT,
5189 DMA_CCMD_DEVICE_INVL);
5190 }
5191
5192 /* Enable PASID support in the device, if it wasn't already */
5193 if (!info->pasid_enabled)
5194 iommu_enable_dev_iotlb(info);
5195
5196 if (info->ats_enabled) {
5197 sdev->dev_iotlb = 1;
5198 sdev->qdep = info->ats_qdep;
5199 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5200 sdev->qdep = 0;
5201 }
5202 ret = 0;
5203
5204 out:
5205 spin_unlock(&iommu->lock);
5206 spin_unlock_irqrestore(&device_domain_lock, flags);
5207
5208 return ret;
5209}
5210
5211struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5212{
5213 struct intel_iommu *iommu;
5214 u8 bus, devfn;
5215
5216 if (iommu_dummy(dev)) {
5217 dev_warn(dev,
5218 "No IOMMU translation for device; cannot enable SVM\n");
5219 return NULL;
5220 }
5221
5222 iommu = device_to_iommu(dev, &bus, &devfn);
5223 if ((!iommu)) {
b9997e38 5224 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
2f26e0a9
DW		 5225 return NULL;
5226 }
5227
5228 if (!iommu->pasid_table) {
b9997e38 5229 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
2f26e0a9
DW		 5230 return NULL;
5231 }
5232
5233 return iommu;
5234}
5235#endif /* CONFIG_INTEL_IOMMU_SVM */
5236
b22f6434 5237static const struct iommu_ops intel_iommu_ops = {
5d587b8d 5238 .capable = intel_iommu_capable,
00a77deb
JR		 5239 .domain_alloc = intel_iommu_domain_alloc,
5240 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 5241 .attach_dev = intel_iommu_attach_device,
5242 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 5243 .map = intel_iommu_map,
5244 .unmap = intel_iommu_unmap,
315786eb 5245 .map_sg = default_iommu_map_sg,
a8bcbb0d 5246 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 5247 .add_device = intel_iommu_add_device,
5248 .remove_device = intel_iommu_remove_device,
a960fadb 5249 .device_group = pci_device_group,
6d1c56a9 5250 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 5251};
9af88143 5252
9452618e
DV		 5253static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5254{
5255 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 5256 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 5257 dmar_map_gfx = 0;
5258}
5259
5260DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5261DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5262DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5263DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5264DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5265DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5266DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5267
d34d6517 5268static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 5269{
5270 /*
5271 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 5272 * but needs it. Same seems to hold for the desktop versions.
9af88143 5273 */
9f10e5bf 5274 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 5275 rwbf_quirk = 1;
5276}
5277
5278DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 5279DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5280DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5281DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5282DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5283DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5284DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 5285
eecfd57f
AJ		 5286#define GGC 0x52
5287#define GGC_MEMORY_SIZE_MASK (0xf << 8)
5288#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5289#define GGC_MEMORY_SIZE_1M (0x1 << 8)
5290#define GGC_MEMORY_SIZE_2M (0x3 << 8)
5291#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5292#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5293#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5294#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5295
d34d6517 5296static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 5297{
5298 unsigned short ggc;
5299
eecfd57f 5300 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 5301 return;
5302
eecfd57f 5303 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 5304 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 5305 dmar_map_gfx = 0;
6fbcfb3e
DW		 5306 } else if (dmar_map_gfx) {
5307 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 5308 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 5309 intel_iommu_strict = 1;
5310 }
9eecabcb
DW		 5311}
5312DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5313DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5314DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5315DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5316
e0fc7e0b
DW		 5317/* On Tylersburg chipsets, some BIOSes have been known to enable the
5318 ISOCH DMAR unit for the Azalia sound device, but not give it any
5319 TLB entries, which causes it to deadlock. Check for that. We do
5320 this in a function called from init_dmars(), instead of in a PCI
5321 quirk, because we don't want to print the obnoxious "BIOS broken"
5322 message if VT-d is actually disabled.
5323*/
5324static void __init check_tylersburg_isoch(void)
5325{
5326 struct pci_dev *pdev;
5327 uint32_t vtisochctrl;
5328
5329 /* If there's no Azalia in the system anyway, forget it. */
5330 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5331 if (!pdev)
5332 return;
5333 pci_dev_put(pdev);
5334
5335 /* System Management Registers. Might be hidden, in which case
5336 we can't do the sanity check. But that's OK, because the
5337 known-broken BIOSes _don't_ actually hide it, so far. */
5338 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5339 if (!pdev)
5340 return;
5341
5342 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5343 pci_dev_put(pdev);
5344 return;
5345 }
5346
5347 pci_dev_put(pdev);
5348
5349 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5350 if (vtisochctrl & 1)
5351 return;
5352
5353 /* Drop all bits other than the number of TLB entries */
5354 vtisochctrl &= 0x1c;
5355
5356 /* If we have the recommended number of TLB entries (16), fine. */
5357 if (vtisochctrl == 0x10)
5358 return;
5359
5360 /* Zero TLB entries? You get to ride the short bus to school. */
5361 if (!vtisochctrl) {
5362 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5363 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5364 dmi_get_system_info(DMI_BIOS_VENDOR),
5365 dmi_get_system_info(DMI_BIOS_VERSION),
5366 dmi_get_system_info(DMI_PRODUCT_VERSION));
5367 iommu_identity_mapping |= IDENTMAP_AZALIA;
5368 return;
5369 }
9f10e5bf
JR		 5370
5371 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5372 vtisochctrl);
5373}
Linux 6.x block layer and io_uring tree(s)