projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mmc-v4.7-rc1' of git://git.linaro.org/people/ulf.hansson/mmc
[linux-2.6-block.git] / drivers / iommu / dmar.c
CommitLineData
10e5247f
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
10e5247f 21 *
e61d98d8 22 * This file implements early detection/parsing of Remapping Devices
10e5247f
KA		 23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
e61d98d8
SS		 25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
10e5247f
KA		 27 */
28
9f10e5bf 29#define pr_fmt(fmt) "DMAR: " fmt
e9071b0b 30
10e5247f
KA		 31#include <linux/pci.h>
32#include <linux/dmar.h>
38717946
KA		 33#include <linux/iova.h>
34#include <linux/intel-iommu.h>
fe962e90 35#include <linux/timer.h>
0ac2491f
SS		 36#include <linux/irq.h>
37#include <linux/interrupt.h>
69575d38 38#include <linux/tboot.h>
eb27cae8 39#include <linux/dmi.h>
5a0e3ad6 40#include <linux/slab.h>
a5459cfe 41#include <linux/iommu.h>
8a8f422d 42#include <asm/irq_remapping.h>
4db77ff3 43#include <asm/iommu_table.h>
10e5247f 44
078e1ee2
JR		 45#include "irq_remapping.h"
46
c2a0b538
JL		 47typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
48struct dmar_res_callback {
49 dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
50 void *arg[ACPI_DMAR_TYPE_RESERVED];
51 bool ignore_unhandled;
52 bool print_entry;
53};
54
3a5670e8
JL		 55/*
56 * Assumptions:
57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
58 * before IO devices managed by that unit.
59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
60 * after IO devices managed by that unit.
61 * 3) Hotplug events are rare.
62 *
63 * Locking rules for DMA and interrupt remapping related global data structures:
64 * 1) Use dmar_global_lock in process context
65 * 2) Use RCU in interrupt context
10e5247f 66 */
3a5670e8 67DECLARE_RWSEM(dmar_global_lock);
10e5247f 68LIST_HEAD(dmar_drhd_units);
10e5247f 69
41750d31 70struct acpi_table_header * __initdata dmar_tbl;
8e1568f3 71static acpi_size dmar_tbl_size;
2e455289 72static int dmar_dev_scope_status = 1;
78d8e704 73static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
10e5247f 74
694835dc 75static int alloc_iommu(struct dmar_drhd_unit *drhd);
a868e6b7 76static void free_iommu(struct intel_iommu *iommu);
694835dc 77
6b197249 78static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
10e5247f
KA		 79{
80 /*
81 * add INCLUDE_ALL at the tail, so scan the list will find it at
82 * the very end.
83 */
84 if (drhd->include_all)
0e242612 85 list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
10e5247f 86 else
0e242612 87 list_add_rcu(&drhd->list, &dmar_drhd_units);
10e5247f
KA		 88}
89
bb3a6b78 90void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
10e5247f
KA		 91{
92 struct acpi_dmar_device_scope *scope;
10e5247f
KA		 93
94 *cnt = 0;
95 while (start < end) {
96 scope = start;
83118b0d 97 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
07cb52ff 98 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
10e5247f
KA		 99 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
100 (*cnt)++;
ae3e7f3a
LC		 101 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
102 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
e9071b0b 103 pr_warn("Unsupported device scope\n");
5715f0f9 104 }
10e5247f
KA		 105 start += scope->length;
106 }
107 if (*cnt == 0)
bb3a6b78
JL		 108 return NULL;
109
832bd858 110 return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
bb3a6b78
JL		 111}
112
832bd858 113void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
ada4d4b2 114{
b683b230 115 int i;
832bd858 116 struct device *tmp_dev;
b683b230 117
ada4d4b2 118 if (*devices && *cnt) {
b683b230 119 for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
832bd858 120 put_device(tmp_dev);
ada4d4b2 121 kfree(*devices);
ada4d4b2 122 }
0e242612
JL		 123
124 *devices = NULL;
125 *cnt = 0;
ada4d4b2
JL		 126}
127
59ce0515
JL		 128/* Optimize out kzalloc()/kfree() for normal cases */
129static char dmar_pci_notify_info_buf[64];
130
131static struct dmar_pci_notify_info *
132dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
133{
134 int level = 0;
135 size_t size;
136 struct pci_dev *tmp;
137 struct dmar_pci_notify_info *info;
138
139 BUG_ON(dev->is_virtfn);
140
141 /* Only generate path[] for device addition event */
142 if (event == BUS_NOTIFY_ADD_DEVICE)
143 for (tmp = dev; tmp; tmp = tmp->bus->self)
144 level++;
145
146 size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
147 if (size <= sizeof(dmar_pci_notify_info_buf)) {
148 info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
149 } else {
150 info = kzalloc(size, GFP_KERNEL);
151 if (!info) {
152 pr_warn("Out of memory when allocating notify_info "
153 "for %s.\n", pci_name(dev));
2e455289
JL		 154 if (dmar_dev_scope_status == 0)
155 dmar_dev_scope_status = -ENOMEM;
59ce0515
JL		 156 return NULL;
157 }
158 }
159
160 info->event = event;
161 info->dev = dev;
162 info->seg = pci_domain_nr(dev->bus);
163 info->level = level;
164 if (event == BUS_NOTIFY_ADD_DEVICE) {
5ae0566a
JL		 165 for (tmp = dev; tmp; tmp = tmp->bus->self) {
166 level--;
57384592 167 info->path[level].bus = tmp->bus->number;
59ce0515
JL		 168 info->path[level].device = PCI_SLOT(tmp->devfn);
169 info->path[level].function = PCI_FUNC(tmp->devfn);
170 if (pci_is_root_bus(tmp->bus))
171 info->bus = tmp->bus->number;
172 }
173 }
174
175 return info;
176}
177
178static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
179{
180 if ((void *)info != dmar_pci_notify_info_buf)
181 kfree(info);
182}
183
184static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
185 struct acpi_dmar_pci_path *path, int count)
186{
187 int i;
188
189 if (info->bus != bus)
80f7b3d1 190 goto fallback;
59ce0515 191 if (info->level != count)
80f7b3d1 192 goto fallback;
59ce0515
JL		 193
194 for (i = 0; i < count; i++) {
195 if (path[i].device != info->path[i].device ||
196 path[i].function != info->path[i].function)
80f7b3d1 197 goto fallback;
59ce0515
JL		 198 }
199
200 return true;
80f7b3d1
JR		 201
202fallback:
203
204 if (count != 1)
205 return false;
206
207 i = info->level - 1;
208 if (bus == info->path[i].bus &&
209 path[0].device == info->path[i].device &&
210 path[0].function == info->path[i].function) {
211 pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
212 bus, path[0].device, path[0].function);
213 return true;
214 }
215
216 return false;
59ce0515
JL		 217}
218
219/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
220int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
221 void *start, void*end, u16 segment,
832bd858
DW		 222 struct dmar_dev_scope *devices,
223 int devices_cnt)
59ce0515
JL		 224{
225 int i, level;
832bd858 226 struct device *tmp, *dev = &info->dev->dev;
59ce0515
JL		 227 struct acpi_dmar_device_scope *scope;
228 struct acpi_dmar_pci_path *path;
229
230 if (segment != info->seg)
231 return 0;
232
233 for (; start < end; start += scope->length) {
234 scope = start;
235 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
236 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
237 continue;
238
239 path = (struct acpi_dmar_pci_path *)(scope + 1);
240 level = (scope->length - sizeof(*scope)) / sizeof(*path);
241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue;
243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
832bd858 245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
59ce0515 246 pr_warn("Device scope type does not match for %s\n",
832bd858 247 pci_name(info->dev));
59ce0515
JL		 248 return -EINVAL;
249 }
250
251 for_each_dev_scope(devices, devices_cnt, i, tmp)
252 if (tmp == NULL) {
832bd858
DW		 253 devices[i].bus = info->dev->bus->number;
254 devices[i].devfn = info->dev->devfn;
255 rcu_assign_pointer(devices[i].dev,
256 get_device(dev));
59ce0515
JL		 257 return 1;
258 }
259 BUG_ON(i >= devices_cnt);
260 }
261
262 return 0;
263}
264
265int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
832bd858 266 struct dmar_dev_scope *devices, int count)
59ce0515
JL		 267{
268 int index;
832bd858 269 struct device *tmp;
59ce0515
JL		 270
271 if (info->seg != segment)
272 return 0;
273
274 for_each_active_dev_scope(devices, count, index, tmp)
832bd858 275 if (tmp == &info->dev->dev) {
eecbad7d 276 RCU_INIT_POINTER(devices[index].dev, NULL);
59ce0515 277 synchronize_rcu();
832bd858 278 put_device(tmp);
59ce0515
JL		 279 return 1;
280 }
281
282 return 0;
283}
284
285static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
286{
287 int ret = 0;
288 struct dmar_drhd_unit *dmaru;
289 struct acpi_dmar_hardware_unit *drhd;
290
291 for_each_drhd_unit(dmaru) {
292 if (dmaru->include_all)
293 continue;
294
295 drhd = container_of(dmaru->hdr,
296 struct acpi_dmar_hardware_unit, header);
297 ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
298 ((void *)drhd) + drhd->header.length,
299 dmaru->segment,
300 dmaru->devices, dmaru->devices_cnt);
301 if (ret != 0)
302 break;
303 }
304 if (ret >= 0)
305 ret = dmar_iommu_notify_scope_dev(info);
2e455289
JL		 306 if (ret < 0 && dmar_dev_scope_status == 0)
307 dmar_dev_scope_status = ret;
59ce0515
JL		 308
309 return ret;
310}
311
312static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
313{
314 struct dmar_drhd_unit *dmaru;
315
316 for_each_drhd_unit(dmaru)
317 if (dmar_remove_dev_scope(info, dmaru->segment,
318 dmaru->devices, dmaru->devices_cnt))
319 break;
320 dmar_iommu_notify_scope_dev(info);
321}
322
323static int dmar_pci_bus_notifier(struct notifier_block *nb,
324 unsigned long action, void *data)
325{
326 struct pci_dev *pdev = to_pci_dev(data);
327 struct dmar_pci_notify_info *info;
328
329 /* Only care about add/remove events for physical functions */
330 if (pdev->is_virtfn)
331 return NOTIFY_DONE;
e6a8c9b3
JR		 332 if (action != BUS_NOTIFY_ADD_DEVICE &&
333 action != BUS_NOTIFY_REMOVED_DEVICE)
59ce0515
JL		 334 return NOTIFY_DONE;
335
336 info = dmar_alloc_pci_notify_info(pdev, action);
337 if (!info)
338 return NOTIFY_DONE;
339
340 down_write(&dmar_global_lock);
341 if (action == BUS_NOTIFY_ADD_DEVICE)
342 dmar_pci_bus_add_dev(info);
e6a8c9b3 343 else if (action == BUS_NOTIFY_REMOVED_DEVICE)
59ce0515
JL		 344 dmar_pci_bus_del_dev(info);
345 up_write(&dmar_global_lock);
346
347 dmar_free_pci_notify_info(info);
348
349 return NOTIFY_OK;
350}
351
352static struct notifier_block dmar_pci_bus_nb = {
353 .notifier_call = dmar_pci_bus_notifier,
354 .priority = INT_MIN,
355};
356
6b197249
JL		 357static struct dmar_drhd_unit *
358dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
359{
360 struct dmar_drhd_unit *dmaru;
361
362 list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
363 if (dmaru->segment == drhd->segment &&
364 dmaru->reg_base_addr == drhd->address)
365 return dmaru;
366
367 return NULL;
368}
369
10e5247f
KA		 370/**
371 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
372 * structure which uniquely represent one DMA remapping hardware unit
373 * present in the platform
374 */
6b197249 375static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
10e5247f
KA		 376{
377 struct acpi_dmar_hardware_unit *drhd;
378 struct dmar_drhd_unit *dmaru;
379 int ret = 0;
10e5247f 380
e523b38e 381 drhd = (struct acpi_dmar_hardware_unit *)header;
6b197249
JL		 382 dmaru = dmar_find_dmaru(drhd);
383 if (dmaru)
384 goto out;
385
386 dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
10e5247f
KA		 387 if (!dmaru)
388 return -ENOMEM;
389
6b197249
JL		 390 /*
391 * If header is allocated from slab by ACPI _DSM method, we need to
392 * copy the content because the memory buffer will be freed on return.
393 */
394 dmaru->hdr = (void *)(dmaru + 1);
395 memcpy(dmaru->hdr, header, header->length);
10e5247f 396 dmaru->reg_base_addr = drhd->address;
276dbf99 397 dmaru->segment = drhd->segment;
10e5247f 398 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
07cb52ff
DW		 399 dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
400 ((void *)drhd) + drhd->header.length,
401 &dmaru->devices_cnt);
402 if (dmaru->devices_cnt && dmaru->devices == NULL) {
403 kfree(dmaru);
404 return -ENOMEM;
2e455289 405 }
10e5247f 406
1886e8a9
SS		 407 ret = alloc_iommu(dmaru);
408 if (ret) {
07cb52ff
DW		 409 dmar_free_dev_scope(&dmaru->devices,
410 &dmaru->devices_cnt);
1886e8a9
SS		 411 kfree(dmaru);
412 return ret;
413 }
414 dmar_register_drhd_unit(dmaru);
c2a0b538 415
6b197249 416out:
c2a0b538
JL		 417 if (arg)
418 (*(int *)arg)++;
419
1886e8a9
SS		 420 return 0;
421}
422
a868e6b7
JL		 423static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
424{
425 if (dmaru->devices && dmaru->devices_cnt)
426 dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
427 if (dmaru->iommu)
428 free_iommu(dmaru->iommu);
429 kfree(dmaru);
430}
431
c2a0b538
JL		 432static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
433 void *arg)
e625b4a9
DW		 434{
435 struct acpi_dmar_andd *andd = (void *)header;
436
437 /* Check for NUL termination within the designated length */
83118b0d 438 if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
e625b4a9
DW		 439 WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
440 "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
441 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
442 dmi_get_system_info(DMI_BIOS_VENDOR),
443 dmi_get_system_info(DMI_BIOS_VERSION),
444 dmi_get_system_info(DMI_PRODUCT_VERSION));
445 return -EINVAL;
446 }
447 pr_info("ANDD device: %x name: %s\n", andd->device_number,
83118b0d 448 andd->device_name);
e625b4a9
DW		 449
450 return 0;
451}
452
aa697079 453#ifdef CONFIG_ACPI_NUMA
6b197249 454static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
ee34b32d
SS		 455{
456 struct acpi_dmar_rhsa *rhsa;
457 struct dmar_drhd_unit *drhd;
458
459 rhsa = (struct acpi_dmar_rhsa *)header;
aa697079 460 for_each_drhd_unit(drhd) {
ee34b32d
SS		 461 if (drhd->reg_base_addr == rhsa->base_address) {
462 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
463
464 if (!node_online(node))
465 node = -1;
466 drhd->iommu->node = node;
aa697079
DW		 467 return 0;
468 }
ee34b32d 469 }
fd0c8894
BH		 470 WARN_TAINT(
471 1, TAINT_FIRMWARE_WORKAROUND,
472 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
473 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
474 drhd->reg_base_addr,
475 dmi_get_system_info(DMI_BIOS_VENDOR),
476 dmi_get_system_info(DMI_BIOS_VERSION),
477 dmi_get_system_info(DMI_PRODUCT_VERSION));
ee34b32d 478
aa697079 479 return 0;
ee34b32d 480}
c2a0b538
JL		 481#else
482#define dmar_parse_one_rhsa dmar_res_noop
aa697079 483#endif
ee34b32d 484
10e5247f
KA		 485static void __init
486dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
487{
488 struct acpi_dmar_hardware_unit *drhd;
489 struct acpi_dmar_reserved_memory *rmrr;
aa5d2b51 490 struct acpi_dmar_atsr *atsr;
17b60977 491 struct acpi_dmar_rhsa *rhsa;
10e5247f
KA		 492
493 switch (header->type) {
494 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
aa5d2b51
YZ		 495 drhd = container_of(header, struct acpi_dmar_hardware_unit,
496 header);
e9071b0b 497 pr_info("DRHD base: %#016Lx flags: %#x\n",
aa5d2b51 498 (unsigned long long)drhd->address, drhd->flags);
10e5247f
KA		 499 break;
500 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
aa5d2b51
YZ		 501 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
502 header);
e9071b0b 503 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
5b6985ce
FY		 504 (unsigned long long)rmrr->base_address,
505 (unsigned long long)rmrr->end_address);
10e5247f 506 break;
83118b0d 507 case ACPI_DMAR_TYPE_ROOT_ATS:
aa5d2b51 508 atsr = container_of(header, struct acpi_dmar_atsr, header);
e9071b0b 509 pr_info("ATSR flags: %#x\n", atsr->flags);
aa5d2b51 510 break;
83118b0d 511 case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
17b60977 512 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
e9071b0b 513 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
17b60977
RD		 514 (unsigned long long)rhsa->base_address,
515 rhsa->proximity_domain);
516 break;
83118b0d 517 case ACPI_DMAR_TYPE_NAMESPACE:
e625b4a9
DW		 518 /* We don't print this here because we need to sanity-check
519 it first. So print it in dmar_parse_one_andd() instead. */
520 break;
10e5247f
KA		 521 }
522}
523
f6dd5c31
YL		 524/**
525 * dmar_table_detect - checks to see if the platform supports DMAR devices
526 */
527static int __init dmar_table_detect(void)
528{
529 acpi_status status = AE_OK;
530
531 /* if we could find DMAR table, then there are DMAR devices */
8e1568f3
YL		 532 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
533 (struct acpi_table_header **)&dmar_tbl,
534 &dmar_tbl_size);
f6dd5c31
YL		 535
536 if (ACPI_SUCCESS(status) && !dmar_tbl) {
e9071b0b 537 pr_warn("Unable to map DMAR\n");
f6dd5c31
YL		 538 status = AE_NOT_FOUND;
539 }
540
541 return (ACPI_SUCCESS(status) ? 1 : 0);
542}
aaa9d1dd 543
c2a0b538
JL		 544static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
545 size_t len, struct dmar_res_callback *cb)
546{
547 int ret = 0;
548 struct acpi_dmar_header *iter, *next;
549 struct acpi_dmar_header *end = ((void *)start) + len;
550
551 for (iter = start; iter < end && ret == 0; iter = next) {
552 next = (void *)iter + iter->length;
553 if (iter->length == 0) {
554 /* Avoid looping forever on bad ACPI tables */
555 pr_debug(FW_BUG "Invalid 0-length structure\n");
556 break;
557 } else if (next > end) {
558 /* Avoid passing table end */
9f10e5bf 559 pr_warn(FW_BUG "Record passes table end\n");
c2a0b538
JL		 560 ret = -EINVAL;
561 break;
562 }
563
564 if (cb->print_entry)
565 dmar_table_print_dmar_entry(iter);
566
567 if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
568 /* continue for forward compatibility */
569 pr_debug("Unknown DMAR structure type %d\n",
570 iter->type);
571 } else if (cb->cb[iter->type]) {
572 ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
573 } else if (!cb->ignore_unhandled) {
574 pr_warn("No handler for DMAR structure type %d\n",
575 iter->type);
576 ret = -EINVAL;
577 }
578 }
579
580 return ret;
581}
582
583static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
584 struct dmar_res_callback *cb)
585{
586 return dmar_walk_remapping_entries((void *)(dmar + 1),
587 dmar->header.length - sizeof(*dmar), cb);
588}
589
10e5247f
KA		 590/**
591 * parse_dmar_table - parses the DMA reporting table
592 */
593static int __init
594parse_dmar_table(void)
595{
596 struct acpi_table_dmar *dmar;
10e5247f 597 int ret = 0;
7cef3347 598 int drhd_count = 0;
c2a0b538
JL		 599 struct dmar_res_callback cb = {
600 .print_entry = true,
601 .ignore_unhandled = true,
602 .arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
603 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
604 .cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
605 .cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
606 .cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
607 .cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
608 };
10e5247f 609
f6dd5c31
YL		 610 /*
611 * Do it again, earlier dmar_tbl mapping could be mapped with
612 * fixed map.
613 */
614 dmar_table_detect();
615
a59b50e9
JC		 616 /*
617 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
618 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
619 */
620 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
621
10e5247f
KA		 622 dmar = (struct acpi_table_dmar *)dmar_tbl;
623 if (!dmar)
624 return -ENODEV;
625
5b6985ce 626 if (dmar->width < PAGE_SHIFT - 1) {
e9071b0b 627 pr_warn("Invalid DMAR haw\n");
10e5247f
KA		 628 return -EINVAL;
629 }
630
e9071b0b 631 pr_info("Host address width %d\n", dmar->width + 1);
c2a0b538
JL		 632 ret = dmar_walk_dmar_table(dmar, &cb);
633 if (ret == 0 && drhd_count == 0)
7cef3347 634 pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
c2a0b538 635
10e5247f
KA		 636 return ret;
637}
638
832bd858
DW		 639static int dmar_pci_device_match(struct dmar_dev_scope devices[],
640 int cnt, struct pci_dev *dev)
e61d98d8
SS		 641{
642 int index;
832bd858 643 struct device *tmp;
e61d98d8
SS		 644
645 while (dev) {
b683b230 646 for_each_active_dev_scope(devices, cnt, index, tmp)
832bd858 647 if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
e61d98d8
SS		 648 return 1;
649
650 /* Check our parent */
651 dev = dev->bus->self;
652 }
653
654 return 0;
655}
656
657struct dmar_drhd_unit *
658dmar_find_matched_drhd_unit(struct pci_dev *dev)
659{
0e242612 660 struct dmar_drhd_unit *dmaru;
2e824f79
YZ		 661 struct acpi_dmar_hardware_unit *drhd;
662
dda56549
Y		 663 dev = pci_physfn(dev);
664
0e242612 665 rcu_read_lock();
8b161f0e 666 for_each_drhd_unit(dmaru) {
2e824f79
YZ		 667 drhd = container_of(dmaru->hdr,
668 struct acpi_dmar_hardware_unit,
669 header);
670
671 if (dmaru->include_all &&
672 drhd->segment == pci_domain_nr(dev->bus))
0e242612 673 goto out;
e61d98d8 674
2e824f79
YZ		 675 if (dmar_pci_device_match(dmaru->devices,
676 dmaru->devices_cnt, dev))
0e242612 677 goto out;
e61d98d8 678 }
0e242612
JL		 679 dmaru = NULL;
680out:
681 rcu_read_unlock();
e61d98d8 682
0e242612 683 return dmaru;
e61d98d8
SS		 684}
685
ed40356b
DW		 686static void __init dmar_acpi_insert_dev_scope(u8 device_number,
687 struct acpi_device *adev)
688{
689 struct dmar_drhd_unit *dmaru;
690 struct acpi_dmar_hardware_unit *drhd;
691 struct acpi_dmar_device_scope *scope;
692 struct device *tmp;
693 int i;
694 struct acpi_dmar_pci_path *path;
695
696 for_each_drhd_unit(dmaru) {
697 drhd = container_of(dmaru->hdr,
698 struct acpi_dmar_hardware_unit,
699 header);
700
701 for (scope = (void *)(drhd + 1);
702 (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
703 scope = ((void *)scope) + scope->length) {
83118b0d 704 if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
ed40356b
DW		 705 continue;
706 if (scope->enumeration_id != device_number)
707 continue;
708
709 path = (void *)(scope + 1);
710 pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
711 dev_name(&adev->dev), dmaru->reg_base_addr,
712 scope->bus, path->device, path->function);
713 for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
714 if (tmp == NULL) {
715 dmaru->devices[i].bus = scope->bus;
716 dmaru->devices[i].devfn = PCI_DEVFN(path->device,
717 path->function);
718 rcu_assign_pointer(dmaru->devices[i].dev,
719 get_device(&adev->dev));
720 return;
721 }
722 BUG_ON(i >= dmaru->devices_cnt);
723 }
724 }
725 pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
726 device_number, dev_name(&adev->dev));
727}
728
729static int __init dmar_acpi_dev_scope_init(void)
730{
11f1a776
JR		 731 struct acpi_dmar_andd *andd;
732
733 if (dmar_tbl == NULL)
734 return -ENODEV;
735
7713ec06
DW		 736 for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
737 ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
738 andd = ((void *)andd) + andd->header.length) {
83118b0d 739 if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
ed40356b
DW		 740 acpi_handle h;
741 struct acpi_device *adev;
742
743 if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
83118b0d 744 andd->device_name,
ed40356b
DW		 745 &h))) {
746 pr_err("Failed to find handle for ACPI object %s\n",
83118b0d 747 andd->device_name);
ed40356b
DW		 748 continue;
749 }
c0df975f 750 if (acpi_bus_get_device(h, &adev)) {
ed40356b 751 pr_err("Failed to get device for ACPI object %s\n",
83118b0d 752 andd->device_name);
ed40356b
DW		 753 continue;
754 }
755 dmar_acpi_insert_dev_scope(andd->device_number, adev);
756 }
ed40356b
DW		 757 }
758 return 0;
759}
760
1886e8a9
SS		 761int __init dmar_dev_scope_init(void)
762{
2e455289
JL		 763 struct pci_dev *dev = NULL;
764 struct dmar_pci_notify_info *info;
1886e8a9 765
2e455289
JL		 766 if (dmar_dev_scope_status != 1)
767 return dmar_dev_scope_status;
c2c7286a 768
2e455289
JL		 769 if (list_empty(&dmar_drhd_units)) {
770 dmar_dev_scope_status = -ENODEV;
771 } else {
772 dmar_dev_scope_status = 0;
773
63b42624
DW		 774 dmar_acpi_dev_scope_init();
775
2e455289
JL		 776 for_each_pci_dev(dev) {
777 if (dev->is_virtfn)
778 continue;
779
780 info = dmar_alloc_pci_notify_info(dev,
781 BUS_NOTIFY_ADD_DEVICE);
782 if (!info) {
783 return dmar_dev_scope_status;
784 } else {
785 dmar_pci_bus_add_dev(info);
786 dmar_free_pci_notify_info(info);
787 }
788 }
318fe7df 789
2e455289 790 bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1886e8a9
SS		 791 }
792
2e455289 793 return dmar_dev_scope_status;
1886e8a9
SS		 794}
795
10e5247f
KA		 796
797int __init dmar_table_init(void)
798{
1886e8a9 799 static int dmar_table_initialized;
093f87d2
FY		 800 int ret;
801
cc05301f
JL		 802 if (dmar_table_initialized == 0) {
803 ret = parse_dmar_table();
804 if (ret < 0) {
805 if (ret != -ENODEV)
9f10e5bf 806 pr_info("Parse DMAR table failure.\n");
cc05301f
JL		 807 } else if (list_empty(&dmar_drhd_units)) {
808 pr_info("No DMAR devices found\n");
809 ret = -ENODEV;
810 }
093f87d2 811
cc05301f
JL		 812 if (ret < 0)
813 dmar_table_initialized = ret;
814 else
815 dmar_table_initialized = 1;
10e5247f 816 }
093f87d2 817
cc05301f 818 return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
10e5247f
KA		 819}
820
3a8663ee
BH		 821static void warn_invalid_dmar(u64 addr, const char *message)
822{
fd0c8894
BH		 823 WARN_TAINT_ONCE(
824 1, TAINT_FIRMWARE_WORKAROUND,
825 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
826 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
827 addr, message,
828 dmi_get_system_info(DMI_BIOS_VENDOR),
829 dmi_get_system_info(DMI_BIOS_VERSION),
830 dmi_get_system_info(DMI_PRODUCT_VERSION));
3a8663ee 831}
6ecbf01c 832
c2a0b538
JL		 833static int __ref
834dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
86cf898e 835{
86cf898e 836 struct acpi_dmar_hardware_unit *drhd;
c2a0b538
JL		 837 void __iomem *addr;
838 u64 cap, ecap;
86cf898e 839
c2a0b538
JL		 840 drhd = (void *)entry;
841 if (!drhd->address) {
842 warn_invalid_dmar(0, "");
843 return -EINVAL;
844 }
2c992208 845
6b197249
JL		 846 if (arg)
847 addr = ioremap(drhd->address, VTD_PAGE_SIZE);
848 else
849 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
c2a0b538 850 if (!addr) {
9f10e5bf 851 pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
c2a0b538
JL		 852 return -EINVAL;
853 }
6b197249 854
c2a0b538
JL		 855 cap = dmar_readq(addr + DMAR_CAP_REG);
856 ecap = dmar_readq(addr + DMAR_ECAP_REG);
6b197249
JL		 857
858 if (arg)
859 iounmap(addr);
860 else
861 early_iounmap(addr, VTD_PAGE_SIZE);
86cf898e 862
c2a0b538
JL		 863 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
864 warn_invalid_dmar(drhd->address, " returns all ones");
865 return -EINVAL;
86cf898e 866 }
2c992208 867
2c992208 868 return 0;
86cf898e
DW		 869}
870
480125ba 871int __init detect_intel_iommu(void)
2ae21010
SS		 872{
873 int ret;
c2a0b538
JL		 874 struct dmar_res_callback validate_drhd_cb = {
875 .cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
876 .ignore_unhandled = true,
877 };
2ae21010 878
3a5670e8 879 down_write(&dmar_global_lock);
f6dd5c31 880 ret = dmar_table_detect();
86cf898e 881 if (ret)
c2a0b538
JL		 882 ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
883 &validate_drhd_cb);
884 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
885 iommu_detected = 1;
886 /* Make sure ACS will be enabled */
887 pci_request_acs();
888 }
f5d1b97b 889
9d5ce73a 890#ifdef CONFIG_X86
c2a0b538
JL		 891 if (ret)
892 x86_init.iommu.iommu_init = intel_iommu_init;
2ae21010 893#endif
c2a0b538 894
b707cb02 895 early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
f6dd5c31 896 dmar_tbl = NULL;
3a5670e8 897 up_write(&dmar_global_lock);
480125ba 898
4db77ff3 899 return ret ? 1 : -ENODEV;
2ae21010
SS		 900}
901
902
6f5cf521
DD		 903static void unmap_iommu(struct intel_iommu *iommu)
904{
905 iounmap(iommu->reg);
906 release_mem_region(iommu->reg_phys, iommu->reg_size);
907}
908
909/**
910 * map_iommu: map the iommu's registers
911 * @iommu: the iommu to map
912 * @phys_addr: the physical address of the base resgister
e9071b0b 913 *
6f5cf521 914 * Memory map the iommu's registers. Start w/ a single page, and
e9071b0b 915 * possibly expand if that turns out to be insufficent.
6f5cf521
DD		 916 */
917static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
918{
919 int map_size, err=0;
920
921 iommu->reg_phys = phys_addr;
922 iommu->reg_size = VTD_PAGE_SIZE;
923
924 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
9f10e5bf 925 pr_err("Can't reserve memory\n");
6f5cf521
DD		 926 err = -EBUSY;
927 goto out;
928 }
929
930 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
931 if (!iommu->reg) {
9f10e5bf 932 pr_err("Can't map the region\n");
6f5cf521
DD		 933 err = -ENOMEM;
934 goto release;
935 }
936
937 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
938 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
939
940 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
941 err = -EINVAL;
942 warn_invalid_dmar(phys_addr, " returns all ones");
943 goto unmap;
944 }
945
946 /* the registers might be more than one page */
947 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
948 cap_max_fault_reg_offset(iommu->cap));
949 map_size = VTD_PAGE_ALIGN(map_size);
950 if (map_size > iommu->reg_size) {
951 iounmap(iommu->reg);
952 release_mem_region(iommu->reg_phys, iommu->reg_size);
953 iommu->reg_size = map_size;
954 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
955 iommu->name)) {
9f10e5bf 956 pr_err("Can't reserve memory\n");
6f5cf521
DD		 957 err = -EBUSY;
958 goto out;
959 }
960 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
961 if (!iommu->reg) {
9f10e5bf 962 pr_err("Can't map the region\n");
6f5cf521
DD		 963 err = -ENOMEM;
964 goto release;
965 }
966 }
967 err = 0;
968 goto out;
969
970unmap:
971 iounmap(iommu->reg);
972release:
973 release_mem_region(iommu->reg_phys, iommu->reg_size);
974out:
975 return err;
976}
977
78d8e704
JL		 978static int dmar_alloc_seq_id(struct intel_iommu *iommu)
979{
980 iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
981 DMAR_UNITS_SUPPORTED);
982 if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
983 iommu->seq_id = -1;
984 } else {
985 set_bit(iommu->seq_id, dmar_seq_ids);
986 sprintf(iommu->name, "dmar%d", iommu->seq_id);
987 }
988
989 return iommu->seq_id;
990}
991
992static void dmar_free_seq_id(struct intel_iommu *iommu)
993{
994 if (iommu->seq_id >= 0) {
995 clear_bit(iommu->seq_id, dmar_seq_ids);
996 iommu->seq_id = -1;
997 }
998}
999
694835dc 1000static int alloc_iommu(struct dmar_drhd_unit *drhd)
e61d98d8 1001{
c42d9f32 1002 struct intel_iommu *iommu;
3a93c841 1003 u32 ver, sts;
43f7392b 1004 int agaw = 0;
4ed0d3e6 1005 int msagaw = 0;
6f5cf521 1006 int err;
c42d9f32 1007
6ecbf01c 1008 if (!drhd->reg_base_addr) {
3a8663ee 1009 warn_invalid_dmar(0, "");
6ecbf01c
DW		 1010 return -EINVAL;
1011 }
1012
c42d9f32
SS		 1013 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1014 if (!iommu)
1886e8a9 1015 return -ENOMEM;
c42d9f32 1016
78d8e704 1017 if (dmar_alloc_seq_id(iommu) < 0) {
9f10e5bf 1018 pr_err("Failed to allocate seq_id\n");
78d8e704
JL		 1019 err = -ENOSPC;
1020 goto error;
1021 }
e61d98d8 1022
6f5cf521
DD		 1023 err = map_iommu(iommu, drhd->reg_base_addr);
1024 if (err) {
9f10e5bf 1025 pr_err("Failed to map %s\n", iommu->name);
78d8e704 1026 goto error_free_seq_id;
e61d98d8 1027 }
0815565a 1028
6f5cf521 1029 err = -EINVAL;
1b573683
WH		 1030 agaw = iommu_calculate_agaw(iommu);
1031 if (agaw < 0) {
bf947fcb
DD		 1032 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1033 iommu->seq_id);
0815565a 1034 goto err_unmap;
4ed0d3e6
FY		 1035 }
1036 msagaw = iommu_calculate_max_sagaw(iommu);
1037 if (msagaw < 0) {
bf947fcb 1038 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1b573683 1039 iommu->seq_id);
0815565a 1040 goto err_unmap;
1b573683
WH		 1041 }
1042 iommu->agaw = agaw;
4ed0d3e6 1043 iommu->msagaw = msagaw;
67ccac41 1044 iommu->segment = drhd->segment;
1b573683 1045
ee34b32d
SS		 1046 iommu->node = -1;
1047
e61d98d8 1048 ver = readl(iommu->reg + DMAR_VER_REG);
9f10e5bf
JR		 1049 pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1050 iommu->name,
5b6985ce
FY		 1051 (unsigned long long)drhd->reg_base_addr,
1052 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1053 (unsigned long long)iommu->cap,
1054 (unsigned long long)iommu->ecap);
e61d98d8 1055
3a93c841
TI		 1056 /* Reflect status in gcmd */
1057 sts = readl(iommu->reg + DMAR_GSTS_REG);
1058 if (sts & DMA_GSTS_IRES)
1059 iommu->gcmd |= DMA_GCMD_IRE;
1060 if (sts & DMA_GSTS_TES)
1061 iommu->gcmd |= DMA_GCMD_TE;
1062 if (sts & DMA_GSTS_QIES)
1063 iommu->gcmd |= DMA_GCMD_QIE;
1064
1f5b3c3f 1065 raw_spin_lock_init(&iommu->register_lock);
e61d98d8 1066
bc847454 1067 if (intel_iommu_enabled) {
a5459cfe
AW		 1068 iommu->iommu_dev = iommu_device_create(NULL, iommu,
1069 intel_iommu_groups,
2439d4aa 1070 "%s", iommu->name);
a5459cfe 1071
bc847454
JR		 1072 if (IS_ERR(iommu->iommu_dev)) {
1073 err = PTR_ERR(iommu->iommu_dev);
1074 goto err_unmap;
1075 }
59203379
NK		 1076 }
1077
bc847454
JR		 1078 drhd->iommu = iommu;
1079
1886e8a9 1080 return 0;
0815565a 1081
78d8e704 1082err_unmap:
6f5cf521 1083 unmap_iommu(iommu);
78d8e704
JL		 1084error_free_seq_id:
1085 dmar_free_seq_id(iommu);
1086error:
e61d98d8 1087 kfree(iommu);
6f5cf521 1088 return err;
e61d98d8
SS		 1089}
1090
a868e6b7 1091static void free_iommu(struct intel_iommu *iommu)
e61d98d8 1092{
a5459cfe
AW		 1093 iommu_device_destroy(iommu->iommu_dev);
1094
a868e6b7 1095 if (iommu->irq) {
1208225c
DW		 1096 if (iommu->pr_irq) {
1097 free_irq(iommu->pr_irq, iommu);
1098 dmar_free_hwirq(iommu->pr_irq);
1099 iommu->pr_irq = 0;
1100 }
a868e6b7 1101 free_irq(iommu->irq, iommu);
a553b142 1102 dmar_free_hwirq(iommu->irq);
34742db8 1103 iommu->irq = 0;
a868e6b7 1104 }
e61d98d8 1105
a84da70b
JL		 1106 if (iommu->qi) {
1107 free_page((unsigned long)iommu->qi->desc);
1108 kfree(iommu->qi->desc_status);
1109 kfree(iommu->qi);
1110 }
1111
e61d98d8 1112 if (iommu->reg)
6f5cf521
DD		 1113 unmap_iommu(iommu);
1114
78d8e704 1115 dmar_free_seq_id(iommu);
e61d98d8
SS		 1116 kfree(iommu);
1117}
fe962e90
SS		 1118
1119/*
1120 * Reclaim all the submitted descriptors which have completed its work.
1121 */
1122static inline void reclaim_free_desc(struct q_inval *qi)
1123{
6ba6c3a4
YZ		 1124 while (qi->desc_status[qi->free_tail] == QI_DONE ||
1125 qi->desc_status[qi->free_tail] == QI_ABORT) {
fe962e90
SS		 1126 qi->desc_status[qi->free_tail] = QI_FREE;
1127 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1128 qi->free_cnt++;
1129 }
1130}
1131
704126ad
YZ		 1132static int qi_check_fault(struct intel_iommu *iommu, int index)
1133{
1134 u32 fault;
6ba6c3a4 1135 int head, tail;
704126ad
YZ		 1136 struct q_inval *qi = iommu->qi;
1137 int wait_index = (index + 1) % QI_LENGTH;
1138
6ba6c3a4
YZ		 1139 if (qi->desc_status[wait_index] == QI_ABORT)
1140 return -EAGAIN;
1141
704126ad
YZ		 1142 fault = readl(iommu->reg + DMAR_FSTS_REG);
1143
1144 /*
1145 * If IQE happens, the head points to the descriptor associated
1146 * with the error. No new descriptors are fetched until the IQE
1147 * is cleared.
1148 */
1149 if (fault & DMA_FSTS_IQE) {
1150 head = readl(iommu->reg + DMAR_IQH_REG);
6ba6c3a4 1151 if ((head >> DMAR_IQ_SHIFT) == index) {
bf947fcb 1152 pr_err("VT-d detected invalid descriptor: "
6ba6c3a4
YZ		 1153 "low=%llx, high=%llx\n",
1154 (unsigned long long)qi->desc[index].low,
1155 (unsigned long long)qi->desc[index].high);
704126ad
YZ		 1156 memcpy(&qi->desc[index], &qi->desc[wait_index],
1157 sizeof(struct qi_desc));
1158 __iommu_flush_cache(iommu, &qi->desc[index],
1159 sizeof(struct qi_desc));
1160 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1161 return -EINVAL;
1162 }
1163 }
1164
6ba6c3a4
YZ		 1165 /*
1166 * If ITE happens, all pending wait_desc commands are aborted.
1167 * No new descriptors are fetched until the ITE is cleared.
1168 */
1169 if (fault & DMA_FSTS_ITE) {
1170 head = readl(iommu->reg + DMAR_IQH_REG);
1171 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1172 head |= 1;
1173 tail = readl(iommu->reg + DMAR_IQT_REG);
1174 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1175
1176 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1177
1178 do {
1179 if (qi->desc_status[head] == QI_IN_USE)
1180 qi->desc_status[head] = QI_ABORT;
1181 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1182 } while (head != tail);
1183
1184 if (qi->desc_status[wait_index] == QI_ABORT)
1185 return -EAGAIN;
1186 }
1187
1188 if (fault & DMA_FSTS_ICE)
1189 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1190
704126ad
YZ		 1191 return 0;
1192}
1193
fe962e90
SS		 1194/*
1195 * Submit the queued invalidation descriptor to the remapping
1196 * hardware unit and wait for its completion.
1197 */
704126ad 1198int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
fe962e90 1199{
6ba6c3a4 1200 int rc;
fe962e90
SS		 1201 struct q_inval *qi = iommu->qi;
1202 struct qi_desc *hw, wait_desc;
1203 int wait_index, index;
1204 unsigned long flags;
1205
1206 if (!qi)
704126ad 1207 return 0;
fe962e90
SS		 1208
1209 hw = qi->desc;
1210
6ba6c3a4
YZ		 1211restart:
1212 rc = 0;
1213
3b8f4048 1214 raw_spin_lock_irqsave(&qi->q_lock, flags);
fe962e90 1215 while (qi->free_cnt < 3) {
3b8f4048 1216 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
fe962e90 1217 cpu_relax();
3b8f4048 1218 raw_spin_lock_irqsave(&qi->q_lock, flags);
fe962e90
SS		 1219 }
1220
1221 index = qi->free_head;
1222 wait_index = (index + 1) % QI_LENGTH;
1223
1224 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1225
1226 hw[index] = *desc;
1227
704126ad
YZ		 1228 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1229 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
fe962e90
SS		 1230 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1231
1232 hw[wait_index] = wait_desc;
1233
1234 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1235 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1236
1237 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1238 qi->free_cnt -= 2;
1239
fe962e90
SS		 1240 /*
1241 * update the HW tail register indicating the presence of
1242 * new descriptors.
1243 */
6ba6c3a4 1244 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
fe962e90
SS		 1245
1246 while (qi->desc_status[wait_index] != QI_DONE) {
f05810c9
SS		 1247 /*
1248 * We will leave the interrupts disabled, to prevent interrupt
1249 * context to queue another cmd while a cmd is already submitted
1250 * and waiting for completion on this cpu. This is to avoid
1251 * a deadlock where the interrupt context can wait indefinitely
1252 * for free slots in the queue.
1253 */
704126ad
YZ		 1254 rc = qi_check_fault(iommu, index);
1255 if (rc)
6ba6c3a4 1256 break;
704126ad 1257
3b8f4048 1258 raw_spin_unlock(&qi->q_lock);
fe962e90 1259 cpu_relax();
3b8f4048 1260 raw_spin_lock(&qi->q_lock);
fe962e90 1261 }
6ba6c3a4
YZ		 1262
1263 qi->desc_status[index] = QI_DONE;
fe962e90
SS		 1264
1265 reclaim_free_desc(qi);
3b8f4048 1266 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
704126ad 1267
6ba6c3a4
YZ		 1268 if (rc == -EAGAIN)
1269 goto restart;
1270
704126ad 1271 return rc;
fe962e90
SS		 1272}
1273
1274/*
1275 * Flush the global interrupt entry cache.
1276 */
1277void qi_global_iec(struct intel_iommu *iommu)
1278{
1279 struct qi_desc desc;
1280
1281 desc.low = QI_IEC_TYPE;
1282 desc.high = 0;
1283
704126ad 1284 /* should never fail */
fe962e90
SS		 1285 qi_submit_sync(&desc, iommu);
1286}
1287
4c25a2c1
DW		 1288void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1289 u64 type)
3481f210 1290{
3481f210
YS		 1291 struct qi_desc desc;
1292
3481f210
YS		 1293 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1294 | QI_CC_GRAN(type) | QI_CC_TYPE;
1295 desc.high = 0;
1296
4c25a2c1 1297 qi_submit_sync(&desc, iommu);
3481f210
YS		 1298}
1299
1f0ef2aa
DW		 1300void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1301 unsigned int size_order, u64 type)
3481f210
YS		 1302{
1303 u8 dw = 0, dr = 0;
1304
1305 struct qi_desc desc;
1306 int ih = 0;
1307
3481f210
YS		 1308 if (cap_write_drain(iommu->cap))
1309 dw = 1;
1310
1311 if (cap_read_drain(iommu->cap))
1312 dr = 1;
1313
1314 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1315 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1316 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1317 | QI_IOTLB_AM(size_order);
1318
1f0ef2aa 1319 qi_submit_sync(&desc, iommu);
3481f210
YS		 1320}
1321
6ba6c3a4
YZ		 1322void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1323 u64 addr, unsigned mask)
1324{
1325 struct qi_desc desc;
1326
1327 if (mask) {
1328 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1329 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1330 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1331 } else
1332 desc.high = QI_DEV_IOTLB_ADDR(addr);
1333
1334 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1335 qdep = 0;
1336
1337 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1338 QI_DIOTLB_TYPE;
1339
1340 qi_submit_sync(&desc, iommu);
1341}
1342
eba67e5d
SS		 1343/*
1344 * Disable Queued Invalidation interface.
1345 */
1346void dmar_disable_qi(struct intel_iommu *iommu)
1347{
1348 unsigned long flags;
1349 u32 sts;
1350 cycles_t start_time = get_cycles();
1351
1352 if (!ecap_qis(iommu->ecap))
1353 return;
1354
1f5b3c3f 1355 raw_spin_lock_irqsave(&iommu->register_lock, flags);
eba67e5d 1356
fda3bec1 1357 sts = readl(iommu->reg + DMAR_GSTS_REG);
eba67e5d
SS		 1358 if (!(sts & DMA_GSTS_QIES))
1359 goto end;
1360
1361 /*
1362 * Give a chance to HW to complete the pending invalidation requests.
1363 */
1364 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1365 readl(iommu->reg + DMAR_IQH_REG)) &&
1366 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1367 cpu_relax();
1368
1369 iommu->gcmd &= ~DMA_GCMD_QIE;
eba67e5d
SS		 1370 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1371
1372 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1373 !(sts & DMA_GSTS_QIES), sts);
1374end:
1f5b3c3f 1375 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
eba67e5d
SS		 1376}
1377
eb4a52bc
FY		 1378/*
1379 * Enable queued invalidation.
1380 */
1381static void __dmar_enable_qi(struct intel_iommu *iommu)
1382{
c416daa9 1383 u32 sts;
eb4a52bc
FY		 1384 unsigned long flags;
1385 struct q_inval *qi = iommu->qi;
1386
1387 qi->free_head = qi->free_tail = 0;
1388 qi->free_cnt = QI_LENGTH;
1389
1f5b3c3f 1390 raw_spin_lock_irqsave(&iommu->register_lock, flags);
eb4a52bc
FY		 1391
1392 /* write zero to the tail reg */
1393 writel(0, iommu->reg + DMAR_IQT_REG);
1394
1395 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1396
eb4a52bc 1397 iommu->gcmd |= DMA_GCMD_QIE;
c416daa9 1398 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
eb4a52bc
FY		 1399
1400 /* Make sure hardware complete it */
1401 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1402
1f5b3c3f 1403 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
eb4a52bc
FY		 1404}
1405
fe962e90
SS		 1406/*
1407 * Enable Queued Invalidation interface. This is a must to support
1408 * interrupt-remapping. Also used by DMA-remapping, which replaces
1409 * register based IOTLB invalidation.
1410 */
1411int dmar_enable_qi(struct intel_iommu *iommu)
1412{
fe962e90 1413 struct q_inval *qi;
751cafe3 1414 struct page *desc_page;
fe962e90
SS		 1415
1416 if (!ecap_qis(iommu->ecap))
1417 return -ENOENT;
1418
1419 /*
1420 * queued invalidation is already setup and enabled.
1421 */
1422 if (iommu->qi)
1423 return 0;
1424
fa4b57cc 1425 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
fe962e90
SS		 1426 if (!iommu->qi)
1427 return -ENOMEM;
1428
1429 qi = iommu->qi;
1430
751cafe3
SS		 1431
1432 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1433 if (!desc_page) {
fe962e90 1434 kfree(qi);
b707cb02 1435 iommu->qi = NULL;
fe962e90
SS		 1436 return -ENOMEM;
1437 }
1438
751cafe3
SS		 1439 qi->desc = page_address(desc_page);
1440
37a40710 1441 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
fe962e90
SS		 1442 if (!qi->desc_status) {
1443 free_page((unsigned long) qi->desc);
1444 kfree(qi);
b707cb02 1445 iommu->qi = NULL;
fe962e90
SS		 1446 return -ENOMEM;
1447 }
1448
3b8f4048 1449 raw_spin_lock_init(&qi->q_lock);
fe962e90 1450
eb4a52bc 1451 __dmar_enable_qi(iommu);
fe962e90
SS		 1452
1453 return 0;
1454}
0ac2491f
SS		 1455
1456/* iommu interrupt handling. Most stuff are MSI-like. */
1457
9d783ba0
SS		 1458enum faulttype {
1459 DMA_REMAP,
1460 INTR_REMAP,
1461 UNKNOWN,
1462};
1463
1464static const char *dma_remap_fault_reasons[] =
0ac2491f
SS		 1465{
1466 "Software",
1467 "Present bit in root entry is clear",
1468 "Present bit in context entry is clear",
1469 "Invalid context entry",
1470 "Access beyond MGAW",
1471 "PTE Write access is not set",
1472 "PTE Read access is not set",
1473 "Next page table ptr is invalid",
1474 "Root table address invalid",
1475 "Context table ptr is invalid",
1476 "non-zero reserved fields in RTP",
1477 "non-zero reserved fields in CTP",
1478 "non-zero reserved fields in PTE",
4ecccd9e 1479 "PCE for translation request specifies blocking",
0ac2491f 1480};
9d783ba0 1481
95a02e97 1482static const char *irq_remap_fault_reasons[] =
9d783ba0
SS		 1483{
1484 "Detected reserved fields in the decoded interrupt-remapped request",
1485 "Interrupt index exceeded the interrupt-remapping table size",
1486 "Present field in the IRTE entry is clear",
1487 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1488 "Detected reserved fields in the IRTE entry",
1489 "Blocked a compatibility format interrupt request",
1490 "Blocked an interrupt request due to source-id verification failure",
1491};
1492
21004dcd 1493static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
0ac2491f 1494{
fefe1ed1
DC		 1495 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1496 ARRAY_SIZE(irq_remap_fault_reasons))) {
9d783ba0 1497 *fault_type = INTR_REMAP;
95a02e97 1498 return irq_remap_fault_reasons[fault_reason - 0x20];
9d783ba0
SS		 1499 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1500 *fault_type = DMA_REMAP;
1501 return dma_remap_fault_reasons[fault_reason];
1502 } else {
1503 *fault_type = UNKNOWN;
0ac2491f 1504 return "Unknown";
9d783ba0 1505 }
0ac2491f
SS		 1506}
1507
1208225c
DW		 1508
1509static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1510{
1511 if (iommu->irq == irq)
1512 return DMAR_FECTL_REG;
1513 else if (iommu->pr_irq == irq)
1514 return DMAR_PECTL_REG;
1515 else
1516 BUG();
1517}
1518
5c2837fb 1519void dmar_msi_unmask(struct irq_data *data)
0ac2491f 1520{
dced35ae 1521 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1208225c 1522 int reg = dmar_msi_reg(iommu, data->irq);
0ac2491f
SS		 1523 unsigned long flag;
1524
1525 /* unmask it */
1f5b3c3f 1526 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1208225c 1527 writel(0, iommu->reg + reg);
0ac2491f 1528 /* Read a reg to force flush the post write */
1208225c 1529 readl(iommu->reg + reg);
1f5b3c3f 1530 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f
SS		 1531}
1532
5c2837fb 1533void dmar_msi_mask(struct irq_data *data)
0ac2491f 1534{
dced35ae 1535 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1208225c
DW		 1536 int reg = dmar_msi_reg(iommu, data->irq);
1537 unsigned long flag;
0ac2491f
SS		 1538
1539 /* mask it */
1f5b3c3f 1540 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1208225c 1541 writel(DMA_FECTL_IM, iommu->reg + reg);
0ac2491f 1542 /* Read a reg to force flush the post write */
1208225c 1543 readl(iommu->reg + reg);
1f5b3c3f 1544 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f
SS		 1545}
1546
1547void dmar_msi_write(int irq, struct msi_msg *msg)
1548{
dced35ae 1549 struct intel_iommu *iommu = irq_get_handler_data(irq);
1208225c 1550 int reg = dmar_msi_reg(iommu, irq);
0ac2491f
SS		 1551 unsigned long flag;
1552
1f5b3c3f 1553 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1208225c
DW		 1554 writel(msg->data, iommu->reg + reg + 4);
1555 writel(msg->address_lo, iommu->reg + reg + 8);
1556 writel(msg->address_hi, iommu->reg + reg + 12);
1f5b3c3f 1557 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f
SS		 1558}
1559
1560void dmar_msi_read(int irq, struct msi_msg *msg)
1561{
dced35ae 1562 struct intel_iommu *iommu = irq_get_handler_data(irq);
1208225c 1563 int reg = dmar_msi_reg(iommu, irq);
0ac2491f
SS		 1564 unsigned long flag;
1565
1f5b3c3f 1566 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1208225c
DW		 1567 msg->data = readl(iommu->reg + reg + 4);
1568 msg->address_lo = readl(iommu->reg + reg + 8);
1569 msg->address_hi = readl(iommu->reg + reg + 12);
1f5b3c3f 1570 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f
SS		 1571}
1572
1573static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1574 u8 fault_reason, u16 source_id, unsigned long long addr)
1575{
1576 const char *reason;
9d783ba0 1577 int fault_type;
0ac2491f 1578
9d783ba0 1579 reason = dmar_get_fault_reason(fault_reason, &fault_type);
0ac2491f 1580
9d783ba0 1581 if (fault_type == INTR_REMAP)
a0fe14d7
AW		 1582 pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
1583 source_id >> 8, PCI_SLOT(source_id & 0xFF),
9d783ba0
SS		 1584 PCI_FUNC(source_id & 0xFF), addr >> 48,
1585 fault_reason, reason);
1586 else
a0fe14d7
AW		 1587 pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
1588 type ? "DMA Read" : "DMA Write",
1589 source_id >> 8, PCI_SLOT(source_id & 0xFF),
9d783ba0 1590 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
0ac2491f
SS		 1591 return 0;
1592}
1593
1594#define PRIMARY_FAULT_REG_LEN (16)
1531a6a6 1595irqreturn_t dmar_fault(int irq, void *dev_id)
0ac2491f
SS		 1596{
1597 struct intel_iommu *iommu = dev_id;
1598 int reg, fault_index;
1599 u32 fault_status;
1600 unsigned long flag;
c43fce4e
AW		 1601 bool ratelimited;
1602 static DEFINE_RATELIMIT_STATE(rs,
1603 DEFAULT_RATELIMIT_INTERVAL,
1604 DEFAULT_RATELIMIT_BURST);
1605
1606 /* Disable printing, simply clear the fault when ratelimited */
1607 ratelimited = !__ratelimit(&rs);
0ac2491f 1608
1f5b3c3f 1609 raw_spin_lock_irqsave(&iommu->register_lock, flag);
0ac2491f 1610 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
c43fce4e 1611 if (fault_status && !ratelimited)
bf947fcb 1612 pr_err("DRHD: handling fault status reg %x\n", fault_status);
0ac2491f
SS		 1613
1614 /* TBD: ignore advanced fault log currently */
1615 if (!(fault_status & DMA_FSTS_PPF))
bd5cdad0 1616 goto unlock_exit;
0ac2491f
SS		 1617
1618 fault_index = dma_fsts_fault_record_index(fault_status);
1619 reg = cap_fault_reg_offset(iommu->cap);
1620 while (1) {
1621 u8 fault_reason;
1622 u16 source_id;
1623 u64 guest_addr;
1624 int type;
1625 u32 data;
1626
1627 /* highest 32 bits */
1628 data = readl(iommu->reg + reg +
1629 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1630 if (!(data & DMA_FRCD_F))
1631 break;
1632
c43fce4e
AW		 1633 if (!ratelimited) {
1634 fault_reason = dma_frcd_fault_reason(data);
1635 type = dma_frcd_type(data);
0ac2491f 1636
c43fce4e
AW		 1637 data = readl(iommu->reg + reg +
1638 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1639 source_id = dma_frcd_source_id(data);
1640
1641 guest_addr = dmar_readq(iommu->reg + reg +
1642 fault_index * PRIMARY_FAULT_REG_LEN);
1643 guest_addr = dma_frcd_page_addr(guest_addr);
1644 }
0ac2491f 1645
0ac2491f
SS		 1646 /* clear the fault */
1647 writel(DMA_FRCD_F, iommu->reg + reg +
1648 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1649
1f5b3c3f 1650 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f 1651
c43fce4e
AW		 1652 if (!ratelimited)
1653 dmar_fault_do_one(iommu, type, fault_reason,
1654 source_id, guest_addr);
0ac2491f
SS		 1655
1656 fault_index++;
8211a7b5 1657 if (fault_index >= cap_num_fault_regs(iommu->cap))
0ac2491f 1658 fault_index = 0;
1f5b3c3f 1659 raw_spin_lock_irqsave(&iommu->register_lock, flag);
0ac2491f 1660 }
0ac2491f 1661
bd5cdad0
LZH		 1662 writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1663
1664unlock_exit:
1f5b3c3f 1665 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
0ac2491f
SS		 1666 return IRQ_HANDLED;
1667}
1668
1669int dmar_set_interrupt(struct intel_iommu *iommu)
1670{
1671 int irq, ret;
1672
9d783ba0
SS		 1673 /*
1674 * Check if the fault interrupt is already initialized.
1675 */
1676 if (iommu->irq)
1677 return 0;
1678
34742db8
JL		 1679 irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1680 if (irq > 0) {
1681 iommu->irq = irq;
1682 } else {
9f10e5bf 1683 pr_err("No free IRQ vectors\n");
0ac2491f
SS		 1684 return -EINVAL;
1685 }
1686
477694e7 1687 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
0ac2491f 1688 if (ret)
9f10e5bf 1689 pr_err("Can't request irq\n");
0ac2491f
SS		 1690 return ret;
1691}
9d783ba0
SS		 1692
1693int __init enable_drhd_fault_handling(void)
1694{
1695 struct dmar_drhd_unit *drhd;
7c919779 1696 struct intel_iommu *iommu;
9d783ba0
SS		 1697
1698 /*
1699 * Enable fault control interrupt.
1700 */
7c919779 1701 for_each_iommu(iommu, drhd) {
bd5cdad0 1702 u32 fault_status;
7c919779 1703 int ret = dmar_set_interrupt(iommu);
9d783ba0
SS		 1704
1705 if (ret) {
e9071b0b 1706 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
9d783ba0
SS		 1707 (unsigned long long)drhd->reg_base_addr, ret);
1708 return -1;
1709 }
7f99d946
SS		 1710
1711 /*
1712 * Clear any previous faults.
1713 */
1714 dmar_fault(iommu->irq, iommu);
bd5cdad0
LZH		 1715 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1716 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
9d783ba0
SS		 1717 }
1718
1719 return 0;
1720}
eb4a52bc
FY		 1721
1722/*
1723 * Re-enable Queued Invalidation interface.
1724 */
1725int dmar_reenable_qi(struct intel_iommu *iommu)
1726{
1727 if (!ecap_qis(iommu->ecap))
1728 return -ENOENT;
1729
1730 if (!iommu->qi)
1731 return -ENOENT;
1732
1733 /*
1734 * First disable queued invalidation.
1735 */
1736 dmar_disable_qi(iommu);
1737 /*
1738 * Then enable queued invalidation again. Since there is no pending
1739 * invalidation requests now, it's safe to re-enable queued
1740 * invalidation.
1741 */
1742 __dmar_enable_qi(iommu);
1743
1744 return 0;
1745}
074835f0
YS		 1746
1747/*
1748 * Check interrupt remapping support in DMAR table description.
1749 */
0b8973a8 1750int __init dmar_ir_support(void)
074835f0
YS		 1751{
1752 struct acpi_table_dmar *dmar;
1753 dmar = (struct acpi_table_dmar *)dmar_tbl;
4f506e07
AP		 1754 if (!dmar)
1755 return 0;
074835f0
YS		 1756 return dmar->flags & 0x1;
1757}
694835dc 1758
6b197249
JL		 1759/* Check whether DMAR units are in use */
1760static inline bool dmar_in_use(void)
1761{
1762 return irq_remapping_enabled || intel_iommu_enabled;
1763}
1764
a868e6b7
JL		 1765static int __init dmar_free_unused_resources(void)
1766{
1767 struct dmar_drhd_unit *dmaru, *dmaru_n;
1768
6b197249 1769 if (dmar_in_use())
a868e6b7
JL		 1770 return 0;
1771
2e455289
JL		 1772 if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1773 bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
59ce0515 1774
3a5670e8 1775 down_write(&dmar_global_lock);
a868e6b7
JL		 1776 list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1777 list_del(&dmaru->list);
1778 dmar_free_drhd(dmaru);
1779 }
3a5670e8 1780 up_write(&dmar_global_lock);
a868e6b7
JL		 1781
1782 return 0;
1783}
1784
1785late_initcall(dmar_free_unused_resources);
4db77ff3 1786IOMMU_INIT_POST(detect_intel_iommu);
6b197249
JL		 1787
1788/*
1789 * DMAR Hotplug Support
1790 * For more details, please refer to Intel(R) Virtualization Technology
1791 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1792 * "Remapping Hardware Unit Hot Plug".
1793 */
1794static u8 dmar_hp_uuid[] = {
1795 /* 0000 */ 0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1796 /* 0008 */ 0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1797};
1798
1799/*
1800 * Currently there's only one revision and BIOS will not check the revision id,
1801 * so use 0 for safety.
1802 */
1803#define DMAR_DSM_REV_ID 0
1804#define DMAR_DSM_FUNC_DRHD 1
1805#define DMAR_DSM_FUNC_ATSR 2
1806#define DMAR_DSM_FUNC_RHSA 3
1807
1808static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1809{
1810 return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1811}
1812
1813static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1814 dmar_res_handler_t handler, void *arg)
1815{
1816 int ret = -ENODEV;
1817 union acpi_object *obj;
1818 struct acpi_dmar_header *start;
1819 struct dmar_res_callback callback;
1820 static int res_type[] = {
1821 [DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1822 [DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1823 [DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1824 };
1825
1826 if (!dmar_detect_dsm(handle, func))
1827 return 0;
1828
1829 obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1830 func, NULL, ACPI_TYPE_BUFFER);
1831 if (!obj)
1832 return -ENODEV;
1833
1834 memset(&callback, 0, sizeof(callback));
1835 callback.cb[res_type[func]] = handler;
1836 callback.arg[res_type[func]] = arg;
1837 start = (struct acpi_dmar_header *)obj->buffer.pointer;
1838 ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1839
1840 ACPI_FREE(obj);
1841
1842 return ret;
1843}
1844
1845static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1846{
1847 int ret;
1848 struct dmar_drhd_unit *dmaru;
1849
1850 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1851 if (!dmaru)
1852 return -ENODEV;
1853
1854 ret = dmar_ir_hotplug(dmaru, true);
1855 if (ret == 0)
1856 ret = dmar_iommu_hotplug(dmaru, true);
1857
1858 return ret;
1859}
1860
1861static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1862{
1863 int i, ret;
1864 struct device *dev;
1865 struct dmar_drhd_unit *dmaru;
1866
1867 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1868 if (!dmaru)
1869 return 0;
1870
1871 /*
1872 * All PCI devices managed by this unit should have been destroyed.
1873 */
1874 if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1875 for_each_active_dev_scope(dmaru->devices,
1876 dmaru->devices_cnt, i, dev)
1877 return -EBUSY;
1878
1879 ret = dmar_ir_hotplug(dmaru, false);
1880 if (ret == 0)
1881 ret = dmar_iommu_hotplug(dmaru, false);
1882
1883 return ret;
1884}
1885
1886static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1887{
1888 struct dmar_drhd_unit *dmaru;
1889
1890 dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1891 if (dmaru) {
1892 list_del_rcu(&dmaru->list);
1893 synchronize_rcu();
1894 dmar_free_drhd(dmaru);
1895 }
1896
1897 return 0;
1898}
1899
1900static int dmar_hotplug_insert(acpi_handle handle)
1901{
1902 int ret;
1903 int drhd_count = 0;
1904
1905 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1906 &dmar_validate_one_drhd, (void *)1);
1907 if (ret)
1908 goto out;
1909
1910 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1911 &dmar_parse_one_drhd, (void *)&drhd_count);
1912 if (ret == 0 && drhd_count == 0) {
1913 pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1914 goto out;
1915 } else if (ret) {
1916 goto release_drhd;
1917 }
1918
1919 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1920 &dmar_parse_one_rhsa, NULL);
1921 if (ret)
1922 goto release_drhd;
1923
1924 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1925 &dmar_parse_one_atsr, NULL);
1926 if (ret)
1927 goto release_atsr;
1928
1929 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1930 &dmar_hp_add_drhd, NULL);
1931 if (!ret)
1932 return 0;
1933
1934 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1935 &dmar_hp_remove_drhd, NULL);
1936release_atsr:
1937 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1938 &dmar_release_one_atsr, NULL);
1939release_drhd:
1940 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1941 &dmar_hp_release_drhd, NULL);
1942out:
1943 return ret;
1944}
1945
1946static int dmar_hotplug_remove(acpi_handle handle)
1947{
1948 int ret;
1949
1950 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1951 &dmar_check_one_atsr, NULL);
1952 if (ret)
1953 return ret;
1954
1955 ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1956 &dmar_hp_remove_drhd, NULL);
1957 if (ret == 0) {
1958 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1959 &dmar_release_one_atsr, NULL));
1960 WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1961 &dmar_hp_release_drhd, NULL));
1962 } else {
1963 dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1964 &dmar_hp_add_drhd, NULL);
1965 }
1966
1967 return ret;
1968}
1969
d35165a9
JL		 1970static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1971 void *context, void **retval)
1972{
1973 acpi_handle *phdl = retval;
1974
1975 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1976 *phdl = handle;
1977 return AE_CTRL_TERMINATE;
1978 }
1979
1980 return AE_OK;
1981}
1982
6b197249
JL		 1983static int dmar_device_hotplug(acpi_handle handle, bool insert)
1984{
1985 int ret;
d35165a9
JL		 1986 acpi_handle tmp = NULL;
1987 acpi_status status;
6b197249
JL		 1988
1989 if (!dmar_in_use())
1990 return 0;
1991
d35165a9
JL		 1992 if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1993 tmp = handle;
1994 } else {
1995 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1996 ACPI_UINT32_MAX,
1997 dmar_get_dsm_handle,
1998 NULL, NULL, &tmp);
1999 if (ACPI_FAILURE(status)) {
2000 pr_warn("Failed to locate _DSM method.\n");
2001 return -ENXIO;
2002 }
2003 }
2004 if (tmp == NULL)
6b197249
JL		 2005 return 0;
2006
2007 down_write(&dmar_global_lock);
2008 if (insert)
d35165a9 2009 ret = dmar_hotplug_insert(tmp);
6b197249 2010 else
d35165a9 2011 ret = dmar_hotplug_remove(tmp);
6b197249
JL		 2012 up_write(&dmar_global_lock);
2013
2014 return ret;
2015}
2016
2017int dmar_device_add(acpi_handle handle)
2018{
2019 return dmar_device_hotplug(handle, true);
2020}
2021
2022int dmar_device_remove(acpi_handle handle)
2023{
2024 return dmar_device_hotplug(handle, false);
2025}
Linux 6.x block layer and io_uring tree(s)