Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
ea8c64ac | 2 | #include <linux/dma-direct.h> |
2118d0c5 | 3 | #include <linux/dma-debug.h> |
cb5867a5 | 4 | #include <linux/dmar.h> |
69c60c88 | 5 | #include <linux/export.h> |
57c8a661 | 6 | #include <linux/memblock.h> |
5a0e3ad6 | 7 | #include <linux/gfp.h> |
bca5c096 | 8 | #include <linux/pci.h> |
cb5867a5 | 9 | |
116890d5 GC |
10 | #include <asm/proto.h> |
11 | #include <asm/dma.h> | |
46a7fa27 | 12 | #include <asm/iommu.h> |
1d9b16d1 | 13 | #include <asm/gart.h> |
cb5867a5 | 14 | #include <asm/calgary.h> |
b4941a9a | 15 | #include <asm/x86_init.h> |
ee1f284f | 16 | #include <asm/iommu_table.h> |
459121c9 | 17 | |
0ead51c3 | 18 | static bool disable_dac_quirk __read_mostly; |
3b15e581 | 19 | |
fec777c3 | 20 | const struct dma_map_ops *dma_ops = &dma_direct_ops; |
85c246ee GC |
21 | EXPORT_SYMBOL(dma_ops); |
22 | ||
f9c258de GC |
23 | #ifdef CONFIG_IOMMU_DEBUG |
24 | int panic_on_overflow __read_mostly = 1; | |
25 | int force_iommu __read_mostly = 1; | |
26 | #else | |
27 | int panic_on_overflow __read_mostly = 0; | |
28 | int force_iommu __read_mostly = 0; | |
29 | #endif | |
30 | ||
fae9a0d8 GC |
31 | int iommu_merge __read_mostly = 0; |
32 | ||
33 | int no_iommu __read_mostly; | |
34 | /* Set this to 1 if there is a HW IOMMU in the system */ | |
35 | int iommu_detected __read_mostly = 0; | |
36 | ||
ac0101d3 JR |
37 | /* |
38 | * This variable becomes 1 if iommu=pt is passed on the kernel command line. | |
e3be785f | 39 | * If this variable is 1, IOMMU implementations do no DMA translation for |
ac0101d3 | 40 | * devices and allow every device to access to whole physical memory. This is |
fb637f3c | 41 | * useful if a user wants to use an IOMMU only for KVM device assignment to |
ac0101d3 | 42 | * guests and not for driver dma translation. |
58d11317 OJ |
43 | * It is also possible to disable by default in kernel config, and enable with |
44 | * iommu=nopt at boot time. | |
ac0101d3 | 45 | */ |
58d11317 OJ |
46 | #ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH |
47 | int iommu_pass_through __read_mostly = 1; | |
48 | #else | |
ac0101d3 | 49 | int iommu_pass_through __read_mostly; |
58d11317 | 50 | #endif |
aed5d5f4 | 51 | |
ee1f284f KRW |
52 | extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; |
53 | ||
eb647138 | 54 | /* Dummy device used for NULL arguments (normally ISA). */ |
6c505ce3 | 55 | struct device x86_dma_fallback_dev = { |
1a927133 | 56 | .init_name = "fallback device", |
eb647138 | 57 | .coherent_dma_mask = ISA_DMA_BIT_MASK, |
6c505ce3 | 58 | .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, |
098cb7f2 | 59 | }; |
6c505ce3 | 60 | EXPORT_SYMBOL(x86_dma_fallback_dev); |
098cb7f2 | 61 | |
116890d5 GC |
62 | void __init pci_iommu_alloc(void) |
63 | { | |
ee1f284f KRW |
64 | struct iommu_table_entry *p; |
65 | ||
ee1f284f KRW |
66 | sort_iommu_table(__iommu_table, __iommu_table_end); |
67 | check_iommu_entries(__iommu_table, __iommu_table_end); | |
116890d5 | 68 | |
ee1f284f KRW |
69 | for (p = __iommu_table; p < __iommu_table_end; p++) { |
70 | if (p && p->detect && p->detect() > 0) { | |
71 | p->flags |= IOMMU_DETECTED; | |
72 | if (p->early_init) | |
73 | p->early_init(); | |
74 | if (p->flags & IOMMU_FINISH_IF_DETECTED) | |
75 | break; | |
76 | } | |
77 | } | |
116890d5 | 78 | } |
0a2b9a6e | 79 | |
884571f0 | 80 | bool arch_dma_alloc_attrs(struct device **dev) |
0c7965ff | 81 | { |
298a96c1 VS |
82 | if (!*dev) |
83 | *dev = &x86_dma_fallback_dev; | |
84 | ||
6894258e CH |
85 | if (!is_device_dma_capable(*dev)) |
86 | return false; | |
87 | return true; | |
0c7965ff | 88 | |
f1dc154f | 89 | } |
6894258e | 90 | EXPORT_SYMBOL(arch_dma_alloc_attrs); |
f1dc154f | 91 | |
fae9a0d8 | 92 | /* |
395cf969 PB |
93 | * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel |
94 | * parameter documentation. | |
fae9a0d8 GC |
95 | */ |
96 | static __init int iommu_setup(char *p) | |
97 | { | |
98 | iommu_merge = 1; | |
99 | ||
100 | if (!p) | |
101 | return -EINVAL; | |
102 | ||
103 | while (*p) { | |
104 | if (!strncmp(p, "off", 3)) | |
105 | no_iommu = 1; | |
106 | /* gart_parse_options has more force support */ | |
107 | if (!strncmp(p, "force", 5)) | |
108 | force_iommu = 1; | |
109 | if (!strncmp(p, "noforce", 7)) { | |
110 | iommu_merge = 0; | |
111 | force_iommu = 0; | |
112 | } | |
113 | ||
114 | if (!strncmp(p, "biomerge", 8)) { | |
fae9a0d8 GC |
115 | iommu_merge = 1; |
116 | force_iommu = 1; | |
117 | } | |
118 | if (!strncmp(p, "panic", 5)) | |
119 | panic_on_overflow = 1; | |
120 | if (!strncmp(p, "nopanic", 7)) | |
121 | panic_on_overflow = 0; | |
122 | if (!strncmp(p, "merge", 5)) { | |
123 | iommu_merge = 1; | |
124 | force_iommu = 1; | |
125 | } | |
126 | if (!strncmp(p, "nomerge", 7)) | |
127 | iommu_merge = 0; | |
128 | if (!strncmp(p, "forcesac", 8)) | |
06e9552f | 129 | pr_warn("forcesac option ignored.\n"); |
fae9a0d8 | 130 | if (!strncmp(p, "allowdac", 8)) |
098afd98 | 131 | pr_warn("allowdac option ignored.\n"); |
fae9a0d8 | 132 | if (!strncmp(p, "nodac", 5)) |
098afd98 | 133 | pr_warn("nodac option ignored.\n"); |
fae9a0d8 | 134 | if (!strncmp(p, "usedac", 6)) { |
0ead51c3 | 135 | disable_dac_quirk = true; |
fae9a0d8 GC |
136 | return 1; |
137 | } | |
138 | #ifdef CONFIG_SWIOTLB | |
139 | if (!strncmp(p, "soft", 4)) | |
140 | swiotlb = 1; | |
3238c0c4 | 141 | #endif |
80286879 | 142 | if (!strncmp(p, "pt", 2)) |
4ed0d3e6 | 143 | iommu_pass_through = 1; |
58d11317 OJ |
144 | if (!strncmp(p, "nopt", 4)) |
145 | iommu_pass_through = 0; | |
fae9a0d8 | 146 | |
fae9a0d8 | 147 | gart_parse_options(p); |
fae9a0d8 GC |
148 | |
149 | #ifdef CONFIG_CALGARY_IOMMU | |
150 | if (!strncmp(p, "calgary", 7)) | |
151 | use_calgary = 1; | |
152 | #endif /* CONFIG_CALGARY_IOMMU */ | |
153 | ||
154 | p += strcspn(p, ","); | |
155 | if (*p == ',') | |
156 | ++p; | |
157 | } | |
158 | return 0; | |
159 | } | |
160 | early_param("iommu", iommu_setup); | |
161 | ||
cb5867a5 GC |
162 | static int __init pci_iommu_init(void) |
163 | { | |
ee1f284f | 164 | struct iommu_table_entry *p; |
2118d0c5 | 165 | |
d07c1be0 FT |
166 | x86_init.iommu.iommu_init(); |
167 | ||
ee1f284f KRW |
168 | for (p = __iommu_table; p < __iommu_table_end; p++) { |
169 | if (p && (p->flags & IOMMU_DETECTED) && p->late_init) | |
170 | p->late_init(); | |
171 | } | |
75f1cdf1 | 172 | |
cb5867a5 GC |
173 | return 0; |
174 | } | |
cb5867a5 | 175 | /* Must execute after PCI subsystem */ |
9a821b23 | 176 | rootfs_initcall(pci_iommu_init); |
3b15e581 FY |
177 | |
178 | #ifdef CONFIG_PCI | |
179 | /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ | |
180 | ||
0ead51c3 CH |
181 | static int via_no_dac_cb(struct pci_dev *pdev, void *data) |
182 | { | |
f07d141f | 183 | pdev->dev.bus_dma_mask = DMA_BIT_MASK(32); |
0ead51c3 CH |
184 | return 0; |
185 | } | |
186 | ||
a18e3690 | 187 | static void via_no_dac(struct pci_dev *dev) |
3b15e581 | 188 | { |
0ead51c3 | 189 | if (!disable_dac_quirk) { |
13bf7576 | 190 | dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); |
0ead51c3 | 191 | pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL); |
3b15e581 FY |
192 | } |
193 | } | |
c484b241 YL |
194 | DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, |
195 | PCI_CLASS_BRIDGE_PCI, 8, via_no_dac); | |
3b15e581 | 196 | #endif |