Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 | 2 | /* |
1beb6a7d | 3 | * arch/powerpc/sysdev/dart_iommu.c |
1da177e4 | 4 | * |
91f14480 | 5 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1beb6a7d BH |
6 | * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>, |
7 | * IBM Corporation | |
1da177e4 LT |
8 | * |
9 | * Based on pSeries_iommu.c: | |
10 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation | |
91f14480 | 11 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1da177e4 | 12 | * |
1beb6a7d | 13 | * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu. |
1da177e4 LT |
14 | */ |
15 | ||
1da177e4 LT |
16 | #include <linux/init.h> |
17 | #include <linux/types.h> | |
1da177e4 LT |
18 | #include <linux/mm.h> |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/string.h> | |
21 | #include <linux/pci.h> | |
22 | #include <linux/dma-mapping.h> | |
23 | #include <linux/vmalloc.h> | |
7e11580b | 24 | #include <linux/suspend.h> |
95f72d1e | 25 | #include <linux/memblock.h> |
5a0e3ad6 | 26 | #include <linux/gfp.h> |
e6f6390a | 27 | #include <linux/of_address.h> |
1da177e4 | 28 | #include <asm/io.h> |
1da177e4 LT |
29 | #include <asm/iommu.h> |
30 | #include <asm/pci-bridge.h> | |
31 | #include <asm/machdep.h> | |
1da177e4 | 32 | #include <asm/cacheflush.h> |
d387899f | 33 | #include <asm/ppc-pci.h> |
1da177e4 | 34 | |
9933f299 DG |
35 | #include "dart.h" |
36 | ||
c40785ad BH |
37 | /* DART table address and size */ |
38 | static u32 *dart_tablebase; | |
1da177e4 LT |
39 | static unsigned long dart_tablesize; |
40 | ||
1da177e4 | 41 | /* Mapped base address for the dart */ |
6fa2ffe9 | 42 | static unsigned int __iomem *dart; |
1da177e4 LT |
43 | |
44 | /* Dummy val that entries are set to when unused */ | |
45 | static unsigned int dart_emptyval; | |
46 | ||
1beb6a7d BH |
47 | static struct iommu_table iommu_table_dart; |
48 | static int iommu_table_dart_inited; | |
1da177e4 | 49 | static int dart_dirty; |
1beb6a7d | 50 | static int dart_is_u4; |
1da177e4 | 51 | |
8fb07c04 BH |
52 | #define DART_U4_BYPASS_BASE 0x8000000000ull |
53 | ||
1da177e4 LT |
54 | #define DBG(...) |
55 | ||
d900bd73 AB |
56 | static DEFINE_SPINLOCK(invalidate_lock); |
57 | ||
1da177e4 LT |
58 | static inline void dart_tlb_invalidate_all(void) |
59 | { | |
60 | unsigned long l = 0; | |
1beb6a7d | 61 | unsigned int reg, inv_bit; |
1da177e4 | 62 | unsigned long limit; |
d900bd73 AB |
63 | unsigned long flags; |
64 | ||
65 | spin_lock_irqsave(&invalidate_lock, flags); | |
1da177e4 LT |
66 | |
67 | DBG("dart: flush\n"); | |
68 | ||
69 | /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the | |
70 | * control register and wait for it to clear. | |
71 | * | |
72 | * Gotcha: Sometimes, the DART won't detect that the bit gets | |
73 | * set. If so, clear it and set it again. | |
1beb6a7d | 74 | */ |
1da177e4 LT |
75 | |
76 | limit = 0; | |
77 | ||
1beb6a7d | 78 | inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB; |
1da177e4 | 79 | retry: |
1da177e4 | 80 | l = 0; |
1beb6a7d BH |
81 | reg = DART_IN(DART_CNTL); |
82 | reg |= inv_bit; | |
83 | DART_OUT(DART_CNTL, reg); | |
84 | ||
85 | while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit)) | |
1da177e4 | 86 | l++; |
1beb6a7d | 87 | if (l == (1L << limit)) { |
1da177e4 LT |
88 | if (limit < 4) { |
89 | limit++; | |
feb76c7b OJ |
90 | reg = DART_IN(DART_CNTL); |
91 | reg &= ~inv_bit; | |
1beb6a7d | 92 | DART_OUT(DART_CNTL, reg); |
1da177e4 LT |
93 | goto retry; |
94 | } else | |
1beb6a7d | 95 | panic("DART: TLB did not flush after waiting a long " |
1da177e4 LT |
96 | "time. Buggy U3 ?"); |
97 | } | |
d900bd73 AB |
98 | |
99 | spin_unlock_irqrestore(&invalidate_lock, flags); | |
1da177e4 LT |
100 | } |
101 | ||
feb76c7b OJ |
102 | static inline void dart_tlb_invalidate_one(unsigned long bus_rpn) |
103 | { | |
104 | unsigned int reg; | |
105 | unsigned int l, limit; | |
d900bd73 AB |
106 | unsigned long flags; |
107 | ||
108 | spin_lock_irqsave(&invalidate_lock, flags); | |
feb76c7b OJ |
109 | |
110 | reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE | | |
111 | (bus_rpn & DART_CNTL_U4_IONE_MASK); | |
112 | DART_OUT(DART_CNTL, reg); | |
113 | ||
114 | limit = 0; | |
115 | wait_more: | |
116 | l = 0; | |
117 | while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) { | |
118 | rmb(); | |
119 | l++; | |
120 | } | |
121 | ||
122 | if (l == (1L << limit)) { | |
123 | if (limit < 4) { | |
124 | limit++; | |
125 | goto wait_more; | |
126 | } else | |
127 | panic("DART: TLB did not flush after waiting a long " | |
128 | "time. Buggy U4 ?"); | |
129 | } | |
d900bd73 AB |
130 | |
131 | spin_unlock_irqrestore(&invalidate_lock, flags); | |
feb76c7b OJ |
132 | } |
133 | ||
c40785ad BH |
134 | static void dart_cache_sync(unsigned int *base, unsigned int count) |
135 | { | |
136 | /* | |
137 | * We add 1 to the number of entries to flush, following a | |
138 | * comment in Darwin indicating that the memory controller | |
139 | * can prefetch unmapped memory under some circumstances. | |
140 | */ | |
141 | unsigned long start = (unsigned long)base; | |
142 | unsigned long end = start + (count + 1) * sizeof(unsigned int); | |
143 | unsigned int tmp; | |
144 | ||
145 | /* Perform a standard cache flush */ | |
1cfb725f | 146 | flush_dcache_range(start, end); |
c40785ad BH |
147 | |
148 | /* | |
149 | * Perform the sequence described in the CPC925 manual to | |
150 | * ensure all the data gets to a point the cache incoherent | |
151 | * DART hardware will see. | |
152 | */ | |
153 | asm volatile(" sync;" | |
154 | " isync;" | |
155 | " dcbf 0,%1;" | |
156 | " sync;" | |
157 | " isync;" | |
158 | " lwz %0,0(%1);" | |
159 | " isync" : "=r" (tmp) : "r" (end) : "memory"); | |
160 | } | |
161 | ||
1da177e4 LT |
162 | static void dart_flush(struct iommu_table *tbl) |
163 | { | |
eeac5c14 | 164 | mb(); |
feb76c7b | 165 | if (dart_dirty) { |
1da177e4 | 166 | dart_tlb_invalidate_all(); |
feb76c7b OJ |
167 | dart_dirty = 0; |
168 | } | |
1da177e4 LT |
169 | } |
170 | ||
6490c490 | 171 | static int dart_build(struct iommu_table *tbl, long index, |
1da177e4 | 172 | long npages, unsigned long uaddr, |
4f3dd8a0 | 173 | enum dma_data_direction direction, |
00085f1e | 174 | unsigned long attrs) |
1da177e4 | 175 | { |
c40785ad | 176 | unsigned int *dp, *orig_dp; |
1da177e4 | 177 | unsigned int rpn; |
feb76c7b | 178 | long l; |
1da177e4 LT |
179 | |
180 | DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); | |
181 | ||
c40785ad | 182 | orig_dp = dp = ((unsigned int*)tbl->it_base) + index; |
1beb6a7d | 183 | |
af901ca1 | 184 | /* On U3, all memory is contiguous, so we can move this |
1da177e4 LT |
185 | * out of the loop. |
186 | */ | |
feb76c7b OJ |
187 | l = npages; |
188 | while (l--) { | |
579468a9 | 189 | rpn = __pa(uaddr) >> DART_PAGE_SHIFT; |
1da177e4 LT |
190 | |
191 | *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK); | |
192 | ||
d0035c62 | 193 | uaddr += DART_PAGE_SIZE; |
1da177e4 | 194 | } |
c40785ad | 195 | dart_cache_sync(orig_dp, npages); |
eeac5c14 | 196 | |
feb76c7b OJ |
197 | if (dart_is_u4) { |
198 | rpn = index; | |
feb76c7b OJ |
199 | while (npages--) |
200 | dart_tlb_invalidate_one(rpn++); | |
201 | } else { | |
202 | dart_dirty = 1; | |
203 | } | |
6490c490 | 204 | return 0; |
1da177e4 LT |
205 | } |
206 | ||
207 | ||
208 | static void dart_free(struct iommu_table *tbl, long index, long npages) | |
209 | { | |
c40785ad BH |
210 | unsigned int *dp, *orig_dp; |
211 | long orig_npages = npages; | |
1beb6a7d | 212 | |
1da177e4 LT |
213 | /* We don't worry about flushing the TLB cache. The only drawback of |
214 | * not doing it is that we won't catch buggy device drivers doing | |
215 | * bad DMAs, but then no 32-bit architecture ever does either. | |
216 | */ | |
217 | ||
218 | DBG("dart: free at: %lx, %lx\n", index, npages); | |
219 | ||
c40785ad | 220 | orig_dp = dp = ((unsigned int *)tbl->it_base) + index; |
1beb6a7d | 221 | |
1da177e4 LT |
222 | while (npages--) |
223 | *(dp++) = dart_emptyval; | |
1da177e4 | 224 | |
c40785ad BH |
225 | dart_cache_sync(orig_dp, orig_npages); |
226 | } | |
1da177e4 | 227 | |
6c552983 | 228 | static void __init allocate_dart(void) |
1da177e4 | 229 | { |
c40785ad | 230 | unsigned long tmp; |
1da177e4 | 231 | |
c40785ad BH |
232 | /* 512 pages (2MB) is max DART tablesize. */ |
233 | dart_tablesize = 1UL << 21; | |
1beb6a7d | 234 | |
c40785ad BH |
235 | /* |
236 | * 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we | |
237 | * will blow up an entire large page anyway in the kernel mapping. | |
1da177e4 | 238 | */ |
f806714f MR |
239 | dart_tablebase = memblock_alloc_try_nid_raw(SZ_16M, SZ_16M, |
240 | MEMBLOCK_LOW_LIMIT, SZ_2G, | |
241 | NUMA_NO_NODE); | |
242 | if (!dart_tablebase) | |
243 | panic("Failed to allocate 16MB below 2GB for DART table\n"); | |
c40785ad | 244 | |
1da177e4 LT |
245 | /* Allocate a spare page to map all invalid DART pages. We need to do |
246 | * that to work around what looks like a problem with the HT bridge | |
247 | * prefetching into invalid pages and corrupting data | |
248 | */ | |
9a8dd708 | 249 | tmp = memblock_phys_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); |
ecc3e771 MR |
250 | if (!tmp) |
251 | panic("DART: table allocation failed\n"); | |
252 | ||
1beb6a7d BH |
253 | dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & |
254 | DARTMAP_RPNMASK); | |
1da177e4 | 255 | |
c40785ad BH |
256 | printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase); |
257 | } | |
258 | ||
259 | static int __init dart_init(struct device_node *dart_node) | |
260 | { | |
261 | unsigned int i; | |
262 | unsigned long base, size; | |
263 | struct resource r; | |
264 | ||
265 | /* IOMMU disabled by the user ? bail out */ | |
266 | if (iommu_is_off) | |
267 | return -ENODEV; | |
268 | ||
269 | /* | |
270 | * Only use the DART if the machine has more than 1GB of RAM | |
271 | * or if requested with iommu=on on cmdline. | |
272 | * | |
273 | * 1GB of RAM is picked as limit because some default devices | |
274 | * (i.e. Airport Extreme) have 30 bit address range limits. | |
275 | */ | |
276 | ||
277 | if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull) | |
278 | return -ENODEV; | |
279 | ||
280 | /* Get DART registers */ | |
281 | if (of_address_to_resource(dart_node, 0, &r)) | |
282 | panic("DART: can't get register base ! "); | |
283 | ||
1beb6a7d | 284 | /* Map in DART registers */ |
28f65c11 | 285 | dart = ioremap(r.start, resource_size(&r)); |
1da177e4 | 286 | if (dart == NULL) |
1beb6a7d | 287 | panic("DART: Cannot map registers!"); |
1da177e4 | 288 | |
c40785ad BH |
289 | /* Allocate the DART and dummy page */ |
290 | allocate_dart(); | |
1da177e4 LT |
291 | |
292 | /* Fill initial table */ | |
293 | for (i = 0; i < dart_tablesize/4; i++) | |
c40785ad BH |
294 | dart_tablebase[i] = dart_emptyval; |
295 | ||
296 | /* Push to memory */ | |
297 | dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32)); | |
1da177e4 LT |
298 | |
299 | /* Initialize DART with table base and enable it. */ | |
c40785ad | 300 | base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT; |
1beb6a7d BH |
301 | size = dart_tablesize >> DART_PAGE_SHIFT; |
302 | if (dart_is_u4) { | |
56c8eaee | 303 | size &= DART_SIZE_U4_SIZE_MASK; |
1beb6a7d BH |
304 | DART_OUT(DART_BASE_U4, base); |
305 | DART_OUT(DART_SIZE_U4, size); | |
306 | DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE); | |
307 | } else { | |
56c8eaee | 308 | size &= DART_CNTL_U3_SIZE_MASK; |
1beb6a7d BH |
309 | DART_OUT(DART_CNTL, |
310 | DART_CNTL_U3_ENABLE | | |
311 | (base << DART_CNTL_U3_BASE_SHIFT) | | |
312 | (size << DART_CNTL_U3_SIZE_SHIFT)); | |
313 | } | |
1da177e4 LT |
314 | |
315 | /* Invalidate DART to get rid of possible stale TLBs */ | |
316 | dart_tlb_invalidate_all(); | |
317 | ||
1beb6a7d BH |
318 | printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n", |
319 | dart_is_u4 ? "U4" : "U3"); | |
1da177e4 LT |
320 | |
321 | return 0; | |
322 | } | |
323 | ||
da004c36 AK |
324 | static struct iommu_table_ops iommu_dart_ops = { |
325 | .set = dart_build, | |
326 | .clear = dart_free, | |
327 | .flush = dart_flush, | |
328 | }; | |
329 | ||
1beb6a7d | 330 | static void iommu_table_dart_setup(void) |
1da177e4 | 331 | { |
1beb6a7d BH |
332 | iommu_table_dart.it_busno = 0; |
333 | iommu_table_dart.it_offset = 0; | |
1da177e4 | 334 | /* it_size is in number of entries */ |
5d2efba6 | 335 | iommu_table_dart.it_size = dart_tablesize / sizeof(u32); |
67bfa0ee | 336 | iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K; |
1da177e4 LT |
337 | |
338 | /* Initialize the common IOMMU code */ | |
c40785ad | 339 | iommu_table_dart.it_base = (unsigned long)dart_tablebase; |
1beb6a7d BH |
340 | iommu_table_dart.it_index = 0; |
341 | iommu_table_dart.it_blocksize = 1; | |
da004c36 | 342 | iommu_table_dart.it_ops = &iommu_dart_ops; |
4be518d8 AK |
343 | if (!iommu_init_table(&iommu_table_dart, -1, 0, 0)) |
344 | panic("Failed to initialize iommu table"); | |
1da177e4 LT |
345 | |
346 | /* Reserve the last page of the DART to avoid possible prefetch | |
347 | * past the DART mapped area | |
348 | */ | |
1beb6a7d | 349 | set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map); |
1da177e4 LT |
350 | } |
351 | ||
12d04eef | 352 | static void pci_dma_bus_setup_dart(struct pci_bus *bus) |
1da177e4 | 353 | { |
1beb6a7d BH |
354 | if (!iommu_table_dart_inited) { |
355 | iommu_table_dart_inited = 1; | |
356 | iommu_table_dart_setup(); | |
1da177e4 | 357 | } |
1da177e4 LT |
358 | } |
359 | ||
8fb07c04 BH |
360 | static bool dart_device_on_pcie(struct device *dev) |
361 | { | |
362 | struct device_node *np = of_node_get(dev->of_node); | |
363 | ||
364 | while(np) { | |
365 | if (of_device_is_compatible(np, "U4-pcie") || | |
366 | of_device_is_compatible(np, "u4-pcie")) { | |
367 | of_node_put(np); | |
368 | return true; | |
369 | } | |
370 | np = of_get_next_parent(np); | |
371 | } | |
372 | return false; | |
373 | } | |
374 | ||
9f4a68d4 | 375 | static void pci_dma_dev_setup_dart(struct pci_dev *dev) |
8fb07c04 | 376 | { |
9f4a68d4 | 377 | if (dart_is_u4 && dart_device_on_pcie(&dev->dev)) |
0617fc0c | 378 | dev->dev.archdata.dma_offset = DART_U4_BYPASS_BASE; |
9f4a68d4 CH |
379 | set_iommu_table_base(&dev->dev, &iommu_table_dart); |
380 | } | |
8fb07c04 | 381 | |
9f4a68d4 CH |
382 | static bool iommu_bypass_supported_dart(struct pci_dev *dev, u64 mask) |
383 | { | |
384 | return dart_is_u4 && | |
385 | dart_device_on_pcie(&dev->dev) && | |
386 | mask >= DMA_BIT_MASK(40); | |
8fb07c04 BH |
387 | } |
388 | ||
798248a3 | 389 | void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops) |
1da177e4 LT |
390 | { |
391 | struct device_node *dn; | |
392 | ||
393 | /* Find the DART in the device-tree */ | |
394 | dn = of_find_compatible_node(NULL, "dart", "u3-dart"); | |
1beb6a7d BH |
395 | if (dn == NULL) { |
396 | dn = of_find_compatible_node(NULL, "dart", "u4-dart"); | |
397 | if (dn == NULL) | |
34c4d012 | 398 | return; /* use default direct_dma_ops */ |
1beb6a7d BH |
399 | dart_is_u4 = 1; |
400 | } | |
1da177e4 | 401 | |
8fb07c04 | 402 | /* Initialize the DART HW */ |
57b742a5 PW |
403 | if (dart_init(dn) != 0) { |
404 | of_node_put(dn); | |
ee69049e | 405 | return; |
57b742a5 | 406 | } |
9f4a68d4 CH |
407 | /* |
408 | * U4 supports a DART bypass, we use it for 64-bit capable devices to | |
409 | * improve performance. However, that only works for devices connected | |
410 | * to the U4 own PCIe interface, not bridged through hypertransport. | |
411 | * We need the device to support at least 40 bits of addresses. | |
412 | */ | |
771e569e DA |
413 | controller_ops->dma_dev_setup = pci_dma_dev_setup_dart; |
414 | controller_ops->dma_bus_setup = pci_dma_bus_setup_dart; | |
9f4a68d4 | 415 | controller_ops->iommu_bypass_supported = iommu_bypass_supported_dart; |
771e569e | 416 | |
8fb07c04 BH |
417 | /* Setup pci_dma ops */ |
418 | set_pci_dma_ops(&dma_iommu_ops); | |
57b742a5 | 419 | of_node_put(dn); |
1da177e4 LT |
420 | } |
421 | ||
7e11580b | 422 | #ifdef CONFIG_PM |
7e11580b JB |
423 | static void iommu_dart_restore(void) |
424 | { | |
c40785ad | 425 | dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32)); |
7e11580b JB |
426 | dart_tlb_invalidate_all(); |
427 | } | |
428 | ||
429 | static int __init iommu_init_late_dart(void) | |
430 | { | |
7e11580b JB |
431 | if (!dart_tablebase) |
432 | return 0; | |
433 | ||
7e11580b JB |
434 | ppc_md.iommu_restore = iommu_dart_restore; |
435 | ||
436 | return 0; | |
437 | } | |
438 | ||
439 | late_initcall(iommu_init_late_dart); | |
c40785ad | 440 | #endif /* CONFIG_PM */ |