Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
1beb6a7d | 2 | * arch/powerpc/sysdev/dart_iommu.c |
1da177e4 | 3 | * |
91f14480 | 4 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1beb6a7d BH |
5 | * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>, |
6 | * IBM Corporation | |
1da177e4 LT |
7 | * |
8 | * Based on pSeries_iommu.c: | |
9 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation | |
91f14480 | 10 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1da177e4 | 11 | * |
1beb6a7d BH |
12 | * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu. |
13 | * | |
1da177e4 | 14 | * |
1da177e4 LT |
15 | * This program is free software; you can redistribute it and/or modify |
16 | * it under the terms of the GNU General Public License as published by | |
17 | * the Free Software Foundation; either version 2 of the License, or | |
18 | * (at your option) any later version. | |
1beb6a7d | 19 | * |
1da177e4 LT |
20 | * This program is distributed in the hope that it will be useful, |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
1beb6a7d | 24 | * |
1da177e4 LT |
25 | * You should have received a copy of the GNU General Public License |
26 | * along with this program; if not, write to the Free Software | |
27 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
28 | */ | |
29 | ||
1da177e4 LT |
30 | #include <linux/init.h> |
31 | #include <linux/types.h> | |
1da177e4 LT |
32 | #include <linux/mm.h> |
33 | #include <linux/spinlock.h> | |
34 | #include <linux/string.h> | |
35 | #include <linux/pci.h> | |
36 | #include <linux/dma-mapping.h> | |
37 | #include <linux/vmalloc.h> | |
7e11580b | 38 | #include <linux/suspend.h> |
95f72d1e | 39 | #include <linux/memblock.h> |
5a0e3ad6 | 40 | #include <linux/gfp.h> |
1da177e4 LT |
41 | #include <asm/io.h> |
42 | #include <asm/prom.h> | |
1da177e4 LT |
43 | #include <asm/iommu.h> |
44 | #include <asm/pci-bridge.h> | |
45 | #include <asm/machdep.h> | |
1da177e4 | 46 | #include <asm/cacheflush.h> |
d387899f | 47 | #include <asm/ppc-pci.h> |
1da177e4 | 48 | |
9933f299 DG |
49 | #include "dart.h" |
50 | ||
c40785ad BH |
51 | /* DART table address and size */ |
52 | static u32 *dart_tablebase; | |
1da177e4 LT |
53 | static unsigned long dart_tablesize; |
54 | ||
1da177e4 | 55 | /* Mapped base address for the dart */ |
6fa2ffe9 | 56 | static unsigned int __iomem *dart; |
1da177e4 LT |
57 | |
58 | /* Dummy val that entries are set to when unused */ | |
59 | static unsigned int dart_emptyval; | |
60 | ||
1beb6a7d BH |
61 | static struct iommu_table iommu_table_dart; |
62 | static int iommu_table_dart_inited; | |
1da177e4 | 63 | static int dart_dirty; |
1beb6a7d | 64 | static int dart_is_u4; |
1da177e4 | 65 | |
8fb07c04 BH |
66 | #define DART_U4_BYPASS_BASE 0x8000000000ull |
67 | ||
1da177e4 LT |
68 | #define DBG(...) |
69 | ||
d900bd73 AB |
70 | static DEFINE_SPINLOCK(invalidate_lock); |
71 | ||
1da177e4 LT |
72 | static inline void dart_tlb_invalidate_all(void) |
73 | { | |
74 | unsigned long l = 0; | |
1beb6a7d | 75 | unsigned int reg, inv_bit; |
1da177e4 | 76 | unsigned long limit; |
d900bd73 AB |
77 | unsigned long flags; |
78 | ||
79 | spin_lock_irqsave(&invalidate_lock, flags); | |
1da177e4 LT |
80 | |
81 | DBG("dart: flush\n"); | |
82 | ||
83 | /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the | |
84 | * control register and wait for it to clear. | |
85 | * | |
86 | * Gotcha: Sometimes, the DART won't detect that the bit gets | |
87 | * set. If so, clear it and set it again. | |
1beb6a7d | 88 | */ |
1da177e4 LT |
89 | |
90 | limit = 0; | |
91 | ||
1beb6a7d | 92 | inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB; |
1da177e4 | 93 | retry: |
1da177e4 | 94 | l = 0; |
1beb6a7d BH |
95 | reg = DART_IN(DART_CNTL); |
96 | reg |= inv_bit; | |
97 | DART_OUT(DART_CNTL, reg); | |
98 | ||
99 | while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit)) | |
1da177e4 | 100 | l++; |
1beb6a7d | 101 | if (l == (1L << limit)) { |
1da177e4 LT |
102 | if (limit < 4) { |
103 | limit++; | |
feb76c7b OJ |
104 | reg = DART_IN(DART_CNTL); |
105 | reg &= ~inv_bit; | |
1beb6a7d | 106 | DART_OUT(DART_CNTL, reg); |
1da177e4 LT |
107 | goto retry; |
108 | } else | |
1beb6a7d | 109 | panic("DART: TLB did not flush after waiting a long " |
1da177e4 LT |
110 | "time. Buggy U3 ?"); |
111 | } | |
d900bd73 AB |
112 | |
113 | spin_unlock_irqrestore(&invalidate_lock, flags); | |
1da177e4 LT |
114 | } |
115 | ||
feb76c7b OJ |
116 | static inline void dart_tlb_invalidate_one(unsigned long bus_rpn) |
117 | { | |
118 | unsigned int reg; | |
119 | unsigned int l, limit; | |
d900bd73 AB |
120 | unsigned long flags; |
121 | ||
122 | spin_lock_irqsave(&invalidate_lock, flags); | |
feb76c7b OJ |
123 | |
124 | reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE | | |
125 | (bus_rpn & DART_CNTL_U4_IONE_MASK); | |
126 | DART_OUT(DART_CNTL, reg); | |
127 | ||
128 | limit = 0; | |
129 | wait_more: | |
130 | l = 0; | |
131 | while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) { | |
132 | rmb(); | |
133 | l++; | |
134 | } | |
135 | ||
136 | if (l == (1L << limit)) { | |
137 | if (limit < 4) { | |
138 | limit++; | |
139 | goto wait_more; | |
140 | } else | |
141 | panic("DART: TLB did not flush after waiting a long " | |
142 | "time. Buggy U4 ?"); | |
143 | } | |
d900bd73 AB |
144 | |
145 | spin_unlock_irqrestore(&invalidate_lock, flags); | |
feb76c7b OJ |
146 | } |
147 | ||
c40785ad BH |
148 | static void dart_cache_sync(unsigned int *base, unsigned int count) |
149 | { | |
150 | /* | |
151 | * We add 1 to the number of entries to flush, following a | |
152 | * comment in Darwin indicating that the memory controller | |
153 | * can prefetch unmapped memory under some circumstances. | |
154 | */ | |
155 | unsigned long start = (unsigned long)base; | |
156 | unsigned long end = start + (count + 1) * sizeof(unsigned int); | |
157 | unsigned int tmp; | |
158 | ||
159 | /* Perform a standard cache flush */ | |
160 | flush_inval_dcache_range(start, end); | |
161 | ||
162 | /* | |
163 | * Perform the sequence described in the CPC925 manual to | |
164 | * ensure all the data gets to a point the cache incoherent | |
165 | * DART hardware will see. | |
166 | */ | |
167 | asm volatile(" sync;" | |
168 | " isync;" | |
169 | " dcbf 0,%1;" | |
170 | " sync;" | |
171 | " isync;" | |
172 | " lwz %0,0(%1);" | |
173 | " isync" : "=r" (tmp) : "r" (end) : "memory"); | |
174 | } | |
175 | ||
1da177e4 LT |
176 | static void dart_flush(struct iommu_table *tbl) |
177 | { | |
eeac5c14 | 178 | mb(); |
feb76c7b | 179 | if (dart_dirty) { |
1da177e4 | 180 | dart_tlb_invalidate_all(); |
feb76c7b OJ |
181 | dart_dirty = 0; |
182 | } | |
1da177e4 LT |
183 | } |
184 | ||
6490c490 | 185 | static int dart_build(struct iommu_table *tbl, long index, |
1da177e4 | 186 | long npages, unsigned long uaddr, |
4f3dd8a0 MN |
187 | enum dma_data_direction direction, |
188 | struct dma_attrs *attrs) | |
1da177e4 | 189 | { |
c40785ad | 190 | unsigned int *dp, *orig_dp; |
1da177e4 | 191 | unsigned int rpn; |
feb76c7b | 192 | long l; |
1da177e4 LT |
193 | |
194 | DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); | |
195 | ||
c40785ad | 196 | orig_dp = dp = ((unsigned int*)tbl->it_base) + index; |
1beb6a7d | 197 | |
af901ca1 | 198 | /* On U3, all memory is contiguous, so we can move this |
1da177e4 LT |
199 | * out of the loop. |
200 | */ | |
feb76c7b OJ |
201 | l = npages; |
202 | while (l--) { | |
579468a9 | 203 | rpn = __pa(uaddr) >> DART_PAGE_SHIFT; |
1da177e4 LT |
204 | |
205 | *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK); | |
206 | ||
d0035c62 | 207 | uaddr += DART_PAGE_SIZE; |
1da177e4 | 208 | } |
c40785ad | 209 | dart_cache_sync(orig_dp, npages); |
eeac5c14 | 210 | |
feb76c7b OJ |
211 | if (dart_is_u4) { |
212 | rpn = index; | |
feb76c7b OJ |
213 | while (npages--) |
214 | dart_tlb_invalidate_one(rpn++); | |
215 | } else { | |
216 | dart_dirty = 1; | |
217 | } | |
6490c490 | 218 | return 0; |
1da177e4 LT |
219 | } |
220 | ||
221 | ||
222 | static void dart_free(struct iommu_table *tbl, long index, long npages) | |
223 | { | |
c40785ad BH |
224 | unsigned int *dp, *orig_dp; |
225 | long orig_npages = npages; | |
1beb6a7d | 226 | |
1da177e4 LT |
227 | /* We don't worry about flushing the TLB cache. The only drawback of |
228 | * not doing it is that we won't catch buggy device drivers doing | |
229 | * bad DMAs, but then no 32-bit architecture ever does either. | |
230 | */ | |
231 | ||
232 | DBG("dart: free at: %lx, %lx\n", index, npages); | |
233 | ||
c40785ad | 234 | orig_dp = dp = ((unsigned int *)tbl->it_base) + index; |
1beb6a7d | 235 | |
1da177e4 LT |
236 | while (npages--) |
237 | *(dp++) = dart_emptyval; | |
1da177e4 | 238 | |
c40785ad BH |
239 | dart_cache_sync(orig_dp, orig_npages); |
240 | } | |
1da177e4 | 241 | |
c40785ad | 242 | static void allocate_dart(void) |
1da177e4 | 243 | { |
c40785ad | 244 | unsigned long tmp; |
1da177e4 | 245 | |
c40785ad BH |
246 | /* 512 pages (2MB) is max DART tablesize. */ |
247 | dart_tablesize = 1UL << 21; | |
1beb6a7d | 248 | |
c40785ad BH |
249 | /* |
250 | * 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we | |
251 | * will blow up an entire large page anyway in the kernel mapping. | |
1da177e4 | 252 | */ |
c40785ad BH |
253 | dart_tablebase = __va(memblock_alloc_base(1UL<<24, |
254 | 1UL<<24, 0x80000000L)); | |
255 | ||
256 | /* There is no point scanning the DART space for leaks*/ | |
257 | kmemleak_no_scan((void *)dart_tablebase); | |
1da177e4 LT |
258 | |
259 | /* Allocate a spare page to map all invalid DART pages. We need to do | |
260 | * that to work around what looks like a problem with the HT bridge | |
261 | * prefetching into invalid pages and corrupting data | |
262 | */ | |
95f72d1e | 263 | tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); |
1beb6a7d BH |
264 | dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & |
265 | DARTMAP_RPNMASK); | |
1da177e4 | 266 | |
c40785ad BH |
267 | printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase); |
268 | } | |
269 | ||
270 | static int __init dart_init(struct device_node *dart_node) | |
271 | { | |
272 | unsigned int i; | |
273 | unsigned long base, size; | |
274 | struct resource r; | |
275 | ||
276 | /* IOMMU disabled by the user ? bail out */ | |
277 | if (iommu_is_off) | |
278 | return -ENODEV; | |
279 | ||
280 | /* | |
281 | * Only use the DART if the machine has more than 1GB of RAM | |
282 | * or if requested with iommu=on on cmdline. | |
283 | * | |
284 | * 1GB of RAM is picked as limit because some default devices | |
285 | * (i.e. Airport Extreme) have 30 bit address range limits. | |
286 | */ | |
287 | ||
288 | if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull) | |
289 | return -ENODEV; | |
290 | ||
291 | /* Get DART registers */ | |
292 | if (of_address_to_resource(dart_node, 0, &r)) | |
293 | panic("DART: can't get register base ! "); | |
294 | ||
1beb6a7d | 295 | /* Map in DART registers */ |
28f65c11 | 296 | dart = ioremap(r.start, resource_size(&r)); |
1da177e4 | 297 | if (dart == NULL) |
1beb6a7d | 298 | panic("DART: Cannot map registers!"); |
1da177e4 | 299 | |
c40785ad BH |
300 | /* Allocate the DART and dummy page */ |
301 | allocate_dart(); | |
1da177e4 LT |
302 | |
303 | /* Fill initial table */ | |
304 | for (i = 0; i < dart_tablesize/4; i++) | |
c40785ad BH |
305 | dart_tablebase[i] = dart_emptyval; |
306 | ||
307 | /* Push to memory */ | |
308 | dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32)); | |
1da177e4 LT |
309 | |
310 | /* Initialize DART with table base and enable it. */ | |
c40785ad | 311 | base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT; |
1beb6a7d BH |
312 | size = dart_tablesize >> DART_PAGE_SHIFT; |
313 | if (dart_is_u4) { | |
56c8eaee | 314 | size &= DART_SIZE_U4_SIZE_MASK; |
1beb6a7d BH |
315 | DART_OUT(DART_BASE_U4, base); |
316 | DART_OUT(DART_SIZE_U4, size); | |
317 | DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE); | |
318 | } else { | |
56c8eaee | 319 | size &= DART_CNTL_U3_SIZE_MASK; |
1beb6a7d BH |
320 | DART_OUT(DART_CNTL, |
321 | DART_CNTL_U3_ENABLE | | |
322 | (base << DART_CNTL_U3_BASE_SHIFT) | | |
323 | (size << DART_CNTL_U3_SIZE_SHIFT)); | |
324 | } | |
1da177e4 LT |
325 | |
326 | /* Invalidate DART to get rid of possible stale TLBs */ | |
327 | dart_tlb_invalidate_all(); | |
328 | ||
1beb6a7d BH |
329 | printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n", |
330 | dart_is_u4 ? "U4" : "U3"); | |
1da177e4 LT |
331 | |
332 | return 0; | |
333 | } | |
334 | ||
da004c36 AK |
335 | static struct iommu_table_ops iommu_dart_ops = { |
336 | .set = dart_build, | |
337 | .clear = dart_free, | |
338 | .flush = dart_flush, | |
339 | }; | |
340 | ||
1beb6a7d | 341 | static void iommu_table_dart_setup(void) |
1da177e4 | 342 | { |
1beb6a7d BH |
343 | iommu_table_dart.it_busno = 0; |
344 | iommu_table_dart.it_offset = 0; | |
1da177e4 | 345 | /* it_size is in number of entries */ |
5d2efba6 | 346 | iommu_table_dart.it_size = dart_tablesize / sizeof(u32); |
67bfa0ee | 347 | iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K; |
1da177e4 LT |
348 | |
349 | /* Initialize the common IOMMU code */ | |
c40785ad | 350 | iommu_table_dart.it_base = (unsigned long)dart_tablebase; |
1beb6a7d BH |
351 | iommu_table_dart.it_index = 0; |
352 | iommu_table_dart.it_blocksize = 1; | |
da004c36 | 353 | iommu_table_dart.it_ops = &iommu_dart_ops; |
ca1588e7 | 354 | iommu_init_table(&iommu_table_dart, -1); |
1da177e4 LT |
355 | |
356 | /* Reserve the last page of the DART to avoid possible prefetch | |
357 | * past the DART mapped area | |
358 | */ | |
1beb6a7d | 359 | set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map); |
1da177e4 LT |
360 | } |
361 | ||
8fb07c04 BH |
362 | static void pci_dma_dev_setup_dart(struct pci_dev *dev) |
363 | { | |
e91c2511 BH |
364 | if (dart_is_u4) |
365 | set_dma_offset(&dev->dev, DART_U4_BYPASS_BASE); | |
366 | set_iommu_table_base(&dev->dev, &iommu_table_dart); | |
1da177e4 LT |
367 | } |
368 | ||
12d04eef | 369 | static void pci_dma_bus_setup_dart(struct pci_bus *bus) |
1da177e4 | 370 | { |
1beb6a7d BH |
371 | if (!iommu_table_dart_inited) { |
372 | iommu_table_dart_inited = 1; | |
373 | iommu_table_dart_setup(); | |
1da177e4 | 374 | } |
1da177e4 LT |
375 | } |
376 | ||
8fb07c04 BH |
377 | static bool dart_device_on_pcie(struct device *dev) |
378 | { | |
379 | struct device_node *np = of_node_get(dev->of_node); | |
380 | ||
381 | while(np) { | |
382 | if (of_device_is_compatible(np, "U4-pcie") || | |
383 | of_device_is_compatible(np, "u4-pcie")) { | |
384 | of_node_put(np); | |
385 | return true; | |
386 | } | |
387 | np = of_get_next_parent(np); | |
388 | } | |
389 | return false; | |
390 | } | |
391 | ||
392 | static int dart_dma_set_mask(struct device *dev, u64 dma_mask) | |
393 | { | |
394 | if (!dev->dma_mask || !dma_supported(dev, dma_mask)) | |
395 | return -EIO; | |
396 | ||
397 | /* U4 supports a DART bypass, we use it for 64-bit capable | |
398 | * devices to improve performances. However, that only works | |
399 | * for devices connected to U4 own PCIe interface, not bridged | |
400 | * through hypertransport. We need the device to support at | |
401 | * least 40 bits of addresses. | |
402 | */ | |
403 | if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) { | |
404 | dev_info(dev, "Using 64-bit DMA iommu bypass\n"); | |
405 | set_dma_ops(dev, &dma_direct_ops); | |
406 | } else { | |
407 | dev_info(dev, "Using 32-bit DMA via iommu\n"); | |
408 | set_dma_ops(dev, &dma_iommu_ops); | |
409 | } | |
8fb07c04 BH |
410 | |
411 | *dev->dma_mask = dma_mask; | |
412 | return 0; | |
413 | } | |
414 | ||
798248a3 | 415 | void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops) |
1da177e4 LT |
416 | { |
417 | struct device_node *dn; | |
418 | ||
419 | /* Find the DART in the device-tree */ | |
420 | dn = of_find_compatible_node(NULL, "dart", "u3-dart"); | |
1beb6a7d BH |
421 | if (dn == NULL) { |
422 | dn = of_find_compatible_node(NULL, "dart", "u4-dart"); | |
423 | if (dn == NULL) | |
34c4d012 | 424 | return; /* use default direct_dma_ops */ |
1beb6a7d BH |
425 | dart_is_u4 = 1; |
426 | } | |
1da177e4 | 427 | |
8fb07c04 BH |
428 | /* Initialize the DART HW */ |
429 | if (dart_init(dn) != 0) | |
430 | goto bail; | |
431 | ||
8fb07c04 BH |
432 | /* Setup bypass if supported */ |
433 | if (dart_is_u4) | |
434 | ppc_md.dma_set_mask = dart_dma_set_mask; | |
1da177e4 | 435 | |
771e569e DA |
436 | controller_ops->dma_dev_setup = pci_dma_dev_setup_dart; |
437 | controller_ops->dma_bus_setup = pci_dma_bus_setup_dart; | |
438 | ||
8fb07c04 BH |
439 | /* Setup pci_dma ops */ |
440 | set_pci_dma_ops(&dma_iommu_ops); | |
441 | return; | |
1beb6a7d BH |
442 | |
443 | bail: | |
444 | /* If init failed, use direct iommu and null setup functions */ | |
771e569e DA |
445 | controller_ops->dma_dev_setup = NULL; |
446 | controller_ops->dma_bus_setup = NULL; | |
1beb6a7d BH |
447 | |
448 | /* Setup pci_dma ops */ | |
98747770 | 449 | set_pci_dma_ops(&dma_direct_ops); |
1da177e4 LT |
450 | } |
451 | ||
7e11580b | 452 | #ifdef CONFIG_PM |
7e11580b JB |
453 | static void iommu_dart_restore(void) |
454 | { | |
c40785ad | 455 | dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32)); |
7e11580b JB |
456 | dart_tlb_invalidate_all(); |
457 | } | |
458 | ||
459 | static int __init iommu_init_late_dart(void) | |
460 | { | |
7e11580b JB |
461 | if (!dart_tablebase) |
462 | return 0; | |
463 | ||
7e11580b JB |
464 | ppc_md.iommu_restore = iommu_dart_restore; |
465 | ||
466 | return 0; | |
467 | } | |
468 | ||
469 | late_initcall(iommu_init_late_dart); | |
c40785ad | 470 | #endif /* CONFIG_PM */ |