Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* $Id: sbus.c,v 1.19 2002/01/23 11:27:32 davem Exp $ |
2 | * sbus.c: UltraSparc SBUS controller support. | |
3 | * | |
4 | * Copyright (C) 1999 David S. Miller (davem@redhat.com) | |
5 | */ | |
6 | ||
7 | #include <linux/kernel.h> | |
8 | #include <linux/types.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/interrupt.h> | |
14 | ||
15 | #include <asm/page.h> | |
16 | #include <asm/sbus.h> | |
17 | #include <asm/io.h> | |
18 | #include <asm/upa.h> | |
19 | #include <asm/cache.h> | |
20 | #include <asm/dma.h> | |
21 | #include <asm/irq.h> | |
25c7581b | 22 | #include <asm/prom.h> |
1da177e4 LT |
23 | #include <asm/starfire.h> |
24 | ||
25 | #include "iommu_common.h" | |
26 | ||
27 | /* These should be allocated on an SMP_CACHE_BYTES | |
28 | * aligned boundary for optimal performance. | |
29 | * | |
30 | * On SYSIO, using an 8K page size we have 1GB of SBUS | |
31 | * DMA space mapped. We divide this space into equally | |
32 | * sized clusters. We allocate a DMA mapping from the | |
33 | * cluster that matches the order of the allocation, or | |
34 | * if the order is greater than the number of clusters, | |
35 | * we try to allocate from the last cluster. | |
36 | */ | |
37 | ||
38 | #define NCLUSTERS 8UL | |
39 | #define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL) | |
40 | #define CLUSTER_SIZE (ONE_GIG / NCLUSTERS) | |
41 | #define CLUSTER_MASK (CLUSTER_SIZE - 1) | |
42 | #define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT) | |
43 | #define MAP_BASE ((u32)0xc0000000) | |
44 | ||
45 | struct sbus_iommu { | |
46 | /*0x00*/spinlock_t lock; | |
47 | ||
48 | /*0x08*/iopte_t *page_table; | |
49 | /*0x10*/unsigned long strbuf_regs; | |
50 | /*0x18*/unsigned long iommu_regs; | |
51 | /*0x20*/unsigned long sbus_control_reg; | |
52 | ||
53 | /*0x28*/volatile unsigned long strbuf_flushflag; | |
54 | ||
55 | /* If NCLUSTERS is ever decresed to 4 or lower, | |
56 | * you must increase the size of the type of | |
57 | * these counters. You have been duly warned. -DaveM | |
58 | */ | |
59 | /*0x30*/struct { | |
60 | u16 next; | |
61 | u16 flush; | |
62 | } alloc_info[NCLUSTERS]; | |
63 | ||
64 | /* The lowest used consistent mapping entry. Since | |
65 | * we allocate consistent maps out of cluster 0 this | |
66 | * is relative to the beginning of closter 0. | |
67 | */ | |
68 | /*0x50*/u32 lowest_consistent_map; | |
69 | }; | |
70 | ||
71 | /* Offsets from iommu_regs */ | |
72 | #define SYSIO_IOMMUREG_BASE 0x2400UL | |
73 | #define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */ | |
74 | #define IOMMU_TSBBASE (0x2408UL - 0x2400UL) /* TSB base address register */ | |
75 | #define IOMMU_FLUSH (0x2410UL - 0x2400UL) /* IOMMU flush register */ | |
76 | #define IOMMU_VADIAG (0x4400UL - 0x2400UL) /* SBUS virtual address diagnostic */ | |
77 | #define IOMMU_TAGCMP (0x4408UL - 0x2400UL) /* TLB tag compare diagnostics */ | |
78 | #define IOMMU_LRUDIAG (0x4500UL - 0x2400UL) /* IOMMU LRU queue diagnostics */ | |
79 | #define IOMMU_TAGDIAG (0x4580UL - 0x2400UL) /* TLB tag diagnostics */ | |
80 | #define IOMMU_DRAMDIAG (0x4600UL - 0x2400UL) /* TLB data RAM diagnostics */ | |
81 | ||
82 | #define IOMMU_DRAM_VALID (1UL << 30UL) | |
83 | ||
84 | static void __iommu_flushall(struct sbus_iommu *iommu) | |
85 | { | |
86 | unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG; | |
87 | int entry; | |
88 | ||
89 | for (entry = 0; entry < 16; entry++) { | |
90 | upa_writeq(0, tag); | |
91 | tag += 8UL; | |
92 | } | |
93 | upa_readq(iommu->sbus_control_reg); | |
94 | ||
95 | for (entry = 0; entry < NCLUSTERS; entry++) { | |
96 | iommu->alloc_info[entry].flush = | |
97 | iommu->alloc_info[entry].next; | |
98 | } | |
99 | } | |
100 | ||
101 | static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
102 | { | |
103 | while (npages--) | |
104 | upa_writeq(base + (npages << IO_PAGE_SHIFT), | |
105 | iommu->iommu_regs + IOMMU_FLUSH); | |
106 | upa_readq(iommu->sbus_control_reg); | |
107 | } | |
108 | ||
109 | /* Offsets from strbuf_regs */ | |
110 | #define SYSIO_STRBUFREG_BASE 0x2800UL | |
111 | #define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */ | |
112 | #define STRBUF_PFLUSH (0x2808UL - 0x2800UL) /* Page flush/invalidate */ | |
113 | #define STRBUF_FSYNC (0x2810UL - 0x2800UL) /* Flush synchronization */ | |
114 | #define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL) /* data RAM diagnostic */ | |
115 | #define STRBUF_ERRDIAG (0x5400UL - 0x2800UL) /* error status diagnostics */ | |
116 | #define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL) /* Page tag diagnostics */ | |
117 | #define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL) /* Line tag diagnostics */ | |
118 | ||
119 | #define STRBUF_TAG_VALID 0x02UL | |
120 | ||
7c963ad1 | 121 | static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages, int direction) |
1da177e4 | 122 | { |
4dbc30fb DM |
123 | unsigned long n; |
124 | int limit; | |
125 | ||
4dbc30fb DM |
126 | n = npages; |
127 | while (n--) | |
128 | upa_writeq(base + (n << IO_PAGE_SHIFT), | |
1da177e4 LT |
129 | iommu->strbuf_regs + STRBUF_PFLUSH); |
130 | ||
7c963ad1 DM |
131 | /* If the device could not have possibly put dirty data into |
132 | * the streaming cache, no flush-flag synchronization needs | |
133 | * to be performed. | |
134 | */ | |
135 | if (direction == SBUS_DMA_TODEVICE) | |
136 | return; | |
137 | ||
138 | iommu->strbuf_flushflag = 0UL; | |
139 | ||
1da177e4 LT |
140 | /* Whoopee cushion! */ |
141 | upa_writeq(__pa(&iommu->strbuf_flushflag), | |
142 | iommu->strbuf_regs + STRBUF_FSYNC); | |
143 | upa_readq(iommu->sbus_control_reg); | |
4dbc30fb | 144 | |
a228dfd5 | 145 | limit = 100000; |
4dbc30fb DM |
146 | while (iommu->strbuf_flushflag == 0UL) { |
147 | limit--; | |
148 | if (!limit) | |
149 | break; | |
a228dfd5 | 150 | udelay(1); |
4f07118f | 151 | rmb(); |
4dbc30fb DM |
152 | } |
153 | if (!limit) | |
154 | printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " | |
155 | "vaddr[%08x] npages[%ld]\n", | |
156 | base, npages); | |
1da177e4 LT |
157 | } |
158 | ||
159 | static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages) | |
160 | { | |
161 | iopte_t *iopte, *limit, *first, *cluster; | |
162 | unsigned long cnum, ent, nent, flush_point, found; | |
163 | ||
164 | cnum = 0; | |
165 | nent = 1; | |
166 | while ((1UL << cnum) < npages) | |
167 | cnum++; | |
168 | if(cnum >= NCLUSTERS) { | |
169 | nent = 1UL << (cnum - NCLUSTERS); | |
170 | cnum = NCLUSTERS - 1; | |
171 | } | |
172 | iopte = iommu->page_table + (cnum * CLUSTER_NPAGES); | |
173 | ||
174 | if (cnum == 0) | |
175 | limit = (iommu->page_table + | |
176 | iommu->lowest_consistent_map); | |
177 | else | |
178 | limit = (iopte + CLUSTER_NPAGES); | |
179 | ||
180 | iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); | |
181 | flush_point = iommu->alloc_info[cnum].flush; | |
182 | ||
183 | first = iopte; | |
184 | cluster = NULL; | |
185 | found = 0; | |
186 | for (;;) { | |
187 | if (iopte_val(*iopte) == 0UL) { | |
188 | found++; | |
189 | if (!cluster) | |
190 | cluster = iopte; | |
191 | } else { | |
192 | /* Used cluster in the way */ | |
193 | cluster = NULL; | |
194 | found = 0; | |
195 | } | |
196 | ||
197 | if (found == nent) | |
198 | break; | |
199 | ||
200 | iopte += (1 << cnum); | |
201 | ent++; | |
202 | if (iopte >= limit) { | |
203 | iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES)); | |
204 | ent = 0; | |
205 | ||
206 | /* Multiple cluster allocations must not wrap */ | |
207 | cluster = NULL; | |
208 | found = 0; | |
209 | } | |
210 | if (ent == flush_point) | |
211 | __iommu_flushall(iommu); | |
212 | if (iopte == first) | |
213 | goto bad; | |
214 | } | |
215 | ||
216 | /* ent/iopte points to the last cluster entry we're going to use, | |
217 | * so save our place for the next allocation. | |
218 | */ | |
219 | if ((iopte + (1 << cnum)) >= limit) | |
220 | ent = 0; | |
221 | else | |
222 | ent = ent + 1; | |
223 | iommu->alloc_info[cnum].next = ent; | |
224 | if (ent == flush_point) | |
225 | __iommu_flushall(iommu); | |
226 | ||
227 | /* I've got your streaming cluster right here buddy boy... */ | |
228 | return cluster; | |
229 | ||
230 | bad: | |
231 | printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n", | |
232 | npages); | |
233 | return NULL; | |
234 | } | |
235 | ||
236 | static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
237 | { | |
238 | unsigned long cnum, ent, nent; | |
239 | iopte_t *iopte; | |
240 | ||
241 | cnum = 0; | |
242 | nent = 1; | |
243 | while ((1UL << cnum) < npages) | |
244 | cnum++; | |
245 | if(cnum >= NCLUSTERS) { | |
246 | nent = 1UL << (cnum - NCLUSTERS); | |
247 | cnum = NCLUSTERS - 1; | |
248 | } | |
249 | ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum); | |
250 | iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); | |
251 | do { | |
252 | iopte_val(*iopte) = 0UL; | |
253 | iopte += 1 << cnum; | |
254 | } while(--nent); | |
255 | ||
256 | /* If the global flush might not have caught this entry, | |
257 | * adjust the flush point such that we will flush before | |
258 | * ever trying to reuse it. | |
259 | */ | |
260 | #define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) | |
261 | if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush)) | |
262 | iommu->alloc_info[cnum].flush = ent; | |
263 | #undef between | |
264 | } | |
265 | ||
266 | /* We allocate consistent mappings from the end of cluster zero. */ | |
267 | static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages) | |
268 | { | |
269 | iopte_t *iopte; | |
270 | ||
271 | iopte = iommu->page_table + (1 * CLUSTER_NPAGES); | |
272 | while (iopte > iommu->page_table) { | |
273 | iopte--; | |
274 | if (!(iopte_val(*iopte) & IOPTE_VALID)) { | |
275 | unsigned long tmp = npages; | |
276 | ||
277 | while (--tmp) { | |
278 | iopte--; | |
279 | if (iopte_val(*iopte) & IOPTE_VALID) | |
280 | break; | |
281 | } | |
282 | if (tmp == 0) { | |
283 | u32 entry = (iopte - iommu->page_table); | |
284 | ||
285 | if (entry < iommu->lowest_consistent_map) | |
286 | iommu->lowest_consistent_map = entry; | |
287 | return iopte; | |
288 | } | |
289 | } | |
290 | } | |
291 | return NULL; | |
292 | } | |
293 | ||
294 | static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) | |
295 | { | |
296 | iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); | |
297 | ||
298 | if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) { | |
299 | iopte_t *walk = iopte + npages; | |
300 | iopte_t *limit; | |
301 | ||
302 | limit = iommu->page_table + CLUSTER_NPAGES; | |
303 | while (walk < limit) { | |
304 | if (iopte_val(*walk) != 0UL) | |
305 | break; | |
306 | walk++; | |
307 | } | |
308 | iommu->lowest_consistent_map = | |
309 | (walk - iommu->page_table); | |
310 | } | |
311 | ||
312 | while (npages--) | |
313 | *iopte++ = __iopte(0UL); | |
314 | } | |
315 | ||
316 | void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr) | |
317 | { | |
318 | unsigned long order, first_page, flags; | |
319 | struct sbus_iommu *iommu; | |
320 | iopte_t *iopte; | |
321 | void *ret; | |
322 | int npages; | |
323 | ||
324 | if (size <= 0 || sdev == NULL || dvma_addr == NULL) | |
325 | return NULL; | |
326 | ||
327 | size = IO_PAGE_ALIGN(size); | |
328 | order = get_order(size); | |
329 | if (order >= 10) | |
330 | return NULL; | |
f3d48f03 | 331 | first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order); |
1da177e4 LT |
332 | if (first_page == 0UL) |
333 | return NULL; | |
334 | memset((char *)first_page, 0, PAGE_SIZE << order); | |
335 | ||
336 | iommu = sdev->bus->iommu; | |
337 | ||
338 | spin_lock_irqsave(&iommu->lock, flags); | |
339 | iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); | |
340 | if (iopte == NULL) { | |
341 | spin_unlock_irqrestore(&iommu->lock, flags); | |
342 | free_pages(first_page, order); | |
343 | return NULL; | |
344 | } | |
345 | ||
346 | /* Ok, we're committed at this point. */ | |
347 | *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
348 | ret = (void *) first_page; | |
349 | npages = size >> IO_PAGE_SHIFT; | |
350 | while (npages--) { | |
351 | *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE | | |
352 | (__pa(first_page) & IOPTE_PAGE)); | |
353 | first_page += IO_PAGE_SIZE; | |
354 | } | |
355 | iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT); | |
356 | spin_unlock_irqrestore(&iommu->lock, flags); | |
357 | ||
358 | return ret; | |
359 | } | |
360 | ||
361 | void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma) | |
362 | { | |
363 | unsigned long order, npages; | |
364 | struct sbus_iommu *iommu; | |
365 | ||
366 | if (size <= 0 || sdev == NULL || cpu == NULL) | |
367 | return; | |
368 | ||
369 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | |
370 | iommu = sdev->bus->iommu; | |
371 | ||
372 | spin_lock_irq(&iommu->lock); | |
373 | free_consistent_cluster(iommu, dvma, npages); | |
374 | iommu_flush(iommu, dvma, npages); | |
375 | spin_unlock_irq(&iommu->lock); | |
376 | ||
377 | order = get_order(size); | |
378 | if (order < 10) | |
379 | free_pages((unsigned long)cpu, order); | |
380 | } | |
381 | ||
382 | dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir) | |
383 | { | |
384 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
385 | unsigned long npages, pbase, flags; | |
386 | iopte_t *iopte; | |
387 | u32 dma_base, offset; | |
388 | unsigned long iopte_bits; | |
389 | ||
390 | if (dir == SBUS_DMA_NONE) | |
391 | BUG(); | |
392 | ||
393 | pbase = (unsigned long) ptr; | |
394 | offset = (u32) (pbase & ~IO_PAGE_MASK); | |
395 | size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK)); | |
396 | pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK); | |
397 | ||
398 | spin_lock_irqsave(&iommu->lock, flags); | |
399 | npages = size >> IO_PAGE_SHIFT; | |
400 | iopte = alloc_streaming_cluster(iommu, npages); | |
401 | if (iopte == NULL) | |
402 | goto bad; | |
403 | dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
404 | npages = size >> IO_PAGE_SHIFT; | |
405 | iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; | |
406 | if (dir != SBUS_DMA_TODEVICE) | |
407 | iopte_bits |= IOPTE_WRITE; | |
408 | while (npages--) { | |
409 | *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE)); | |
410 | pbase += IO_PAGE_SIZE; | |
411 | } | |
412 | npages = size >> IO_PAGE_SHIFT; | |
413 | spin_unlock_irqrestore(&iommu->lock, flags); | |
414 | ||
415 | return (dma_base | offset); | |
416 | ||
417 | bad: | |
418 | spin_unlock_irqrestore(&iommu->lock, flags); | |
419 | BUG(); | |
420 | return 0; | |
421 | } | |
422 | ||
423 | void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction) | |
424 | { | |
425 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
426 | u32 dma_base = dma_addr & IO_PAGE_MASK; | |
427 | unsigned long flags; | |
428 | ||
429 | size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base); | |
430 | ||
431 | spin_lock_irqsave(&iommu->lock, flags); | |
432 | free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT); | |
7c963ad1 | 433 | sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
434 | spin_unlock_irqrestore(&iommu->lock, flags); |
435 | } | |
436 | ||
437 | #define SG_ENT_PHYS_ADDRESS(SG) \ | |
438 | (__pa(page_address((SG)->page)) + (SG)->offset) | |
439 | ||
440 | static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits) | |
441 | { | |
442 | struct scatterlist *dma_sg = sg; | |
443 | struct scatterlist *sg_end = sg + nelems; | |
444 | int i; | |
445 | ||
446 | for (i = 0; i < nused; i++) { | |
447 | unsigned long pteval = ~0UL; | |
448 | u32 dma_npages; | |
449 | ||
450 | dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + | |
451 | dma_sg->dma_length + | |
452 | ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; | |
453 | do { | |
454 | unsigned long offset; | |
455 | signed int len; | |
456 | ||
457 | /* If we are here, we know we have at least one | |
458 | * more page to map. So walk forward until we | |
459 | * hit a page crossing, and begin creating new | |
460 | * mappings from that spot. | |
461 | */ | |
462 | for (;;) { | |
463 | unsigned long tmp; | |
464 | ||
465 | tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg); | |
466 | len = sg->length; | |
467 | if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { | |
468 | pteval = tmp & IO_PAGE_MASK; | |
469 | offset = tmp & (IO_PAGE_SIZE - 1UL); | |
470 | break; | |
471 | } | |
472 | if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { | |
473 | pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; | |
474 | offset = 0UL; | |
475 | len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); | |
476 | break; | |
477 | } | |
478 | sg++; | |
479 | } | |
480 | ||
481 | pteval = ((pteval & IOPTE_PAGE) | iopte_bits); | |
482 | while (len > 0) { | |
483 | *iopte++ = __iopte(pteval); | |
484 | pteval += IO_PAGE_SIZE; | |
485 | len -= (IO_PAGE_SIZE - offset); | |
486 | offset = 0; | |
487 | dma_npages--; | |
488 | } | |
489 | ||
490 | pteval = (pteval & IOPTE_PAGE) + len; | |
491 | sg++; | |
492 | ||
493 | /* Skip over any tail mappings we've fully mapped, | |
494 | * adjusting pteval along the way. Stop when we | |
495 | * detect a page crossing event. | |
496 | */ | |
497 | while (sg < sg_end && | |
498 | (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && | |
499 | (pteval == SG_ENT_PHYS_ADDRESS(sg)) && | |
500 | ((pteval ^ | |
501 | (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { | |
502 | pteval += sg->length; | |
503 | sg++; | |
504 | } | |
505 | if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) | |
506 | pteval = ~0UL; | |
507 | } while (dma_npages != 0); | |
508 | dma_sg++; | |
509 | } | |
510 | } | |
511 | ||
512 | int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir) | |
513 | { | |
514 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
515 | unsigned long flags, npages; | |
516 | iopte_t *iopte; | |
517 | u32 dma_base; | |
518 | struct scatterlist *sgtmp; | |
519 | int used; | |
520 | unsigned long iopte_bits; | |
521 | ||
522 | if (dir == SBUS_DMA_NONE) | |
523 | BUG(); | |
524 | ||
525 | /* Fast path single entry scatterlists. */ | |
526 | if (nents == 1) { | |
527 | sg->dma_address = | |
528 | sbus_map_single(sdev, | |
529 | (page_address(sg->page) + sg->offset), | |
530 | sg->length, dir); | |
531 | sg->dma_length = sg->length; | |
532 | return 1; | |
533 | } | |
534 | ||
535 | npages = prepare_sg(sg, nents); | |
536 | ||
537 | spin_lock_irqsave(&iommu->lock, flags); | |
538 | iopte = alloc_streaming_cluster(iommu, npages); | |
539 | if (iopte == NULL) | |
540 | goto bad; | |
541 | dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); | |
542 | ||
543 | /* Normalize DVMA addresses. */ | |
544 | sgtmp = sg; | |
545 | used = nents; | |
546 | ||
547 | while (used && sgtmp->dma_length) { | |
548 | sgtmp->dma_address += dma_base; | |
549 | sgtmp++; | |
550 | used--; | |
551 | } | |
552 | used = nents - used; | |
553 | ||
554 | iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; | |
555 | if (dir != SBUS_DMA_TODEVICE) | |
556 | iopte_bits |= IOPTE_WRITE; | |
557 | ||
558 | fill_sg(iopte, sg, used, nents, iopte_bits); | |
559 | #ifdef VERIFY_SG | |
560 | verify_sglist(sg, nents, iopte, npages); | |
561 | #endif | |
562 | spin_unlock_irqrestore(&iommu->lock, flags); | |
563 | ||
564 | return used; | |
565 | ||
566 | bad: | |
567 | spin_unlock_irqrestore(&iommu->lock, flags); | |
568 | BUG(); | |
569 | return 0; | |
570 | } | |
571 | ||
572 | void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
573 | { | |
574 | unsigned long size, flags; | |
575 | struct sbus_iommu *iommu; | |
576 | u32 dvma_base; | |
577 | int i; | |
578 | ||
579 | /* Fast path single entry scatterlists. */ | |
580 | if (nents == 1) { | |
581 | sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction); | |
582 | return; | |
583 | } | |
584 | ||
585 | dvma_base = sg[0].dma_address & IO_PAGE_MASK; | |
586 | for (i = 0; i < nents; i++) { | |
587 | if (sg[i].dma_length == 0) | |
588 | break; | |
589 | } | |
590 | i--; | |
591 | size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base; | |
592 | ||
593 | iommu = sdev->bus->iommu; | |
594 | spin_lock_irqsave(&iommu->lock, flags); | |
595 | free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT); | |
7c963ad1 | 596 | sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
597 | spin_unlock_irqrestore(&iommu->lock, flags); |
598 | } | |
599 | ||
600 | void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) | |
601 | { | |
602 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
603 | unsigned long flags; | |
604 | ||
605 | size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK)); | |
606 | ||
607 | spin_lock_irqsave(&iommu->lock, flags); | |
7c963ad1 | 608 | sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
609 | spin_unlock_irqrestore(&iommu->lock, flags); |
610 | } | |
611 | ||
612 | void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) | |
613 | { | |
614 | } | |
615 | ||
616 | void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
617 | { | |
618 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
619 | unsigned long flags, size; | |
620 | u32 base; | |
621 | int i; | |
622 | ||
623 | base = sg[0].dma_address & IO_PAGE_MASK; | |
624 | for (i = 0; i < nents; i++) { | |
625 | if (sg[i].dma_length == 0) | |
626 | break; | |
627 | } | |
628 | i--; | |
629 | size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base; | |
630 | ||
631 | spin_lock_irqsave(&iommu->lock, flags); | |
7c963ad1 | 632 | sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, direction); |
1da177e4 LT |
633 | spin_unlock_irqrestore(&iommu->lock, flags); |
634 | } | |
635 | ||
636 | void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
637 | { | |
638 | } | |
639 | ||
640 | /* Enable 64-bit DVMA mode for the given device. */ | |
641 | void sbus_set_sbus64(struct sbus_dev *sdev, int bursts) | |
642 | { | |
643 | struct sbus_iommu *iommu = sdev->bus->iommu; | |
644 | int slot = sdev->slot; | |
645 | unsigned long cfg_reg; | |
646 | u64 val; | |
647 | ||
648 | cfg_reg = iommu->sbus_control_reg; | |
649 | switch (slot) { | |
650 | case 0: | |
651 | cfg_reg += 0x20UL; | |
652 | break; | |
653 | case 1: | |
654 | cfg_reg += 0x28UL; | |
655 | break; | |
656 | case 2: | |
657 | cfg_reg += 0x30UL; | |
658 | break; | |
659 | case 3: | |
660 | cfg_reg += 0x38UL; | |
661 | break; | |
662 | case 13: | |
663 | cfg_reg += 0x40UL; | |
664 | break; | |
665 | case 14: | |
666 | cfg_reg += 0x48UL; | |
667 | break; | |
668 | case 15: | |
669 | cfg_reg += 0x50UL; | |
670 | break; | |
671 | ||
672 | default: | |
673 | return; | |
674 | }; | |
675 | ||
676 | val = upa_readq(cfg_reg); | |
677 | if (val & (1UL << 14UL)) { | |
678 | /* Extended transfer mode already enabled. */ | |
679 | return; | |
680 | } | |
681 | ||
682 | val |= (1UL << 14UL); | |
683 | ||
684 | if (bursts & DMA_BURST8) | |
685 | val |= (1UL << 1UL); | |
686 | if (bursts & DMA_BURST16) | |
687 | val |= (1UL << 2UL); | |
688 | if (bursts & DMA_BURST32) | |
689 | val |= (1UL << 3UL); | |
690 | if (bursts & DMA_BURST64) | |
691 | val |= (1UL << 4UL); | |
692 | upa_writeq(val, cfg_reg); | |
693 | } | |
694 | ||
1da177e4 LT |
695 | /* INO number to IMAP register offset for SYSIO external IRQ's. |
696 | * This should conform to both Sunfire/Wildfire server and Fusion | |
697 | * desktop designs. | |
698 | */ | |
699 | #define SYSIO_IMAP_SLOT0 0x2c04UL | |
700 | #define SYSIO_IMAP_SLOT1 0x2c0cUL | |
701 | #define SYSIO_IMAP_SLOT2 0x2c14UL | |
702 | #define SYSIO_IMAP_SLOT3 0x2c1cUL | |
703 | #define SYSIO_IMAP_SCSI 0x3004UL | |
704 | #define SYSIO_IMAP_ETH 0x300cUL | |
705 | #define SYSIO_IMAP_BPP 0x3014UL | |
706 | #define SYSIO_IMAP_AUDIO 0x301cUL | |
707 | #define SYSIO_IMAP_PFAIL 0x3024UL | |
708 | #define SYSIO_IMAP_KMS 0x302cUL | |
709 | #define SYSIO_IMAP_FLPY 0x3034UL | |
710 | #define SYSIO_IMAP_SHW 0x303cUL | |
711 | #define SYSIO_IMAP_KBD 0x3044UL | |
712 | #define SYSIO_IMAP_MS 0x304cUL | |
713 | #define SYSIO_IMAP_SER 0x3054UL | |
714 | #define SYSIO_IMAP_TIM0 0x3064UL | |
715 | #define SYSIO_IMAP_TIM1 0x306cUL | |
716 | #define SYSIO_IMAP_UE 0x3074UL | |
717 | #define SYSIO_IMAP_CE 0x307cUL | |
718 | #define SYSIO_IMAP_SBERR 0x3084UL | |
719 | #define SYSIO_IMAP_PMGMT 0x308cUL | |
720 | #define SYSIO_IMAP_GFX 0x3094UL | |
721 | #define SYSIO_IMAP_EUPA 0x309cUL | |
722 | ||
723 | #define bogon ((unsigned long) -1) | |
724 | static unsigned long sysio_irq_offsets[] = { | |
725 | /* SBUS Slot 0 --> 3, level 1 --> 7 */ | |
726 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
727 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
728 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
729 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
730 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
731 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
732 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
733 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
734 | ||
735 | /* Onboard devices (not relevant/used on SunFire). */ | |
736 | SYSIO_IMAP_SCSI, | |
737 | SYSIO_IMAP_ETH, | |
738 | SYSIO_IMAP_BPP, | |
739 | bogon, | |
740 | SYSIO_IMAP_AUDIO, | |
741 | SYSIO_IMAP_PFAIL, | |
742 | bogon, | |
743 | bogon, | |
744 | SYSIO_IMAP_KMS, | |
745 | SYSIO_IMAP_FLPY, | |
746 | SYSIO_IMAP_SHW, | |
747 | SYSIO_IMAP_KBD, | |
748 | SYSIO_IMAP_MS, | |
749 | SYSIO_IMAP_SER, | |
750 | bogon, | |
751 | bogon, | |
752 | SYSIO_IMAP_TIM0, | |
753 | SYSIO_IMAP_TIM1, | |
754 | bogon, | |
755 | bogon, | |
756 | SYSIO_IMAP_UE, | |
757 | SYSIO_IMAP_CE, | |
758 | SYSIO_IMAP_SBERR, | |
759 | SYSIO_IMAP_PMGMT, | |
760 | }; | |
761 | ||
762 | #undef bogon | |
763 | ||
84c1a13a | 764 | #define NUM_SYSIO_OFFSETS ARRAY_SIZE(sysio_irq_offsets) |
1da177e4 LT |
765 | |
766 | /* Convert Interrupt Mapping register pointer to associated | |
767 | * Interrupt Clear register pointer, SYSIO specific version. | |
768 | */ | |
769 | #define SYSIO_ICLR_UNUSED0 0x3400UL | |
770 | #define SYSIO_ICLR_SLOT0 0x340cUL | |
771 | #define SYSIO_ICLR_SLOT1 0x344cUL | |
772 | #define SYSIO_ICLR_SLOT2 0x348cUL | |
773 | #define SYSIO_ICLR_SLOT3 0x34ccUL | |
774 | static unsigned long sysio_imap_to_iclr(unsigned long imap) | |
775 | { | |
776 | unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0; | |
777 | return imap + diff; | |
778 | } | |
779 | ||
780 | unsigned int sbus_build_irq(void *buscookie, unsigned int ino) | |
781 | { | |
782 | struct sbus_bus *sbus = (struct sbus_bus *)buscookie; | |
783 | struct sbus_iommu *iommu = sbus->iommu; | |
784 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
785 | unsigned long imap, iclr; | |
37cdcd9e | 786 | int sbus_level = 0; |
1da177e4 LT |
787 | |
788 | imap = sysio_irq_offsets[ino]; | |
789 | if (imap == ((unsigned long)-1)) { | |
37cdcd9e DM |
790 | prom_printf("get_irq_translations: Bad SYSIO INO[%x]\n", |
791 | ino); | |
1da177e4 LT |
792 | prom_halt(); |
793 | } | |
794 | imap += reg_base; | |
795 | ||
796 | /* SYSIO inconsistency. For external SLOTS, we have to select | |
797 | * the right ICLR register based upon the lower SBUS irq level | |
798 | * bits. | |
799 | */ | |
800 | if (ino >= 0x20) { | |
801 | iclr = sysio_imap_to_iclr(imap); | |
802 | } else { | |
803 | int sbus_slot = (ino & 0x18)>>3; | |
804 | ||
805 | sbus_level = ino & 0x7; | |
806 | ||
807 | switch(sbus_slot) { | |
808 | case 0: | |
809 | iclr = reg_base + SYSIO_ICLR_SLOT0; | |
810 | break; | |
811 | case 1: | |
812 | iclr = reg_base + SYSIO_ICLR_SLOT1; | |
813 | break; | |
814 | case 2: | |
815 | iclr = reg_base + SYSIO_ICLR_SLOT2; | |
816 | break; | |
817 | default: | |
818 | case 3: | |
819 | iclr = reg_base + SYSIO_ICLR_SLOT3; | |
820 | break; | |
821 | }; | |
822 | ||
823 | iclr += ((unsigned long)sbus_level - 1UL) * 8UL; | |
824 | } | |
e18e2a00 | 825 | return build_irq(sbus_level, iclr, imap); |
1da177e4 LT |
826 | } |
827 | ||
828 | /* Error interrupt handling. */ | |
829 | #define SYSIO_UE_AFSR 0x0030UL | |
830 | #define SYSIO_UE_AFAR 0x0038UL | |
831 | #define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
832 | #define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
833 | #define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
834 | #define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ | |
835 | #define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
836 | #define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
837 | #define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
838 | #define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */ | |
839 | #define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
840 | #define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
841 | #define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 842 | static irqreturn_t sysio_ue_handler(int irq, void *dev_id) |
1da177e4 LT |
843 | { |
844 | struct sbus_bus *sbus = dev_id; | |
845 | struct sbus_iommu *iommu = sbus->iommu; | |
846 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
847 | unsigned long afsr_reg, afar_reg; | |
848 | unsigned long afsr, afar, error_bits; | |
849 | int reported; | |
850 | ||
851 | afsr_reg = reg_base + SYSIO_UE_AFSR; | |
852 | afar_reg = reg_base + SYSIO_UE_AFAR; | |
853 | ||
854 | /* Latch error status. */ | |
855 | afsr = upa_readq(afsr_reg); | |
856 | afar = upa_readq(afar_reg); | |
857 | ||
858 | /* Clear primary/secondary error status bits. */ | |
859 | error_bits = afsr & | |
860 | (SYSIO_UEAFSR_PPIO | SYSIO_UEAFSR_PDRD | SYSIO_UEAFSR_PDWR | | |
861 | SYSIO_UEAFSR_SPIO | SYSIO_UEAFSR_SDRD | SYSIO_UEAFSR_SDWR); | |
862 | upa_writeq(error_bits, afsr_reg); | |
863 | ||
864 | /* Log the error. */ | |
865 | printk("SYSIO[%x]: Uncorrectable ECC Error, primary error type[%s]\n", | |
866 | sbus->portid, | |
867 | (((error_bits & SYSIO_UEAFSR_PPIO) ? | |
868 | "PIO" : | |
869 | ((error_bits & SYSIO_UEAFSR_PDRD) ? | |
870 | "DVMA Read" : | |
871 | ((error_bits & SYSIO_UEAFSR_PDWR) ? | |
872 | "DVMA Write" : "???"))))); | |
873 | printk("SYSIO[%x]: DOFF[%lx] SIZE[%lx] MID[%lx]\n", | |
874 | sbus->portid, | |
875 | (afsr & SYSIO_UEAFSR_DOFF) >> 45UL, | |
876 | (afsr & SYSIO_UEAFSR_SIZE) >> 42UL, | |
877 | (afsr & SYSIO_UEAFSR_MID) >> 37UL); | |
878 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
879 | printk("SYSIO[%x]: Secondary UE errors [", sbus->portid); | |
880 | reported = 0; | |
881 | if (afsr & SYSIO_UEAFSR_SPIO) { | |
882 | reported++; | |
883 | printk("(PIO)"); | |
884 | } | |
885 | if (afsr & SYSIO_UEAFSR_SDRD) { | |
886 | reported++; | |
887 | printk("(DVMA Read)"); | |
888 | } | |
889 | if (afsr & SYSIO_UEAFSR_SDWR) { | |
890 | reported++; | |
891 | printk("(DVMA Write)"); | |
892 | } | |
893 | if (!reported) | |
894 | printk("(none)"); | |
895 | printk("]\n"); | |
896 | ||
897 | return IRQ_HANDLED; | |
898 | } | |
899 | ||
900 | #define SYSIO_CE_AFSR 0x0040UL | |
901 | #define SYSIO_CE_AFAR 0x0048UL | |
902 | #define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
903 | #define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
904 | #define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
905 | #define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */ | |
906 | #define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
907 | #define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
908 | #define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */ | |
909 | #define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */ | |
910 | #define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */ | |
911 | #define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
912 | #define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
913 | #define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 914 | static irqreturn_t sysio_ce_handler(int irq, void *dev_id) |
1da177e4 LT |
915 | { |
916 | struct sbus_bus *sbus = dev_id; | |
917 | struct sbus_iommu *iommu = sbus->iommu; | |
918 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
919 | unsigned long afsr_reg, afar_reg; | |
920 | unsigned long afsr, afar, error_bits; | |
921 | int reported; | |
922 | ||
923 | afsr_reg = reg_base + SYSIO_CE_AFSR; | |
924 | afar_reg = reg_base + SYSIO_CE_AFAR; | |
925 | ||
926 | /* Latch error status. */ | |
927 | afsr = upa_readq(afsr_reg); | |
928 | afar = upa_readq(afar_reg); | |
929 | ||
930 | /* Clear primary/secondary error status bits. */ | |
931 | error_bits = afsr & | |
932 | (SYSIO_CEAFSR_PPIO | SYSIO_CEAFSR_PDRD | SYSIO_CEAFSR_PDWR | | |
933 | SYSIO_CEAFSR_SPIO | SYSIO_CEAFSR_SDRD | SYSIO_CEAFSR_SDWR); | |
934 | upa_writeq(error_bits, afsr_reg); | |
935 | ||
936 | printk("SYSIO[%x]: Correctable ECC Error, primary error type[%s]\n", | |
937 | sbus->portid, | |
938 | (((error_bits & SYSIO_CEAFSR_PPIO) ? | |
939 | "PIO" : | |
940 | ((error_bits & SYSIO_CEAFSR_PDRD) ? | |
941 | "DVMA Read" : | |
942 | ((error_bits & SYSIO_CEAFSR_PDWR) ? | |
943 | "DVMA Write" : "???"))))); | |
944 | ||
945 | /* XXX Use syndrome and afar to print out module string just like | |
946 | * XXX UDB CE trap handler does... -DaveM | |
947 | */ | |
948 | printk("SYSIO[%x]: DOFF[%lx] ECC Syndrome[%lx] Size[%lx] MID[%lx]\n", | |
949 | sbus->portid, | |
950 | (afsr & SYSIO_CEAFSR_DOFF) >> 45UL, | |
951 | (afsr & SYSIO_CEAFSR_ESYND) >> 48UL, | |
952 | (afsr & SYSIO_CEAFSR_SIZE) >> 42UL, | |
953 | (afsr & SYSIO_CEAFSR_MID) >> 37UL); | |
954 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
955 | ||
956 | printk("SYSIO[%x]: Secondary CE errors [", sbus->portid); | |
957 | reported = 0; | |
958 | if (afsr & SYSIO_CEAFSR_SPIO) { | |
959 | reported++; | |
960 | printk("(PIO)"); | |
961 | } | |
962 | if (afsr & SYSIO_CEAFSR_SDRD) { | |
963 | reported++; | |
964 | printk("(DVMA Read)"); | |
965 | } | |
966 | if (afsr & SYSIO_CEAFSR_SDWR) { | |
967 | reported++; | |
968 | printk("(DVMA Write)"); | |
969 | } | |
970 | if (!reported) | |
971 | printk("(none)"); | |
972 | printk("]\n"); | |
973 | ||
974 | return IRQ_HANDLED; | |
975 | } | |
976 | ||
977 | #define SYSIO_SBUS_AFSR 0x2010UL | |
978 | #define SYSIO_SBUS_AFAR 0x2018UL | |
979 | #define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */ | |
980 | #define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */ | |
981 | #define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */ | |
982 | #define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */ | |
983 | #define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */ | |
984 | #define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */ | |
985 | #define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
986 | #define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */ | |
987 | #define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */ | |
988 | #define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */ | |
989 | #define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */ | |
990 | #define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 991 | static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id) |
1da177e4 LT |
992 | { |
993 | struct sbus_bus *sbus = dev_id; | |
994 | struct sbus_iommu *iommu = sbus->iommu; | |
995 | unsigned long afsr_reg, afar_reg, reg_base; | |
996 | unsigned long afsr, afar, error_bits; | |
997 | int reported; | |
998 | ||
999 | reg_base = iommu->sbus_control_reg - 0x2000UL; | |
1000 | afsr_reg = reg_base + SYSIO_SBUS_AFSR; | |
1001 | afar_reg = reg_base + SYSIO_SBUS_AFAR; | |
1002 | ||
1003 | afsr = upa_readq(afsr_reg); | |
1004 | afar = upa_readq(afar_reg); | |
1005 | ||
1006 | /* Clear primary/secondary error status bits. */ | |
1007 | error_bits = afsr & | |
1008 | (SYSIO_SBAFSR_PLE | SYSIO_SBAFSR_PTO | SYSIO_SBAFSR_PBERR | | |
1009 | SYSIO_SBAFSR_SLE | SYSIO_SBAFSR_STO | SYSIO_SBAFSR_SBERR); | |
1010 | upa_writeq(error_bits, afsr_reg); | |
1011 | ||
1012 | /* Log the error. */ | |
1013 | printk("SYSIO[%x]: SBUS Error, primary error type[%s] read(%d)\n", | |
1014 | sbus->portid, | |
1015 | (((error_bits & SYSIO_SBAFSR_PLE) ? | |
1016 | "Late PIO Error" : | |
1017 | ((error_bits & SYSIO_SBAFSR_PTO) ? | |
1018 | "Time Out" : | |
1019 | ((error_bits & SYSIO_SBAFSR_PBERR) ? | |
1020 | "Error Ack" : "???")))), | |
1021 | (afsr & SYSIO_SBAFSR_RD) ? 1 : 0); | |
1022 | printk("SYSIO[%x]: size[%lx] MID[%lx]\n", | |
1023 | sbus->portid, | |
1024 | (afsr & SYSIO_SBAFSR_SIZE) >> 42UL, | |
1025 | (afsr & SYSIO_SBAFSR_MID) >> 37UL); | |
1026 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
1027 | printk("SYSIO[%x]: Secondary SBUS errors [", sbus->portid); | |
1028 | reported = 0; | |
1029 | if (afsr & SYSIO_SBAFSR_SLE) { | |
1030 | reported++; | |
1031 | printk("(Late PIO Error)"); | |
1032 | } | |
1033 | if (afsr & SYSIO_SBAFSR_STO) { | |
1034 | reported++; | |
1035 | printk("(Time Out)"); | |
1036 | } | |
1037 | if (afsr & SYSIO_SBAFSR_SBERR) { | |
1038 | reported++; | |
1039 | printk("(Error Ack)"); | |
1040 | } | |
1041 | if (!reported) | |
1042 | printk("(none)"); | |
1043 | printk("]\n"); | |
1044 | ||
1045 | /* XXX check iommu/strbuf for further error status XXX */ | |
1046 | ||
1047 | return IRQ_HANDLED; | |
1048 | } | |
1049 | ||
1050 | #define ECC_CONTROL 0x0020UL | |
1051 | #define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */ | |
1052 | #define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */ | |
1053 | #define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */ | |
1054 | ||
1055 | #define SYSIO_UE_INO 0x34 | |
1056 | #define SYSIO_CE_INO 0x35 | |
1057 | #define SYSIO_SBUSERR_INO 0x36 | |
1058 | ||
1059 | static void __init sysio_register_error_handlers(struct sbus_bus *sbus) | |
1060 | { | |
1061 | struct sbus_iommu *iommu = sbus->iommu; | |
1062 | unsigned long reg_base = iommu->sbus_control_reg - 0x2000UL; | |
1063 | unsigned int irq; | |
1064 | u64 control; | |
1065 | ||
1066 | irq = sbus_build_irq(sbus, SYSIO_UE_INO); | |
1067 | if (request_irq(irq, sysio_ue_handler, | |
d356d7f4 | 1068 | IRQF_SHARED, "SYSIO UE", sbus) < 0) { |
1da177e4 LT |
1069 | prom_printf("SYSIO[%x]: Cannot register UE interrupt.\n", |
1070 | sbus->portid); | |
1071 | prom_halt(); | |
1072 | } | |
1073 | ||
1074 | irq = sbus_build_irq(sbus, SYSIO_CE_INO); | |
1075 | if (request_irq(irq, sysio_ce_handler, | |
d356d7f4 | 1076 | IRQF_SHARED, "SYSIO CE", sbus) < 0) { |
1da177e4 LT |
1077 | prom_printf("SYSIO[%x]: Cannot register CE interrupt.\n", |
1078 | sbus->portid); | |
1079 | prom_halt(); | |
1080 | } | |
1081 | ||
1082 | irq = sbus_build_irq(sbus, SYSIO_SBUSERR_INO); | |
1083 | if (request_irq(irq, sysio_sbus_error_handler, | |
d356d7f4 | 1084 | IRQF_SHARED, "SYSIO SBUS Error", sbus) < 0) { |
1da177e4 LT |
1085 | prom_printf("SYSIO[%x]: Cannot register SBUS Error interrupt.\n", |
1086 | sbus->portid); | |
1087 | prom_halt(); | |
1088 | } | |
1089 | ||
1090 | /* Now turn the error interrupts on and also enable ECC checking. */ | |
1091 | upa_writeq((SYSIO_ECNTRL_ECCEN | | |
1092 | SYSIO_ECNTRL_UEEN | | |
1093 | SYSIO_ECNTRL_CEEN), | |
1094 | reg_base + ECC_CONTROL); | |
1095 | ||
1096 | control = upa_readq(iommu->sbus_control_reg); | |
1097 | control |= 0x100UL; /* SBUS Error Interrupt Enable */ | |
1098 | upa_writeq(control, iommu->sbus_control_reg); | |
1099 | } | |
1100 | ||
1101 | /* Boot time initialization. */ | |
576c352e | 1102 | static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
1da177e4 | 1103 | { |
25c7581b DM |
1104 | struct linux_prom64_registers *pr; |
1105 | struct device_node *dp; | |
1da177e4 LT |
1106 | struct sbus_iommu *iommu; |
1107 | unsigned long regs, tsb_base; | |
1108 | u64 control; | |
25c7581b DM |
1109 | int i; |
1110 | ||
1111 | dp = of_find_node_by_phandle(__node); | |
1da177e4 | 1112 | |
25c7581b | 1113 | sbus->portid = of_getintprop_default(dp, "upa-portid", -1); |
1da177e4 | 1114 | |
25c7581b DM |
1115 | pr = of_get_property(dp, "reg", NULL); |
1116 | if (!pr) { | |
1da177e4 LT |
1117 | prom_printf("sbus_iommu_init: Cannot map SYSIO control registers.\n"); |
1118 | prom_halt(); | |
1119 | } | |
25c7581b | 1120 | regs = pr->phys_addr; |
1da177e4 LT |
1121 | |
1122 | iommu = kmalloc(sizeof(*iommu) + SMP_CACHE_BYTES, GFP_ATOMIC); | |
1123 | if (iommu == NULL) { | |
1124 | prom_printf("sbus_iommu_init: Fatal error, kmalloc(iommu) failed\n"); | |
1125 | prom_halt(); | |
1126 | } | |
1127 | ||
1128 | /* Align on E$ line boundary. */ | |
1129 | iommu = (struct sbus_iommu *) | |
1130 | (((unsigned long)iommu + (SMP_CACHE_BYTES - 1UL)) & | |
1131 | ~(SMP_CACHE_BYTES - 1UL)); | |
1132 | ||
1133 | memset(iommu, 0, sizeof(*iommu)); | |
1134 | ||
1135 | /* We start with no consistent mappings. */ | |
1136 | iommu->lowest_consistent_map = CLUSTER_NPAGES; | |
1137 | ||
1138 | for (i = 0; i < NCLUSTERS; i++) { | |
1139 | iommu->alloc_info[i].flush = 0; | |
1140 | iommu->alloc_info[i].next = 0; | |
1141 | } | |
1142 | ||
1143 | /* Setup spinlock. */ | |
1144 | spin_lock_init(&iommu->lock); | |
1145 | ||
1146 | /* Init register offsets. */ | |
1147 | iommu->iommu_regs = regs + SYSIO_IOMMUREG_BASE; | |
1148 | iommu->strbuf_regs = regs + SYSIO_STRBUFREG_BASE; | |
1149 | ||
1150 | /* The SYSIO SBUS control register is used for dummy reads | |
1151 | * in order to ensure write completion. | |
1152 | */ | |
1153 | iommu->sbus_control_reg = regs + 0x2000UL; | |
1154 | ||
1155 | /* Link into SYSIO software state. */ | |
1156 | sbus->iommu = iommu; | |
1157 | ||
1158 | printk("SYSIO: UPA portID %x, at %016lx\n", | |
1159 | sbus->portid, regs); | |
1160 | ||
1161 | /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ | |
1162 | control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL); | |
1163 | control = ((7UL << 16UL) | | |
1164 | (0UL << 2UL) | | |
1165 | (1UL << 1UL) | | |
1166 | (1UL << 0UL)); | |
1167 | ||
1168 | /* Using the above configuration we need 1MB iommu page | |
1169 | * table (128K ioptes * 8 bytes per iopte). This is | |
1170 | * page order 7 on UltraSparc. | |
1171 | */ | |
1172 | tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE)); | |
1173 | if (tsb_base == 0UL) { | |
1174 | prom_printf("sbus_iommu_init: Fatal error, cannot alloc TSB table.\n"); | |
1175 | prom_halt(); | |
1176 | } | |
1177 | ||
1178 | iommu->page_table = (iopte_t *) tsb_base; | |
1179 | memset(iommu->page_table, 0, IO_TSB_SIZE); | |
1180 | ||
1181 | upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL); | |
1182 | ||
1183 | /* Clean out any cruft in the IOMMU using | |
1184 | * diagnostic accesses. | |
1185 | */ | |
1186 | for (i = 0; i < 16; i++) { | |
1187 | unsigned long dram = iommu->iommu_regs + IOMMU_DRAMDIAG; | |
1188 | unsigned long tag = iommu->iommu_regs + IOMMU_TAGDIAG; | |
1189 | ||
1190 | dram += (unsigned long)i * 8UL; | |
1191 | tag += (unsigned long)i * 8UL; | |
1192 | upa_writeq(0, dram); | |
1193 | upa_writeq(0, tag); | |
1194 | } | |
1195 | upa_readq(iommu->sbus_control_reg); | |
1196 | ||
1197 | /* Give the TSB to SYSIO. */ | |
1198 | upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE); | |
1199 | ||
1200 | /* Setup streaming buffer, DE=1 SB_EN=1 */ | |
1201 | control = (1UL << 1UL) | (1UL << 0UL); | |
1202 | upa_writeq(control, iommu->strbuf_regs + STRBUF_CONTROL); | |
1203 | ||
1204 | /* Clear out the tags using diagnostics. */ | |
1205 | for (i = 0; i < 16; i++) { | |
1206 | unsigned long ptag, ltag; | |
1207 | ||
1208 | ptag = iommu->strbuf_regs + STRBUF_PTAGDIAG; | |
1209 | ltag = iommu->strbuf_regs + STRBUF_LTAGDIAG; | |
1210 | ptag += (unsigned long)i * 8UL; | |
1211 | ltag += (unsigned long)i * 8UL; | |
1212 | ||
1213 | upa_writeq(0UL, ptag); | |
1214 | upa_writeq(0UL, ltag); | |
1215 | } | |
1216 | ||
1217 | /* Enable DVMA arbitration for all devices/slots. */ | |
1218 | control = upa_readq(iommu->sbus_control_reg); | |
1219 | control |= 0x3fUL; | |
1220 | upa_writeq(control, iommu->sbus_control_reg); | |
1221 | ||
1222 | /* Now some Xfire specific grot... */ | |
1223 | if (this_is_starfire) | |
286bbe87 | 1224 | starfire_hookup(sbus->portid); |
1da177e4 LT |
1225 | |
1226 | sysio_register_error_handlers(sbus); | |
1227 | } | |
8fae097d DM |
1228 | |
1229 | void sbus_fill_device_irq(struct sbus_dev *sdev) | |
1230 | { | |
25c7581b DM |
1231 | struct device_node *dp = of_find_node_by_phandle(sdev->prom_node); |
1232 | struct linux_prom_irqs *irqs; | |
8fae097d | 1233 | |
25c7581b DM |
1234 | irqs = of_get_property(dp, "interrupts", NULL); |
1235 | if (!irqs) { | |
8fae097d DM |
1236 | sdev->irqs[0] = 0; |
1237 | sdev->num_irqs = 0; | |
1238 | } else { | |
1239 | unsigned int pri = irqs[0].pri; | |
1240 | ||
1241 | sdev->num_irqs = 1; | |
1242 | if (pri < 0x20) | |
1243 | pri += sdev->slot * 8; | |
1244 | ||
1245 | sdev->irqs[0] = sbus_build_irq(sdev->bus, pri); | |
1246 | } | |
1247 | } | |
576c352e DM |
1248 | |
1249 | void __init sbus_arch_bus_ranges_init(struct device_node *pn, struct sbus_bus *sbus) | |
1250 | { | |
1251 | } | |
1252 | ||
1253 | void __init sbus_setup_iommu(struct sbus_bus *sbus, struct device_node *dp) | |
1254 | { | |
1255 | sbus_iommu_init(dp->node, sbus); | |
1256 | } | |
1257 | ||
1258 | void __init sbus_setup_arch_props(struct sbus_bus *sbus, struct device_node *dp) | |
1259 | { | |
1260 | } | |
1261 | ||
1262 | int __init sbus_arch_preinit(void) | |
1263 | { | |
1264 | return 0; | |
1265 | } | |
1266 | ||
1267 | void __init sbus_arch_postinit(void) | |
1268 | { | |
1269 | extern void firetruck_init(void); | |
576c352e DM |
1270 | |
1271 | firetruck_init(); | |
576c352e | 1272 | } |