Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* $Id: sbus.c,v 1.19 2002/01/23 11:27:32 davem Exp $ |
2 | * sbus.c: UltraSparc SBUS controller support. | |
3 | * | |
4 | * Copyright (C) 1999 David S. Miller (davem@redhat.com) | |
5 | */ | |
6 | ||
7 | #include <linux/kernel.h> | |
8 | #include <linux/types.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/interrupt.h> | |
14 | ||
15 | #include <asm/page.h> | |
16 | #include <asm/sbus.h> | |
17 | #include <asm/io.h> | |
18 | #include <asm/upa.h> | |
19 | #include <asm/cache.h> | |
20 | #include <asm/dma.h> | |
21 | #include <asm/irq.h> | |
25c7581b | 22 | #include <asm/prom.h> |
1da177e4 LT |
23 | #include <asm/starfire.h> |
24 | ||
25 | #include "iommu_common.h" | |
26 | ||
1da177e4 LT |
27 | #define MAP_BASE ((u32)0xc0000000) |
28 | ||
3e4d2650 DM |
29 | struct sbus_info { |
30 | struct iommu iommu; | |
31 | struct strbuf strbuf; | |
1da177e4 LT |
32 | }; |
33 | ||
34 | /* Offsets from iommu_regs */ | |
35 | #define SYSIO_IOMMUREG_BASE 0x2400UL | |
36 | #define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */ | |
37 | #define IOMMU_TSBBASE (0x2408UL - 0x2400UL) /* TSB base address register */ | |
38 | #define IOMMU_FLUSH (0x2410UL - 0x2400UL) /* IOMMU flush register */ | |
39 | #define IOMMU_VADIAG (0x4400UL - 0x2400UL) /* SBUS virtual address diagnostic */ | |
40 | #define IOMMU_TAGCMP (0x4408UL - 0x2400UL) /* TLB tag compare diagnostics */ | |
41 | #define IOMMU_LRUDIAG (0x4500UL - 0x2400UL) /* IOMMU LRU queue diagnostics */ | |
42 | #define IOMMU_TAGDIAG (0x4580UL - 0x2400UL) /* TLB tag diagnostics */ | |
43 | #define IOMMU_DRAMDIAG (0x4600UL - 0x2400UL) /* TLB data RAM diagnostics */ | |
44 | ||
45 | #define IOMMU_DRAM_VALID (1UL << 30UL) | |
46 | ||
3e4d2650 | 47 | static void __iommu_flushall(struct iommu *iommu) |
1da177e4 | 48 | { |
3e4d2650 | 49 | unsigned long tag; |
1da177e4 LT |
50 | int entry; |
51 | ||
3e4d2650 | 52 | tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL); |
1da177e4 LT |
53 | for (entry = 0; entry < 16; entry++) { |
54 | upa_writeq(0, tag); | |
55 | tag += 8UL; | |
56 | } | |
3e4d2650 | 57 | upa_readq(iommu->write_complete_reg); |
1da177e4 LT |
58 | } |
59 | ||
60 | /* Offsets from strbuf_regs */ | |
61 | #define SYSIO_STRBUFREG_BASE 0x2800UL | |
62 | #define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */ | |
63 | #define STRBUF_PFLUSH (0x2808UL - 0x2800UL) /* Page flush/invalidate */ | |
64 | #define STRBUF_FSYNC (0x2810UL - 0x2800UL) /* Flush synchronization */ | |
65 | #define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL) /* data RAM diagnostic */ | |
66 | #define STRBUF_ERRDIAG (0x5400UL - 0x2800UL) /* error status diagnostics */ | |
67 | #define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL) /* Page tag diagnostics */ | |
68 | #define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL) /* Line tag diagnostics */ | |
69 | ||
70 | #define STRBUF_TAG_VALID 0x02UL | |
71 | ||
3e4d2650 | 72 | static void sbus_strbuf_flush(struct iommu *iommu, struct strbuf *strbuf, u32 base, unsigned long npages, int direction) |
1da177e4 | 73 | { |
4dbc30fb DM |
74 | unsigned long n; |
75 | int limit; | |
76 | ||
4dbc30fb DM |
77 | n = npages; |
78 | while (n--) | |
3e4d2650 | 79 | upa_writeq(base + (n << IO_PAGE_SHIFT), strbuf->strbuf_pflush); |
1da177e4 | 80 | |
7c963ad1 DM |
81 | /* If the device could not have possibly put dirty data into |
82 | * the streaming cache, no flush-flag synchronization needs | |
83 | * to be performed. | |
84 | */ | |
85 | if (direction == SBUS_DMA_TODEVICE) | |
86 | return; | |
87 | ||
3e4d2650 | 88 | *(strbuf->strbuf_flushflag) = 0UL; |
7c963ad1 | 89 | |
1da177e4 | 90 | /* Whoopee cushion! */ |
3e4d2650 DM |
91 | upa_writeq(strbuf->strbuf_flushflag_pa, strbuf->strbuf_fsync); |
92 | upa_readq(iommu->write_complete_reg); | |
4dbc30fb | 93 | |
a228dfd5 | 94 | limit = 100000; |
3e4d2650 | 95 | while (*(strbuf->strbuf_flushflag) == 0UL) { |
4dbc30fb DM |
96 | limit--; |
97 | if (!limit) | |
98 | break; | |
a228dfd5 | 99 | udelay(1); |
4f07118f | 100 | rmb(); |
4dbc30fb DM |
101 | } |
102 | if (!limit) | |
103 | printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " | |
104 | "vaddr[%08x] npages[%ld]\n", | |
105 | base, npages); | |
1da177e4 LT |
106 | } |
107 | ||
2f3a2efd | 108 | /* Based largely upon the ppc64 iommu allocator. */ |
3e4d2650 | 109 | static long sbus_arena_alloc(struct iommu *iommu, unsigned long npages) |
1da177e4 | 110 | { |
9b3627f3 | 111 | struct iommu_arena *arena = &iommu->arena; |
2f3a2efd DM |
112 | unsigned long n, i, start, end, limit; |
113 | int pass; | |
114 | ||
115 | limit = arena->limit; | |
116 | start = arena->hint; | |
117 | pass = 0; | |
118 | ||
119 | again: | |
120 | n = find_next_zero_bit(arena->map, limit, start); | |
121 | end = n + npages; | |
122 | if (unlikely(end >= limit)) { | |
123 | if (likely(pass < 1)) { | |
124 | limit = start; | |
125 | start = 0; | |
126 | __iommu_flushall(iommu); | |
127 | pass++; | |
128 | goto again; | |
1da177e4 | 129 | } else { |
2f3a2efd DM |
130 | /* Scanned the whole thing, give up. */ |
131 | return -1; | |
1da177e4 | 132 | } |
2f3a2efd | 133 | } |
1da177e4 | 134 | |
2f3a2efd DM |
135 | for (i = n; i < end; i++) { |
136 | if (test_bit(i, arena->map)) { | |
137 | start = i + 1; | |
138 | goto again; | |
1da177e4 | 139 | } |
1da177e4 LT |
140 | } |
141 | ||
2f3a2efd DM |
142 | for (i = n; i < end; i++) |
143 | __set_bit(i, arena->map); | |
144 | ||
145 | arena->hint = end; | |
146 | ||
147 | return n; | |
1da177e4 LT |
148 | } |
149 | ||
9b3627f3 | 150 | static void sbus_arena_free(struct iommu_arena *arena, unsigned long base, unsigned long npages) |
1da177e4 | 151 | { |
2f3a2efd | 152 | unsigned long i; |
1da177e4 | 153 | |
2f3a2efd DM |
154 | for (i = base; i < (base + npages); i++) |
155 | __clear_bit(i, arena->map); | |
1da177e4 LT |
156 | } |
157 | ||
3e4d2650 | 158 | static void sbus_iommu_table_init(struct iommu *iommu, unsigned int tsbsize) |
1da177e4 | 159 | { |
2f3a2efd | 160 | unsigned long tsbbase, order, sz, num_tsb_entries; |
1da177e4 | 161 | |
2f3a2efd | 162 | num_tsb_entries = tsbsize / sizeof(iopte_t); |
1da177e4 | 163 | |
2f3a2efd DM |
164 | /* Setup initial software IOMMU state. */ |
165 | spin_lock_init(&iommu->lock); | |
3e4d2650 | 166 | iommu->page_table_map_base = MAP_BASE; |
1da177e4 | 167 | |
2f3a2efd DM |
168 | /* Allocate and initialize the free area map. */ |
169 | sz = num_tsb_entries / 8; | |
170 | sz = (sz + 7UL) & ~7UL; | |
171 | iommu->arena.map = kzalloc(sz, GFP_KERNEL); | |
172 | if (!iommu->arena.map) { | |
3e4d2650 | 173 | prom_printf("SBUS_IOMMU: Error, kmalloc(arena.map) failed.\n"); |
2f3a2efd DM |
174 | prom_halt(); |
175 | } | |
176 | iommu->arena.limit = num_tsb_entries; | |
177 | ||
178 | /* Now allocate and setup the IOMMU page table itself. */ | |
179 | order = get_order(tsbsize); | |
180 | tsbbase = __get_free_pages(GFP_KERNEL, order); | |
181 | if (!tsbbase) { | |
182 | prom_printf("IOMMU: Error, gfp(tsb) failed.\n"); | |
183 | prom_halt(); | |
1da177e4 | 184 | } |
2f3a2efd DM |
185 | iommu->page_table = (iopte_t *)tsbbase; |
186 | memset(iommu->page_table, 0, tsbsize); | |
1da177e4 LT |
187 | } |
188 | ||
3e4d2650 | 189 | static inline iopte_t *alloc_npages(struct iommu *iommu, unsigned long npages) |
1da177e4 | 190 | { |
2f3a2efd | 191 | long entry; |
1da177e4 | 192 | |
2f3a2efd DM |
193 | entry = sbus_arena_alloc(iommu, npages); |
194 | if (unlikely(entry < 0)) | |
195 | return NULL; | |
1da177e4 | 196 | |
2f3a2efd DM |
197 | return iommu->page_table + entry; |
198 | } | |
1da177e4 | 199 | |
3e4d2650 | 200 | static inline void free_npages(struct iommu *iommu, dma_addr_t base, unsigned long npages) |
2f3a2efd DM |
201 | { |
202 | sbus_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); | |
1da177e4 LT |
203 | } |
204 | ||
205 | void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr) | |
206 | { | |
3e4d2650 DM |
207 | struct sbus_info *info; |
208 | struct iommu *iommu; | |
1da177e4 | 209 | iopte_t *iopte; |
2f3a2efd | 210 | unsigned long flags, order, first_page; |
1da177e4 LT |
211 | void *ret; |
212 | int npages; | |
213 | ||
1da177e4 LT |
214 | size = IO_PAGE_ALIGN(size); |
215 | order = get_order(size); | |
216 | if (order >= 10) | |
217 | return NULL; | |
2f3a2efd | 218 | |
f3d48f03 | 219 | first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order); |
1da177e4 LT |
220 | if (first_page == 0UL) |
221 | return NULL; | |
222 | memset((char *)first_page, 0, PAGE_SIZE << order); | |
223 | ||
3e4d2650 DM |
224 | info = sdev->bus->iommu; |
225 | iommu = &info->iommu; | |
1da177e4 LT |
226 | |
227 | spin_lock_irqsave(&iommu->lock, flags); | |
2f3a2efd DM |
228 | iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); |
229 | spin_unlock_irqrestore(&iommu->lock, flags); | |
230 | ||
231 | if (unlikely(iopte == NULL)) { | |
1da177e4 LT |
232 | free_pages(first_page, order); |
233 | return NULL; | |
234 | } | |
235 | ||
3e4d2650 | 236 | *dvma_addr = (iommu->page_table_map_base + |
2f3a2efd | 237 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
1da177e4 LT |
238 | ret = (void *) first_page; |
239 | npages = size >> IO_PAGE_SHIFT; | |
2f3a2efd | 240 | first_page = __pa(first_page); |
1da177e4 | 241 | while (npages--) { |
2f3a2efd DM |
242 | iopte_val(*iopte) = (IOPTE_VALID | IOPTE_CACHE | |
243 | IOPTE_WRITE | | |
244 | (first_page & IOPTE_PAGE)); | |
245 | iopte++; | |
1da177e4 LT |
246 | first_page += IO_PAGE_SIZE; |
247 | } | |
1da177e4 LT |
248 | |
249 | return ret; | |
250 | } | |
251 | ||
252 | void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma) | |
253 | { | |
3e4d2650 DM |
254 | struct sbus_info *info; |
255 | struct iommu *iommu; | |
2f3a2efd DM |
256 | iopte_t *iopte; |
257 | unsigned long flags, order, npages; | |
1da177e4 LT |
258 | |
259 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | |
3e4d2650 DM |
260 | info = sdev->bus->iommu; |
261 | iommu = &info->iommu; | |
2f3a2efd | 262 | iopte = iommu->page_table + |
3e4d2650 | 263 | ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
1da177e4 | 264 | |
2f3a2efd DM |
265 | spin_lock_irqsave(&iommu->lock, flags); |
266 | ||
3e4d2650 | 267 | free_npages(iommu, dvma - iommu->page_table_map_base, npages); |
2f3a2efd DM |
268 | |
269 | spin_unlock_irqrestore(&iommu->lock, flags); | |
1da177e4 LT |
270 | |
271 | order = get_order(size); | |
272 | if (order < 10) | |
273 | free_pages((unsigned long)cpu, order); | |
274 | } | |
275 | ||
2f3a2efd | 276 | dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t sz, int direction) |
1da177e4 | 277 | { |
3e4d2650 DM |
278 | struct sbus_info *info; |
279 | struct iommu *iommu; | |
2f3a2efd DM |
280 | iopte_t *base; |
281 | unsigned long flags, npages, oaddr; | |
282 | unsigned long i, base_paddr; | |
283 | u32 bus_addr, ret; | |
284 | unsigned long iopte_protection; | |
1da177e4 | 285 | |
3e4d2650 DM |
286 | info = sdev->bus->iommu; |
287 | iommu = &info->iommu; | |
2f3a2efd DM |
288 | |
289 | if (unlikely(direction == SBUS_DMA_NONE)) | |
1da177e4 LT |
290 | BUG(); |
291 | ||
2f3a2efd DM |
292 | oaddr = (unsigned long)ptr; |
293 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | |
294 | npages >>= IO_PAGE_SHIFT; | |
1da177e4 LT |
295 | |
296 | spin_lock_irqsave(&iommu->lock, flags); | |
2f3a2efd | 297 | base = alloc_npages(iommu, npages); |
1da177e4 LT |
298 | spin_unlock_irqrestore(&iommu->lock, flags); |
299 | ||
2f3a2efd DM |
300 | if (unlikely(!base)) |
301 | BUG(); | |
302 | ||
3e4d2650 | 303 | bus_addr = (iommu->page_table_map_base + |
2f3a2efd DM |
304 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
305 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | |
306 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | |
1da177e4 | 307 | |
2f3a2efd DM |
308 | iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
309 | if (direction != SBUS_DMA_TODEVICE) | |
310 | iopte_protection |= IOPTE_WRITE; | |
311 | ||
312 | for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) | |
313 | iopte_val(*base) = iopte_protection | base_paddr; | |
314 | ||
315 | return ret; | |
1da177e4 LT |
316 | } |
317 | ||
2f3a2efd | 318 | void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) |
1da177e4 | 319 | { |
3e4d2650 DM |
320 | struct sbus_info *info = sdev->bus->iommu; |
321 | struct iommu *iommu = &info->iommu; | |
322 | struct strbuf *strbuf = &info->strbuf; | |
2f3a2efd DM |
323 | iopte_t *base; |
324 | unsigned long flags, npages, i; | |
325 | ||
326 | if (unlikely(direction == SBUS_DMA_NONE)) | |
327 | BUG(); | |
328 | ||
329 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); | |
330 | npages >>= IO_PAGE_SHIFT; | |
331 | base = iommu->page_table + | |
3e4d2650 | 332 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
1da177e4 | 333 | |
2f3a2efd | 334 | bus_addr &= IO_PAGE_MASK; |
1da177e4 LT |
335 | |
336 | spin_lock_irqsave(&iommu->lock, flags); | |
3e4d2650 | 337 | sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); |
2f3a2efd DM |
338 | for (i = 0; i < npages; i++) |
339 | iopte_val(base[i]) = 0UL; | |
3e4d2650 | 340 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
1da177e4 LT |
341 | spin_unlock_irqrestore(&iommu->lock, flags); |
342 | } | |
343 | ||
344 | #define SG_ENT_PHYS_ADDRESS(SG) \ | |
345 | (__pa(page_address((SG)->page)) + (SG)->offset) | |
346 | ||
2f3a2efd DM |
347 | static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, |
348 | int nused, int nelems, unsigned long iopte_protection) | |
1da177e4 LT |
349 | { |
350 | struct scatterlist *dma_sg = sg; | |
351 | struct scatterlist *sg_end = sg + nelems; | |
352 | int i; | |
353 | ||
354 | for (i = 0; i < nused; i++) { | |
355 | unsigned long pteval = ~0UL; | |
356 | u32 dma_npages; | |
357 | ||
358 | dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + | |
359 | dma_sg->dma_length + | |
360 | ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; | |
361 | do { | |
362 | unsigned long offset; | |
363 | signed int len; | |
364 | ||
365 | /* If we are here, we know we have at least one | |
366 | * more page to map. So walk forward until we | |
367 | * hit a page crossing, and begin creating new | |
368 | * mappings from that spot. | |
369 | */ | |
370 | for (;;) { | |
371 | unsigned long tmp; | |
372 | ||
2f3a2efd | 373 | tmp = SG_ENT_PHYS_ADDRESS(sg); |
1da177e4 LT |
374 | len = sg->length; |
375 | if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { | |
376 | pteval = tmp & IO_PAGE_MASK; | |
377 | offset = tmp & (IO_PAGE_SIZE - 1UL); | |
378 | break; | |
379 | } | |
380 | if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { | |
381 | pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; | |
382 | offset = 0UL; | |
383 | len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); | |
384 | break; | |
385 | } | |
386 | sg++; | |
387 | } | |
388 | ||
2f3a2efd | 389 | pteval = iopte_protection | (pteval & IOPTE_PAGE); |
1da177e4 LT |
390 | while (len > 0) { |
391 | *iopte++ = __iopte(pteval); | |
392 | pteval += IO_PAGE_SIZE; | |
393 | len -= (IO_PAGE_SIZE - offset); | |
394 | offset = 0; | |
395 | dma_npages--; | |
396 | } | |
397 | ||
398 | pteval = (pteval & IOPTE_PAGE) + len; | |
399 | sg++; | |
400 | ||
401 | /* Skip over any tail mappings we've fully mapped, | |
402 | * adjusting pteval along the way. Stop when we | |
403 | * detect a page crossing event. | |
404 | */ | |
405 | while (sg < sg_end && | |
406 | (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && | |
407 | (pteval == SG_ENT_PHYS_ADDRESS(sg)) && | |
408 | ((pteval ^ | |
409 | (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { | |
410 | pteval += sg->length; | |
411 | sg++; | |
412 | } | |
413 | if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) | |
414 | pteval = ~0UL; | |
415 | } while (dma_npages != 0); | |
416 | dma_sg++; | |
417 | } | |
418 | } | |
419 | ||
2f3a2efd | 420 | int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
1da177e4 | 421 | { |
3e4d2650 DM |
422 | struct sbus_info *info; |
423 | struct iommu *iommu; | |
2f3a2efd DM |
424 | unsigned long flags, npages, iopte_protection; |
425 | iopte_t *base; | |
1da177e4 LT |
426 | u32 dma_base; |
427 | struct scatterlist *sgtmp; | |
428 | int used; | |
1da177e4 LT |
429 | |
430 | /* Fast path single entry scatterlists. */ | |
2f3a2efd DM |
431 | if (nelems == 1) { |
432 | sglist->dma_address = | |
1da177e4 | 433 | sbus_map_single(sdev, |
2f3a2efd DM |
434 | (page_address(sglist->page) + sglist->offset), |
435 | sglist->length, direction); | |
436 | sglist->dma_length = sglist->length; | |
1da177e4 LT |
437 | return 1; |
438 | } | |
439 | ||
3e4d2650 DM |
440 | info = sdev->bus->iommu; |
441 | iommu = &info->iommu; | |
2f3a2efd DM |
442 | |
443 | if (unlikely(direction == SBUS_DMA_NONE)) | |
444 | BUG(); | |
445 | ||
446 | npages = prepare_sg(sglist, nelems); | |
1da177e4 LT |
447 | |
448 | spin_lock_irqsave(&iommu->lock, flags); | |
2f3a2efd DM |
449 | base = alloc_npages(iommu, npages); |
450 | spin_unlock_irqrestore(&iommu->lock, flags); | |
451 | ||
452 | if (unlikely(base == NULL)) | |
453 | BUG(); | |
454 | ||
3e4d2650 | 455 | dma_base = iommu->page_table_map_base + |
2f3a2efd | 456 | ((base - iommu->page_table) << IO_PAGE_SHIFT); |
1da177e4 LT |
457 | |
458 | /* Normalize DVMA addresses. */ | |
2f3a2efd | 459 | used = nelems; |
1da177e4 | 460 | |
2f3a2efd | 461 | sgtmp = sglist; |
1da177e4 LT |
462 | while (used && sgtmp->dma_length) { |
463 | sgtmp->dma_address += dma_base; | |
464 | sgtmp++; | |
465 | used--; | |
466 | } | |
2f3a2efd | 467 | used = nelems - used; |
1da177e4 | 468 | |
2f3a2efd DM |
469 | iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
470 | if (direction != SBUS_DMA_TODEVICE) | |
471 | iopte_protection |= IOPTE_WRITE; | |
472 | ||
473 | fill_sg(base, sglist, used, nelems, iopte_protection); | |
1da177e4 | 474 | |
1da177e4 | 475 | #ifdef VERIFY_SG |
2f3a2efd | 476 | verify_sglist(sglist, nelems, base, npages); |
1da177e4 | 477 | #endif |
1da177e4 LT |
478 | |
479 | return used; | |
1da177e4 LT |
480 | } |
481 | ||
2f3a2efd | 482 | void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
1da177e4 | 483 | { |
3e4d2650 DM |
484 | struct sbus_info *info; |
485 | struct iommu *iommu; | |
486 | struct strbuf *strbuf; | |
2f3a2efd DM |
487 | iopte_t *base; |
488 | unsigned long flags, i, npages; | |
489 | u32 bus_addr; | |
1da177e4 | 490 | |
2f3a2efd DM |
491 | if (unlikely(direction == SBUS_DMA_NONE)) |
492 | BUG(); | |
1da177e4 | 493 | |
3e4d2650 DM |
494 | info = sdev->bus->iommu; |
495 | iommu = &info->iommu; | |
496 | strbuf = &info->strbuf; | |
2f3a2efd DM |
497 | |
498 | bus_addr = sglist->dma_address & IO_PAGE_MASK; | |
499 | ||
500 | for (i = 1; i < nelems; i++) | |
501 | if (sglist[i].dma_length == 0) | |
1da177e4 | 502 | break; |
1da177e4 | 503 | i--; |
2f3a2efd DM |
504 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - |
505 | bus_addr) >> IO_PAGE_SHIFT; | |
506 | ||
507 | base = iommu->page_table + | |
3e4d2650 | 508 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
1da177e4 | 509 | |
1da177e4 | 510 | spin_lock_irqsave(&iommu->lock, flags); |
3e4d2650 | 511 | sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); |
2f3a2efd DM |
512 | for (i = 0; i < npages; i++) |
513 | iopte_val(base[i]) = 0UL; | |
3e4d2650 | 514 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
1da177e4 LT |
515 | spin_unlock_irqrestore(&iommu->lock, flags); |
516 | } | |
517 | ||
2f3a2efd | 518 | void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) |
1da177e4 | 519 | { |
3e4d2650 DM |
520 | struct sbus_info *info; |
521 | struct iommu *iommu; | |
522 | struct strbuf *strbuf; | |
2f3a2efd DM |
523 | unsigned long flags, npages; |
524 | ||
3e4d2650 DM |
525 | info = sdev->bus->iommu; |
526 | iommu = &info->iommu; | |
527 | strbuf = &info->strbuf; | |
1da177e4 | 528 | |
2f3a2efd DM |
529 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
530 | npages >>= IO_PAGE_SHIFT; | |
531 | bus_addr &= IO_PAGE_MASK; | |
1da177e4 LT |
532 | |
533 | spin_lock_irqsave(&iommu->lock, flags); | |
3e4d2650 | 534 | sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); |
1da177e4 LT |
535 | spin_unlock_irqrestore(&iommu->lock, flags); |
536 | } | |
537 | ||
538 | void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) | |
539 | { | |
540 | } | |
541 | ||
2f3a2efd | 542 | void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
1da177e4 | 543 | { |
3e4d2650 DM |
544 | struct sbus_info *info; |
545 | struct iommu *iommu; | |
546 | struct strbuf *strbuf; | |
2f3a2efd DM |
547 | unsigned long flags, npages, i; |
548 | u32 bus_addr; | |
549 | ||
3e4d2650 DM |
550 | info = sdev->bus->iommu; |
551 | iommu = &info->iommu; | |
552 | strbuf = &info->strbuf; | |
1da177e4 | 553 | |
2f3a2efd DM |
554 | bus_addr = sglist[0].dma_address & IO_PAGE_MASK; |
555 | for (i = 0; i < nelems; i++) { | |
556 | if (!sglist[i].dma_length) | |
1da177e4 LT |
557 | break; |
558 | } | |
559 | i--; | |
2f3a2efd DM |
560 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) |
561 | - bus_addr) >> IO_PAGE_SHIFT; | |
1da177e4 LT |
562 | |
563 | spin_lock_irqsave(&iommu->lock, flags); | |
3e4d2650 | 564 | sbus_strbuf_flush(iommu, strbuf, bus_addr, npages, direction); |
1da177e4 LT |
565 | spin_unlock_irqrestore(&iommu->lock, flags); |
566 | } | |
567 | ||
568 | void sbus_dma_sync_sg_for_device(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) | |
569 | { | |
570 | } | |
571 | ||
572 | /* Enable 64-bit DVMA mode for the given device. */ | |
573 | void sbus_set_sbus64(struct sbus_dev *sdev, int bursts) | |
574 | { | |
3e4d2650 DM |
575 | struct sbus_info *info = sdev->bus->iommu; |
576 | struct iommu *iommu = &info->iommu; | |
1da177e4 LT |
577 | int slot = sdev->slot; |
578 | unsigned long cfg_reg; | |
579 | u64 val; | |
580 | ||
3e4d2650 | 581 | cfg_reg = iommu->write_complete_reg; |
1da177e4 LT |
582 | switch (slot) { |
583 | case 0: | |
584 | cfg_reg += 0x20UL; | |
585 | break; | |
586 | case 1: | |
587 | cfg_reg += 0x28UL; | |
588 | break; | |
589 | case 2: | |
590 | cfg_reg += 0x30UL; | |
591 | break; | |
592 | case 3: | |
593 | cfg_reg += 0x38UL; | |
594 | break; | |
595 | case 13: | |
596 | cfg_reg += 0x40UL; | |
597 | break; | |
598 | case 14: | |
599 | cfg_reg += 0x48UL; | |
600 | break; | |
601 | case 15: | |
602 | cfg_reg += 0x50UL; | |
603 | break; | |
604 | ||
605 | default: | |
606 | return; | |
607 | }; | |
608 | ||
609 | val = upa_readq(cfg_reg); | |
610 | if (val & (1UL << 14UL)) { | |
611 | /* Extended transfer mode already enabled. */ | |
612 | return; | |
613 | } | |
614 | ||
615 | val |= (1UL << 14UL); | |
616 | ||
617 | if (bursts & DMA_BURST8) | |
618 | val |= (1UL << 1UL); | |
619 | if (bursts & DMA_BURST16) | |
620 | val |= (1UL << 2UL); | |
621 | if (bursts & DMA_BURST32) | |
622 | val |= (1UL << 3UL); | |
623 | if (bursts & DMA_BURST64) | |
624 | val |= (1UL << 4UL); | |
625 | upa_writeq(val, cfg_reg); | |
626 | } | |
627 | ||
1da177e4 LT |
628 | /* INO number to IMAP register offset for SYSIO external IRQ's. |
629 | * This should conform to both Sunfire/Wildfire server and Fusion | |
630 | * desktop designs. | |
631 | */ | |
ec4d18f2 DM |
632 | #define SYSIO_IMAP_SLOT0 0x2c00UL |
633 | #define SYSIO_IMAP_SLOT1 0x2c08UL | |
634 | #define SYSIO_IMAP_SLOT2 0x2c10UL | |
635 | #define SYSIO_IMAP_SLOT3 0x2c18UL | |
636 | #define SYSIO_IMAP_SCSI 0x3000UL | |
637 | #define SYSIO_IMAP_ETH 0x3008UL | |
638 | #define SYSIO_IMAP_BPP 0x3010UL | |
639 | #define SYSIO_IMAP_AUDIO 0x3018UL | |
640 | #define SYSIO_IMAP_PFAIL 0x3020UL | |
641 | #define SYSIO_IMAP_KMS 0x3028UL | |
642 | #define SYSIO_IMAP_FLPY 0x3030UL | |
643 | #define SYSIO_IMAP_SHW 0x3038UL | |
644 | #define SYSIO_IMAP_KBD 0x3040UL | |
645 | #define SYSIO_IMAP_MS 0x3048UL | |
646 | #define SYSIO_IMAP_SER 0x3050UL | |
647 | #define SYSIO_IMAP_TIM0 0x3060UL | |
648 | #define SYSIO_IMAP_TIM1 0x3068UL | |
649 | #define SYSIO_IMAP_UE 0x3070UL | |
650 | #define SYSIO_IMAP_CE 0x3078UL | |
651 | #define SYSIO_IMAP_SBERR 0x3080UL | |
652 | #define SYSIO_IMAP_PMGMT 0x3088UL | |
653 | #define SYSIO_IMAP_GFX 0x3090UL | |
654 | #define SYSIO_IMAP_EUPA 0x3098UL | |
1da177e4 LT |
655 | |
656 | #define bogon ((unsigned long) -1) | |
657 | static unsigned long sysio_irq_offsets[] = { | |
658 | /* SBUS Slot 0 --> 3, level 1 --> 7 */ | |
659 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
660 | SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, | |
661 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
662 | SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, | |
663 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
664 | SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, | |
665 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
666 | SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, | |
667 | ||
668 | /* Onboard devices (not relevant/used on SunFire). */ | |
669 | SYSIO_IMAP_SCSI, | |
670 | SYSIO_IMAP_ETH, | |
671 | SYSIO_IMAP_BPP, | |
672 | bogon, | |
673 | SYSIO_IMAP_AUDIO, | |
674 | SYSIO_IMAP_PFAIL, | |
675 | bogon, | |
676 | bogon, | |
677 | SYSIO_IMAP_KMS, | |
678 | SYSIO_IMAP_FLPY, | |
679 | SYSIO_IMAP_SHW, | |
680 | SYSIO_IMAP_KBD, | |
681 | SYSIO_IMAP_MS, | |
682 | SYSIO_IMAP_SER, | |
683 | bogon, | |
684 | bogon, | |
685 | SYSIO_IMAP_TIM0, | |
686 | SYSIO_IMAP_TIM1, | |
687 | bogon, | |
688 | bogon, | |
689 | SYSIO_IMAP_UE, | |
690 | SYSIO_IMAP_CE, | |
691 | SYSIO_IMAP_SBERR, | |
692 | SYSIO_IMAP_PMGMT, | |
693 | }; | |
694 | ||
695 | #undef bogon | |
696 | ||
84c1a13a | 697 | #define NUM_SYSIO_OFFSETS ARRAY_SIZE(sysio_irq_offsets) |
1da177e4 LT |
698 | |
699 | /* Convert Interrupt Mapping register pointer to associated | |
700 | * Interrupt Clear register pointer, SYSIO specific version. | |
701 | */ | |
702 | #define SYSIO_ICLR_UNUSED0 0x3400UL | |
ec4d18f2 DM |
703 | #define SYSIO_ICLR_SLOT0 0x3408UL |
704 | #define SYSIO_ICLR_SLOT1 0x3448UL | |
705 | #define SYSIO_ICLR_SLOT2 0x3488UL | |
706 | #define SYSIO_ICLR_SLOT3 0x34c8UL | |
1da177e4 LT |
707 | static unsigned long sysio_imap_to_iclr(unsigned long imap) |
708 | { | |
709 | unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0; | |
710 | return imap + diff; | |
711 | } | |
712 | ||
713 | unsigned int sbus_build_irq(void *buscookie, unsigned int ino) | |
714 | { | |
715 | struct sbus_bus *sbus = (struct sbus_bus *)buscookie; | |
3e4d2650 DM |
716 | struct sbus_info *info = sbus->iommu; |
717 | struct iommu *iommu = &info->iommu; | |
718 | unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; | |
1da177e4 | 719 | unsigned long imap, iclr; |
37cdcd9e | 720 | int sbus_level = 0; |
1da177e4 LT |
721 | |
722 | imap = sysio_irq_offsets[ino]; | |
723 | if (imap == ((unsigned long)-1)) { | |
37cdcd9e DM |
724 | prom_printf("get_irq_translations: Bad SYSIO INO[%x]\n", |
725 | ino); | |
1da177e4 LT |
726 | prom_halt(); |
727 | } | |
728 | imap += reg_base; | |
729 | ||
730 | /* SYSIO inconsistency. For external SLOTS, we have to select | |
731 | * the right ICLR register based upon the lower SBUS irq level | |
732 | * bits. | |
733 | */ | |
734 | if (ino >= 0x20) { | |
735 | iclr = sysio_imap_to_iclr(imap); | |
736 | } else { | |
737 | int sbus_slot = (ino & 0x18)>>3; | |
738 | ||
739 | sbus_level = ino & 0x7; | |
740 | ||
741 | switch(sbus_slot) { | |
742 | case 0: | |
743 | iclr = reg_base + SYSIO_ICLR_SLOT0; | |
744 | break; | |
745 | case 1: | |
746 | iclr = reg_base + SYSIO_ICLR_SLOT1; | |
747 | break; | |
748 | case 2: | |
749 | iclr = reg_base + SYSIO_ICLR_SLOT2; | |
750 | break; | |
751 | default: | |
752 | case 3: | |
753 | iclr = reg_base + SYSIO_ICLR_SLOT3; | |
754 | break; | |
755 | }; | |
756 | ||
757 | iclr += ((unsigned long)sbus_level - 1UL) * 8UL; | |
758 | } | |
e18e2a00 | 759 | return build_irq(sbus_level, iclr, imap); |
1da177e4 LT |
760 | } |
761 | ||
762 | /* Error interrupt handling. */ | |
763 | #define SYSIO_UE_AFSR 0x0030UL | |
764 | #define SYSIO_UE_AFAR 0x0038UL | |
765 | #define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
766 | #define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
767 | #define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
768 | #define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ | |
769 | #define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
770 | #define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
771 | #define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
772 | #define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */ | |
773 | #define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
774 | #define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
775 | #define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 776 | static irqreturn_t sysio_ue_handler(int irq, void *dev_id) |
1da177e4 LT |
777 | { |
778 | struct sbus_bus *sbus = dev_id; | |
3e4d2650 DM |
779 | struct sbus_info *info = sbus->iommu; |
780 | struct iommu *iommu = &info->iommu; | |
781 | unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; | |
1da177e4 LT |
782 | unsigned long afsr_reg, afar_reg; |
783 | unsigned long afsr, afar, error_bits; | |
784 | int reported; | |
785 | ||
786 | afsr_reg = reg_base + SYSIO_UE_AFSR; | |
787 | afar_reg = reg_base + SYSIO_UE_AFAR; | |
788 | ||
789 | /* Latch error status. */ | |
790 | afsr = upa_readq(afsr_reg); | |
791 | afar = upa_readq(afar_reg); | |
792 | ||
793 | /* Clear primary/secondary error status bits. */ | |
794 | error_bits = afsr & | |
795 | (SYSIO_UEAFSR_PPIO | SYSIO_UEAFSR_PDRD | SYSIO_UEAFSR_PDWR | | |
796 | SYSIO_UEAFSR_SPIO | SYSIO_UEAFSR_SDRD | SYSIO_UEAFSR_SDWR); | |
797 | upa_writeq(error_bits, afsr_reg); | |
798 | ||
799 | /* Log the error. */ | |
800 | printk("SYSIO[%x]: Uncorrectable ECC Error, primary error type[%s]\n", | |
801 | sbus->portid, | |
802 | (((error_bits & SYSIO_UEAFSR_PPIO) ? | |
803 | "PIO" : | |
804 | ((error_bits & SYSIO_UEAFSR_PDRD) ? | |
805 | "DVMA Read" : | |
806 | ((error_bits & SYSIO_UEAFSR_PDWR) ? | |
807 | "DVMA Write" : "???"))))); | |
808 | printk("SYSIO[%x]: DOFF[%lx] SIZE[%lx] MID[%lx]\n", | |
809 | sbus->portid, | |
810 | (afsr & SYSIO_UEAFSR_DOFF) >> 45UL, | |
811 | (afsr & SYSIO_UEAFSR_SIZE) >> 42UL, | |
812 | (afsr & SYSIO_UEAFSR_MID) >> 37UL); | |
813 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
814 | printk("SYSIO[%x]: Secondary UE errors [", sbus->portid); | |
815 | reported = 0; | |
816 | if (afsr & SYSIO_UEAFSR_SPIO) { | |
817 | reported++; | |
818 | printk("(PIO)"); | |
819 | } | |
820 | if (afsr & SYSIO_UEAFSR_SDRD) { | |
821 | reported++; | |
822 | printk("(DVMA Read)"); | |
823 | } | |
824 | if (afsr & SYSIO_UEAFSR_SDWR) { | |
825 | reported++; | |
826 | printk("(DVMA Write)"); | |
827 | } | |
828 | if (!reported) | |
829 | printk("(none)"); | |
830 | printk("]\n"); | |
831 | ||
832 | return IRQ_HANDLED; | |
833 | } | |
834 | ||
835 | #define SYSIO_CE_AFSR 0x0040UL | |
836 | #define SYSIO_CE_AFAR 0x0048UL | |
837 | #define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ | |
838 | #define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ | |
839 | #define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ | |
840 | #define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */ | |
841 | #define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ | |
842 | #define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ | |
843 | #define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */ | |
844 | #define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */ | |
845 | #define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */ | |
846 | #define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ | |
847 | #define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ | |
848 | #define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 849 | static irqreturn_t sysio_ce_handler(int irq, void *dev_id) |
1da177e4 LT |
850 | { |
851 | struct sbus_bus *sbus = dev_id; | |
3e4d2650 DM |
852 | struct sbus_info *info = sbus->iommu; |
853 | struct iommu *iommu = &info->iommu; | |
854 | unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; | |
1da177e4 LT |
855 | unsigned long afsr_reg, afar_reg; |
856 | unsigned long afsr, afar, error_bits; | |
857 | int reported; | |
858 | ||
859 | afsr_reg = reg_base + SYSIO_CE_AFSR; | |
860 | afar_reg = reg_base + SYSIO_CE_AFAR; | |
861 | ||
862 | /* Latch error status. */ | |
863 | afsr = upa_readq(afsr_reg); | |
864 | afar = upa_readq(afar_reg); | |
865 | ||
866 | /* Clear primary/secondary error status bits. */ | |
867 | error_bits = afsr & | |
868 | (SYSIO_CEAFSR_PPIO | SYSIO_CEAFSR_PDRD | SYSIO_CEAFSR_PDWR | | |
869 | SYSIO_CEAFSR_SPIO | SYSIO_CEAFSR_SDRD | SYSIO_CEAFSR_SDWR); | |
870 | upa_writeq(error_bits, afsr_reg); | |
871 | ||
872 | printk("SYSIO[%x]: Correctable ECC Error, primary error type[%s]\n", | |
873 | sbus->portid, | |
874 | (((error_bits & SYSIO_CEAFSR_PPIO) ? | |
875 | "PIO" : | |
876 | ((error_bits & SYSIO_CEAFSR_PDRD) ? | |
877 | "DVMA Read" : | |
878 | ((error_bits & SYSIO_CEAFSR_PDWR) ? | |
879 | "DVMA Write" : "???"))))); | |
880 | ||
881 | /* XXX Use syndrome and afar to print out module string just like | |
882 | * XXX UDB CE trap handler does... -DaveM | |
883 | */ | |
884 | printk("SYSIO[%x]: DOFF[%lx] ECC Syndrome[%lx] Size[%lx] MID[%lx]\n", | |
885 | sbus->portid, | |
886 | (afsr & SYSIO_CEAFSR_DOFF) >> 45UL, | |
887 | (afsr & SYSIO_CEAFSR_ESYND) >> 48UL, | |
888 | (afsr & SYSIO_CEAFSR_SIZE) >> 42UL, | |
889 | (afsr & SYSIO_CEAFSR_MID) >> 37UL); | |
890 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
891 | ||
892 | printk("SYSIO[%x]: Secondary CE errors [", sbus->portid); | |
893 | reported = 0; | |
894 | if (afsr & SYSIO_CEAFSR_SPIO) { | |
895 | reported++; | |
896 | printk("(PIO)"); | |
897 | } | |
898 | if (afsr & SYSIO_CEAFSR_SDRD) { | |
899 | reported++; | |
900 | printk("(DVMA Read)"); | |
901 | } | |
902 | if (afsr & SYSIO_CEAFSR_SDWR) { | |
903 | reported++; | |
904 | printk("(DVMA Write)"); | |
905 | } | |
906 | if (!reported) | |
907 | printk("(none)"); | |
908 | printk("]\n"); | |
909 | ||
910 | return IRQ_HANDLED; | |
911 | } | |
912 | ||
913 | #define SYSIO_SBUS_AFSR 0x2010UL | |
914 | #define SYSIO_SBUS_AFAR 0x2018UL | |
915 | #define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */ | |
916 | #define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */ | |
917 | #define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */ | |
918 | #define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */ | |
919 | #define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */ | |
920 | #define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */ | |
921 | #define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */ | |
922 | #define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */ | |
923 | #define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */ | |
924 | #define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */ | |
925 | #define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */ | |
926 | #define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */ | |
6d24c8dc | 927 | static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id) |
1da177e4 LT |
928 | { |
929 | struct sbus_bus *sbus = dev_id; | |
3e4d2650 DM |
930 | struct sbus_info *info = sbus->iommu; |
931 | struct iommu *iommu = &info->iommu; | |
1da177e4 LT |
932 | unsigned long afsr_reg, afar_reg, reg_base; |
933 | unsigned long afsr, afar, error_bits; | |
934 | int reported; | |
935 | ||
3e4d2650 | 936 | reg_base = iommu->write_complete_reg - 0x2000UL; |
1da177e4 LT |
937 | afsr_reg = reg_base + SYSIO_SBUS_AFSR; |
938 | afar_reg = reg_base + SYSIO_SBUS_AFAR; | |
939 | ||
940 | afsr = upa_readq(afsr_reg); | |
941 | afar = upa_readq(afar_reg); | |
942 | ||
943 | /* Clear primary/secondary error status bits. */ | |
944 | error_bits = afsr & | |
945 | (SYSIO_SBAFSR_PLE | SYSIO_SBAFSR_PTO | SYSIO_SBAFSR_PBERR | | |
946 | SYSIO_SBAFSR_SLE | SYSIO_SBAFSR_STO | SYSIO_SBAFSR_SBERR); | |
947 | upa_writeq(error_bits, afsr_reg); | |
948 | ||
949 | /* Log the error. */ | |
950 | printk("SYSIO[%x]: SBUS Error, primary error type[%s] read(%d)\n", | |
951 | sbus->portid, | |
952 | (((error_bits & SYSIO_SBAFSR_PLE) ? | |
953 | "Late PIO Error" : | |
954 | ((error_bits & SYSIO_SBAFSR_PTO) ? | |
955 | "Time Out" : | |
956 | ((error_bits & SYSIO_SBAFSR_PBERR) ? | |
957 | "Error Ack" : "???")))), | |
958 | (afsr & SYSIO_SBAFSR_RD) ? 1 : 0); | |
959 | printk("SYSIO[%x]: size[%lx] MID[%lx]\n", | |
960 | sbus->portid, | |
961 | (afsr & SYSIO_SBAFSR_SIZE) >> 42UL, | |
962 | (afsr & SYSIO_SBAFSR_MID) >> 37UL); | |
963 | printk("SYSIO[%x]: AFAR[%016lx]\n", sbus->portid, afar); | |
964 | printk("SYSIO[%x]: Secondary SBUS errors [", sbus->portid); | |
965 | reported = 0; | |
966 | if (afsr & SYSIO_SBAFSR_SLE) { | |
967 | reported++; | |
968 | printk("(Late PIO Error)"); | |
969 | } | |
970 | if (afsr & SYSIO_SBAFSR_STO) { | |
971 | reported++; | |
972 | printk("(Time Out)"); | |
973 | } | |
974 | if (afsr & SYSIO_SBAFSR_SBERR) { | |
975 | reported++; | |
976 | printk("(Error Ack)"); | |
977 | } | |
978 | if (!reported) | |
979 | printk("(none)"); | |
980 | printk("]\n"); | |
981 | ||
982 | /* XXX check iommu/strbuf for further error status XXX */ | |
983 | ||
984 | return IRQ_HANDLED; | |
985 | } | |
986 | ||
987 | #define ECC_CONTROL 0x0020UL | |
988 | #define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */ | |
989 | #define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */ | |
990 | #define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */ | |
991 | ||
992 | #define SYSIO_UE_INO 0x34 | |
993 | #define SYSIO_CE_INO 0x35 | |
994 | #define SYSIO_SBUSERR_INO 0x36 | |
995 | ||
996 | static void __init sysio_register_error_handlers(struct sbus_bus *sbus) | |
997 | { | |
3e4d2650 DM |
998 | struct sbus_info *info = sbus->iommu; |
999 | struct iommu *iommu = &info->iommu; | |
1000 | unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; | |
1da177e4 LT |
1001 | unsigned int irq; |
1002 | u64 control; | |
1003 | ||
1004 | irq = sbus_build_irq(sbus, SYSIO_UE_INO); | |
96a496fd DM |
1005 | if (request_irq(irq, sysio_ue_handler, 0, |
1006 | "SYSIO_UE", sbus) < 0) { | |
1da177e4 LT |
1007 | prom_printf("SYSIO[%x]: Cannot register UE interrupt.\n", |
1008 | sbus->portid); | |
1009 | prom_halt(); | |
1010 | } | |
1011 | ||
1012 | irq = sbus_build_irq(sbus, SYSIO_CE_INO); | |
96a496fd DM |
1013 | if (request_irq(irq, sysio_ce_handler, 0, |
1014 | "SYSIO_CE", sbus) < 0) { | |
1da177e4 LT |
1015 | prom_printf("SYSIO[%x]: Cannot register CE interrupt.\n", |
1016 | sbus->portid); | |
1017 | prom_halt(); | |
1018 | } | |
1019 | ||
1020 | irq = sbus_build_irq(sbus, SYSIO_SBUSERR_INO); | |
96a496fd DM |
1021 | if (request_irq(irq, sysio_sbus_error_handler, 0, |
1022 | "SYSIO_SBERR", sbus) < 0) { | |
1da177e4 LT |
1023 | prom_printf("SYSIO[%x]: Cannot register SBUS Error interrupt.\n", |
1024 | sbus->portid); | |
1025 | prom_halt(); | |
1026 | } | |
1027 | ||
1028 | /* Now turn the error interrupts on and also enable ECC checking. */ | |
1029 | upa_writeq((SYSIO_ECNTRL_ECCEN | | |
1030 | SYSIO_ECNTRL_UEEN | | |
1031 | SYSIO_ECNTRL_CEEN), | |
1032 | reg_base + ECC_CONTROL); | |
1033 | ||
3e4d2650 | 1034 | control = upa_readq(iommu->write_complete_reg); |
1da177e4 | 1035 | control |= 0x100UL; /* SBUS Error Interrupt Enable */ |
3e4d2650 | 1036 | upa_writeq(control, iommu->write_complete_reg); |
1da177e4 LT |
1037 | } |
1038 | ||
1039 | /* Boot time initialization. */ | |
576c352e | 1040 | static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
1da177e4 | 1041 | { |
6a23acf3 | 1042 | const struct linux_prom64_registers *pr; |
25c7581b | 1043 | struct device_node *dp; |
3e4d2650 DM |
1044 | struct sbus_info *info; |
1045 | struct iommu *iommu; | |
1046 | struct strbuf *strbuf; | |
1047 | unsigned long regs, reg_base; | |
1da177e4 | 1048 | u64 control; |
25c7581b DM |
1049 | int i; |
1050 | ||
1051 | dp = of_find_node_by_phandle(__node); | |
1da177e4 | 1052 | |
25c7581b | 1053 | sbus->portid = of_getintprop_default(dp, "upa-portid", -1); |
1da177e4 | 1054 | |
25c7581b DM |
1055 | pr = of_get_property(dp, "reg", NULL); |
1056 | if (!pr) { | |
1da177e4 LT |
1057 | prom_printf("sbus_iommu_init: Cannot map SYSIO control registers.\n"); |
1058 | prom_halt(); | |
1059 | } | |
25c7581b | 1060 | regs = pr->phys_addr; |
1da177e4 | 1061 | |
3e4d2650 DM |
1062 | info = kzalloc(sizeof(*info), GFP_ATOMIC); |
1063 | if (info == NULL) { | |
1064 | prom_printf("sbus_iommu_init: Fatal error, " | |
1065 | "kmalloc(info) failed\n"); | |
1da177e4 LT |
1066 | prom_halt(); |
1067 | } | |
1068 | ||
3e4d2650 DM |
1069 | iommu = &info->iommu; |
1070 | strbuf = &info->strbuf; | |
1da177e4 | 1071 | |
3e4d2650 DM |
1072 | reg_base = regs + SYSIO_IOMMUREG_BASE; |
1073 | iommu->iommu_control = reg_base + IOMMU_CONTROL; | |
1074 | iommu->iommu_tsbbase = reg_base + IOMMU_TSBBASE; | |
1075 | iommu->iommu_flush = reg_base + IOMMU_FLUSH; | |
1da177e4 | 1076 | |
3e4d2650 DM |
1077 | reg_base = regs + SYSIO_STRBUFREG_BASE; |
1078 | strbuf->strbuf_control = reg_base + STRBUF_CONTROL; | |
1079 | strbuf->strbuf_pflush = reg_base + STRBUF_PFLUSH; | |
1080 | strbuf->strbuf_fsync = reg_base + STRBUF_FSYNC; | |
1081 | ||
1082 | strbuf->strbuf_enabled = 1; | |
1da177e4 | 1083 | |
3e4d2650 DM |
1084 | strbuf->strbuf_flushflag = (volatile unsigned long *) |
1085 | ((((unsigned long)&strbuf->__flushflag_buf[0]) | |
1086 | + 63UL) | |
1087 | & ~63UL); | |
1088 | strbuf->strbuf_flushflag_pa = (unsigned long) | |
1089 | __pa(strbuf->strbuf_flushflag); | |
1da177e4 LT |
1090 | |
1091 | /* The SYSIO SBUS control register is used for dummy reads | |
1092 | * in order to ensure write completion. | |
1093 | */ | |
3e4d2650 | 1094 | iommu->write_complete_reg = regs + 0x2000UL; |
1da177e4 LT |
1095 | |
1096 | /* Link into SYSIO software state. */ | |
3e4d2650 | 1097 | sbus->iommu = info; |
1da177e4 LT |
1098 | |
1099 | printk("SYSIO: UPA portID %x, at %016lx\n", | |
1100 | sbus->portid, regs); | |
1101 | ||
1102 | /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ | |
2f3a2efd DM |
1103 | sbus_iommu_table_init(iommu, IO_TSB_SIZE); |
1104 | ||
3e4d2650 | 1105 | control = upa_readq(iommu->iommu_control); |
1da177e4 LT |
1106 | control = ((7UL << 16UL) | |
1107 | (0UL << 2UL) | | |
1108 | (1UL << 1UL) | | |
1109 | (1UL << 0UL)); | |
3e4d2650 | 1110 | upa_writeq(control, iommu->iommu_control); |
1da177e4 LT |
1111 | |
1112 | /* Clean out any cruft in the IOMMU using | |
1113 | * diagnostic accesses. | |
1114 | */ | |
1115 | for (i = 0; i < 16; i++) { | |
3e4d2650 DM |
1116 | unsigned long dram, tag; |
1117 | ||
1118 | dram = iommu->iommu_control + (IOMMU_DRAMDIAG - IOMMU_CONTROL); | |
1119 | tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL); | |
1da177e4 LT |
1120 | |
1121 | dram += (unsigned long)i * 8UL; | |
1122 | tag += (unsigned long)i * 8UL; | |
1123 | upa_writeq(0, dram); | |
1124 | upa_writeq(0, tag); | |
1125 | } | |
3e4d2650 | 1126 | upa_readq(iommu->write_complete_reg); |
1da177e4 LT |
1127 | |
1128 | /* Give the TSB to SYSIO. */ | |
3e4d2650 | 1129 | upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase); |
1da177e4 LT |
1130 | |
1131 | /* Setup streaming buffer, DE=1 SB_EN=1 */ | |
1132 | control = (1UL << 1UL) | (1UL << 0UL); | |
3e4d2650 | 1133 | upa_writeq(control, strbuf->strbuf_control); |
1da177e4 LT |
1134 | |
1135 | /* Clear out the tags using diagnostics. */ | |
1136 | for (i = 0; i < 16; i++) { | |
1137 | unsigned long ptag, ltag; | |
1138 | ||
3e4d2650 DM |
1139 | ptag = strbuf->strbuf_control + |
1140 | (STRBUF_PTAGDIAG - STRBUF_CONTROL); | |
1141 | ltag = strbuf->strbuf_control + | |
1142 | (STRBUF_LTAGDIAG - STRBUF_CONTROL); | |
1da177e4 LT |
1143 | ptag += (unsigned long)i * 8UL; |
1144 | ltag += (unsigned long)i * 8UL; | |
1145 | ||
1146 | upa_writeq(0UL, ptag); | |
1147 | upa_writeq(0UL, ltag); | |
1148 | } | |
1149 | ||
1150 | /* Enable DVMA arbitration for all devices/slots. */ | |
3e4d2650 | 1151 | control = upa_readq(iommu->write_complete_reg); |
1da177e4 | 1152 | control |= 0x3fUL; |
3e4d2650 | 1153 | upa_writeq(control, iommu->write_complete_reg); |
1da177e4 LT |
1154 | |
1155 | /* Now some Xfire specific grot... */ | |
1156 | if (this_is_starfire) | |
286bbe87 | 1157 | starfire_hookup(sbus->portid); |
1da177e4 LT |
1158 | |
1159 | sysio_register_error_handlers(sbus); | |
1160 | } | |
8fae097d DM |
1161 | |
1162 | void sbus_fill_device_irq(struct sbus_dev *sdev) | |
1163 | { | |
25c7581b | 1164 | struct device_node *dp = of_find_node_by_phandle(sdev->prom_node); |
6a23acf3 | 1165 | const struct linux_prom_irqs *irqs; |
8fae097d | 1166 | |
25c7581b DM |
1167 | irqs = of_get_property(dp, "interrupts", NULL); |
1168 | if (!irqs) { | |
8fae097d DM |
1169 | sdev->irqs[0] = 0; |
1170 | sdev->num_irqs = 0; | |
1171 | } else { | |
1172 | unsigned int pri = irqs[0].pri; | |
1173 | ||
1174 | sdev->num_irqs = 1; | |
1175 | if (pri < 0x20) | |
1176 | pri += sdev->slot * 8; | |
1177 | ||
1178 | sdev->irqs[0] = sbus_build_irq(sdev->bus, pri); | |
1179 | } | |
1180 | } | |
576c352e DM |
1181 | |
1182 | void __init sbus_arch_bus_ranges_init(struct device_node *pn, struct sbus_bus *sbus) | |
1183 | { | |
1184 | } | |
1185 | ||
1186 | void __init sbus_setup_iommu(struct sbus_bus *sbus, struct device_node *dp) | |
1187 | { | |
1188 | sbus_iommu_init(dp->node, sbus); | |
1189 | } | |
1190 | ||
1191 | void __init sbus_setup_arch_props(struct sbus_bus *sbus, struct device_node *dp) | |
1192 | { | |
1193 | } | |
1194 | ||
1195 | int __init sbus_arch_preinit(void) | |
1196 | { | |
1197 | return 0; | |
1198 | } | |
1199 | ||
1200 | void __init sbus_arch_postinit(void) | |
1201 | { | |
1202 | extern void firetruck_init(void); | |
576c352e DM |
1203 | |
1204 | firetruck_init(); | |
576c352e | 1205 | } |