projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sparc32: Un-btfixup PAGE_{NONE,COPY,READONLY,SHARED,KERNEL}.
[linux-block.git] / arch / sparc / mm / srmmu.c
CommitLineData
1da177e4
LT		 1/*
2 * srmmu.c: SRMMU specific routines for memory management.
3 *
4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
9 */
10
1da177e4
LT		 11#include <linux/kernel.h>
12#include <linux/mm.h>
1da177e4
LT		 13#include <linux/vmalloc.h>
14#include <linux/pagemap.h>
15#include <linux/init.h>
16#include <linux/spinlock.h>
17#include <linux/bootmem.h>
18#include <linux/fs.h>
19#include <linux/seq_file.h>
1eeb66a1 20#include <linux/kdebug.h>
949e8274 21#include <linux/log2.h>
5a0e3ad6 22#include <linux/gfp.h>
1da177e4
LT		 23
24#include <asm/bitext.h>
25#include <asm/page.h>
26#include <asm/pgalloc.h>
27#include <asm/pgtable.h>
28#include <asm/io.h>
1da177e4
LT		 29#include <asm/vaddrs.h>
30#include <asm/traps.h>
31#include <asm/smp.h>
32#include <asm/mbus.h>
33#include <asm/cache.h>
34#include <asm/oplib.h>
1da177e4
LT		 35#include <asm/asi.h>
36#include <asm/msi.h>
1da177e4
LT		 37#include <asm/mmu_context.h>
38#include <asm/io-unit.h>
39#include <asm/cacheflush.h>
40#include <asm/tlbflush.h>
41
42/* Now the cpu specific definitions. */
43#include <asm/viking.h>
44#include <asm/mxcc.h>
45#include <asm/ross.h>
46#include <asm/tsunami.h>
47#include <asm/swift.h>
48#include <asm/turbosparc.h>
75d9e346 49#include <asm/leon.h>
1da177e4
LT		 50
51#include <asm/btfixup.h>
52
53enum mbus_module srmmu_modtype;
50215d65 54static unsigned int hwbug_bitmask;
1da177e4
LT		 55int vac_cache_size;
56int vac_line_size;
57
58extern struct resource sparc_iomap;
59
60extern unsigned long last_valid_pfn;
61
50215d65 62static pgd_t *srmmu_swapper_pg_dir;
1da177e4
LT		 63
64#ifdef CONFIG_SMP
65#define FLUSH_BEGIN(mm)
66#define FLUSH_END
67#else
68#define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) {
69#define FLUSH_END }
70#endif
71
72BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
73#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
74
75int flush_page_for_dma_global = 1;
76
77#ifdef CONFIG_SMP
78BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long)
79#define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page)
80#endif
81
82char *srmmu_name;
83
84ctxd_t *srmmu_ctx_table_phys;
50215d65 85static ctxd_t *srmmu_context_table;
1da177e4
LT		 86
87int viking_mxcc_present;
88static DEFINE_SPINLOCK(srmmu_context_spinlock);
89
50215d65 90static int is_hypersparc;
1da177e4 91
50215d65 92static int srmmu_cache_pagetables;
1da177e4
LT		 93
94/* these will be initialized in srmmu_nocache_calcsize() */
50215d65
AB		 95static unsigned long srmmu_nocache_size;
96static unsigned long srmmu_nocache_end;
1da177e4
LT		 97
98/* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */
99#define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
100
101/* The context table is a nocache user with the biggest alignment needs. */
102#define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
103
104void *srmmu_nocache_pool;
105void *srmmu_nocache_bitmap;
106static struct bit_map srmmu_nocache_map;
107
1da177e4
LT		 108static inline unsigned long srmmu_pgd_page(pgd_t pgd)
109{ return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); }
110
111
112static inline int srmmu_pte_none(pte_t pte)
113{ return !(pte_val(pte) & 0xFFFFFFF); }
114
115static inline int srmmu_pte_present(pte_t pte)
116{ return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); }
117
1da177e4
LT		 118static inline int srmmu_pmd_none(pmd_t pmd)
119{ return !(pmd_val(pmd) & 0xFFFFFFF); }
120
121static inline int srmmu_pmd_bad(pmd_t pmd)
122{ return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }
123
124static inline int srmmu_pmd_present(pmd_t pmd)
125{ return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }
126
1da177e4
LT		 127static inline int srmmu_pgd_none(pgd_t pgd)
128{ return !(pgd_val(pgd) & 0xFFFFFFF); }
129
130static inline int srmmu_pgd_bad(pgd_t pgd)
131{ return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }
132
133static inline int srmmu_pgd_present(pgd_t pgd)
134{ return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }
135
1da177e4
LT		 136static inline pte_t srmmu_pte_wrprotect(pte_t pte)
137{ return __pte(pte_val(pte) & ~SRMMU_WRITE);}
138
139static inline pte_t srmmu_pte_mkclean(pte_t pte)
140{ return __pte(pte_val(pte) & ~SRMMU_DIRTY);}
141
142static inline pte_t srmmu_pte_mkold(pte_t pte)
143{ return __pte(pte_val(pte) & ~SRMMU_REF);}
144
145static inline pte_t srmmu_pte_mkwrite(pte_t pte)
146{ return __pte(pte_val(pte) | SRMMU_WRITE);}
147
148static inline pte_t srmmu_pte_mkdirty(pte_t pte)
149{ return __pte(pte_val(pte) | SRMMU_DIRTY);}
150
151static inline pte_t srmmu_pte_mkyoung(pte_t pte)
152{ return __pte(pte_val(pte) | SRMMU_REF);}
153
154/*
155 * Conversion functions: convert a page and protection to a page entry,
156 * and a page entry and page directory to the page they refer to.
157 */
158static pte_t srmmu_mk_pte(struct page *page, pgprot_t pgprot)
159{ return __pte((page_to_pfn(page) << (PAGE_SHIFT-4)) | pgprot_val(pgprot)); }
160
161static pte_t srmmu_mk_pte_phys(unsigned long page, pgprot_t pgprot)
162{ return __pte(((page) >> 4) | pgprot_val(pgprot)); }
163
164static pte_t srmmu_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
165{ return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); }
166
167/* XXX should we hyper_flush_whole_icache here - Anton */
168static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
169{ srmmu_set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }
170
171static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
172{ srmmu_set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); }
173
174static void srmmu_pmd_set(pmd_t *pmdp, pte_t *ptep)
175{
176 unsigned long ptp; /* Physical address, shifted right by 4 */
177 int i;
178
179 ptp = __nocache_pa((unsigned long) ptep) >> 4;
180 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
181 srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
182 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
183 }
184}
185
186static void srmmu_pmd_populate(pmd_t *pmdp, struct page *ptep)
187{
188 unsigned long ptp; /* Physical address, shifted right by 4 */
189 int i;
190
191 ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */
192 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) {
193 srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp);
194 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4);
195 }
196}
197
198static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot)
199{ return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); }
200
201/* to find an entry in a top-level page table... */
3115624e 202static inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address)
1da177e4
LT		 203{ return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); }
204
205/* Find an entry in the second-level page table.. */
206static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address)
207{
208 return (pmd_t *) srmmu_pgd_page(*dir) +
209 ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1));
210}
211
212/* Find an entry in the third-level page table.. */
213static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address)
214{
215 void *pte;
216
217 pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4);
218 return (pte_t *) pte +
219 ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1));
220}
221
222static unsigned long srmmu_swp_type(swp_entry_t entry)
223{
224 return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK;
225}
226
227static unsigned long srmmu_swp_offset(swp_entry_t entry)
228{
229 return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK;
230}
231
232static swp_entry_t srmmu_swp_entry(unsigned long type, unsigned long offset)
233{
234 return (swp_entry_t) {
235 (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT
236 | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT };
237}
238
239/*
240 * size: bytes to allocate in the nocache area.
241 * align: bytes, number to align at.
242 * Returns the virtual address of the allocated area.
243 */
244static unsigned long __srmmu_get_nocache(int size, int align)
245{
246 int offset;
247
248 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
249 printk("Size 0x%x too small for nocache request\n", size);
250 size = SRMMU_NOCACHE_BITMAP_SHIFT;
251 }
252 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) {
253 printk("Size 0x%x unaligned int nocache request\n", size);
254 size += SRMMU_NOCACHE_BITMAP_SHIFT-1;
255 }
256 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
257
258 offset = bit_map_string_get(&srmmu_nocache_map,
259 size >> SRMMU_NOCACHE_BITMAP_SHIFT,
260 align >> SRMMU_NOCACHE_BITMAP_SHIFT);
261 if (offset == -1) {
262 printk("srmmu: out of nocache %d: %d/%d\n",
263 size, (int) srmmu_nocache_size,
264 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
265 return 0;
266 }
267
268 return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
269}
270
50215d65 271static unsigned long srmmu_get_nocache(int size, int align)
1da177e4
LT		 272{
273 unsigned long tmp;
274
275 tmp = __srmmu_get_nocache(size, align);
276
277 if (tmp)
278 memset((void *)tmp, 0, size);
279
280 return tmp;
281}
282
50215d65 283static void srmmu_free_nocache(unsigned long vaddr, int size)
1da177e4
LT		 284{
285 int offset;
286
287 if (vaddr < SRMMU_NOCACHE_VADDR) {
288 printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
289 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
290 BUG();
291 }
292 if (vaddr+size > srmmu_nocache_end) {
293 printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
294 vaddr, srmmu_nocache_end);
295 BUG();
296 }
949e8274 297 if (!is_power_of_2(size)) {
1da177e4
LT		 298 printk("Size 0x%x is not a power of 2\n", size);
299 BUG();
300 }
301 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
302 printk("Size 0x%x is too small\n", size);
303 BUG();
304 }
305 if (vaddr & (size-1)) {
306 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
307 BUG();
308 }
309
310 offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
311 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
312
313 bit_map_clear(&srmmu_nocache_map, offset, size);
314}
315
50215d65
AB		 316static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
317 unsigned long end);
1da177e4
LT		 318
319extern unsigned long probe_memory(void); /* in fault.c */
320
321/*
322 * Reserve nocache dynamically proportionally to the amount of
323 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
324 */
50215d65 325static void srmmu_nocache_calcsize(void)
1da177e4
LT		 326{
327 unsigned long sysmemavail = probe_memory() / 1024;
328 int srmmu_nocache_npages;
329
330 srmmu_nocache_npages =
331 sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256;
332
333 /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */
334 // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256;
335 if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES)
336 srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES;
337
338 /* anything above 1280 blows up */
339 if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES)
340 srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES;
341
342 srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE;
343 srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size;
344}
345
50215d65 346static void __init srmmu_nocache_init(void)
1da177e4
LT		 347{
348 unsigned int bitmap_bits;
349 pgd_t *pgd;
350 pmd_t *pmd;
351 pte_t *pte;
352 unsigned long paddr, vaddr;
353 unsigned long pteval;
354
355 bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT;
356
357 srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size,
358 SRMMU_NOCACHE_ALIGN_MAX, 0UL);
359 memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
360
361 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
362 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
363
364 srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
365 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
366 init_mm.pgd = srmmu_swapper_pg_dir;
367
368 srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end);
369
370 paddr = __pa((unsigned long)srmmu_nocache_pool);
371 vaddr = SRMMU_NOCACHE_VADDR;
372
373 while (vaddr < srmmu_nocache_end) {
374 pgd = pgd_offset_k(vaddr);
375 pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr);
376 pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr);
377
378 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
379
380 if (srmmu_cache_pagetables)
381 pteval |= SRMMU_CACHE;
382
383 srmmu_set_pte(__nocache_fix(pte), __pte(pteval));
384
385 vaddr += PAGE_SIZE;
386 paddr += PAGE_SIZE;
387 }
388
389 flush_cache_all();
390 flush_tlb_all();
391}
392
393static inline pgd_t *srmmu_get_pgd_fast(void)
394{
395 pgd_t *pgd = NULL;
396
397 pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
398 if (pgd) {
399 pgd_t *init = pgd_offset_k(0);
400 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
401 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
402 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
403 }
404
405 return pgd;
406}
407
408static void srmmu_free_pgd_fast(pgd_t *pgd)
409{
410 srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE);
411}
412
413static pmd_t *srmmu_pmd_alloc_one(struct mm_struct *mm, unsigned long address)
414{
415 return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
416}
417
418static void srmmu_pmd_free(pmd_t * pmd)
419{
420 srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE);
421}
422
423/*
424 * Hardware needs alignment to 256 only, but we align to whole page size
425 * to reduce fragmentation problems due to the buddy principle.
426 * XXX Provide actual fragmentation statistics in /proc.
427 *
428 * Alignments up to the page size are the same for physical and virtual
429 * addresses of the nocache area.
430 */
431static pte_t *
432srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
433{
434 return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
435}
436
2f569afd 437static pgtable_t
1da177e4
LT		 438srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address)
439{
440 unsigned long pte;
2f569afd 441 struct page *page;
1da177e4
LT		 442
443 if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0)
444 return NULL;
2f569afd
MS		 445 page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
446 pgtable_page_ctor(page);
447 return page;
1da177e4
LT		 448}
449
450static void srmmu_free_pte_fast(pte_t *pte)
451{
452 srmmu_free_nocache((unsigned long)pte, PTE_SIZE);
453}
454
2f569afd 455static void srmmu_pte_free(pgtable_t pte)
1da177e4
LT		 456{
457 unsigned long p;
458
2f569afd 459 pgtable_page_dtor(pte);
1da177e4
LT		 460 p = (unsigned long)page_address(pte); /* Cached address (for test) */
461 if (p == 0)
462 BUG();
463 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */
464 p = (unsigned long) __nocache_va(p); /* Nocached virtual */
465 srmmu_free_nocache(p, PTE_SIZE);
466}
467
468/*
469 */
470static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
471{
472 struct ctx_list *ctxp;
473
474 ctxp = ctx_free.next;
475 if(ctxp != &ctx_free) {
476 remove_from_ctx_list(ctxp);
477 add_to_used_ctxlist(ctxp);
478 mm->context = ctxp->ctx_number;
479 ctxp->ctx_mm = mm;
480 return;
481 }
482 ctxp = ctx_used.next;
483 if(ctxp->ctx_mm == old_mm)
484 ctxp = ctxp->next;
485 if(ctxp == &ctx_used)
486 panic("out of mmu contexts");
487 flush_cache_mm(ctxp->ctx_mm);
488 flush_tlb_mm(ctxp->ctx_mm);
489 remove_from_ctx_list(ctxp);
490 add_to_used_ctxlist(ctxp);
491 ctxp->ctx_mm->context = NO_CONTEXT;
492 ctxp->ctx_mm = mm;
493 mm->context = ctxp->ctx_number;
494}
495
496static inline void free_context(int context)
497{
498 struct ctx_list *ctx_old;
499
500 ctx_old = ctx_list_pool + context;
501 remove_from_ctx_list(ctx_old);
502 add_to_free_ctxlist(ctx_old);
503}
504
505
34d4accf
SR		 506void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
507 struct task_struct *tsk)
1da177e4
LT		 508{
509 if(mm->context == NO_CONTEXT) {
510 spin_lock(&srmmu_context_spinlock);
511 alloc_context(old_mm, mm);
512 spin_unlock(&srmmu_context_spinlock);
513 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
514 }
515
75d9e346
KE		 516 if (sparc_cpu_model == sparc_leon)
517 leon_switch_mm();
518
1da177e4
LT		 519 if (is_hypersparc)
520 hyper_flush_whole_icache();
521
522 srmmu_set_context(mm->context);
523}
524
525/* Low level IO area allocation on the SRMMU. */
526static inline void srmmu_mapioaddr(unsigned long physaddr,
527 unsigned long virt_addr, int bus_type)
528{
529 pgd_t *pgdp;
530 pmd_t *pmdp;
531 pte_t *ptep;
532 unsigned long tmp;
533
534 physaddr &= PAGE_MASK;
535 pgdp = pgd_offset_k(virt_addr);
536 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
537 ptep = srmmu_pte_offset(pmdp, virt_addr);
538 tmp = (physaddr >> 4) | SRMMU_ET_PTE;
539
540 /*
541 * I need to test whether this is consistent over all
542 * sun4m's. The bus_type represents the upper 4 bits of
543 * 36-bit physical address on the I/O space lines...
544 */
545 tmp |= (bus_type << 28);
546 tmp |= SRMMU_PRIV;
547 __flush_page_to_ram(virt_addr);
548 srmmu_set_pte(ptep, __pte(tmp));
549}
550
551static void srmmu_mapiorange(unsigned int bus, unsigned long xpa,
552 unsigned long xva, unsigned int len)
553{
554 while (len != 0) {
555 len -= PAGE_SIZE;
556 srmmu_mapioaddr(xpa, xva, bus);
557 xva += PAGE_SIZE;
558 xpa += PAGE_SIZE;
559 }
560 flush_tlb_all();
561}
562
563static inline void srmmu_unmapioaddr(unsigned long virt_addr)
564{
565 pgd_t *pgdp;
566 pmd_t *pmdp;
567 pte_t *ptep;
568
569 pgdp = pgd_offset_k(virt_addr);
570 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
571 ptep = srmmu_pte_offset(pmdp, virt_addr);
572
573 /* No need to flush uncacheable page. */
a46d6056 574 __pte_clear(ptep);
1da177e4
LT		 575}
576
577static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len)
578{
579 while (len != 0) {
580 len -= PAGE_SIZE;
581 srmmu_unmapioaddr(virt_addr);
582 virt_addr += PAGE_SIZE;
583 }
584 flush_tlb_all();
585}
586
587/*
588 * On the SRMMU we do not have the problems with limited tlb entries
589 * for mapping kernel pages, so we just take things from the free page
590 * pool. As a side effect we are putting a little too much pressure
591 * on the gfp() subsystem. This setup also makes the logic of the
592 * iommu mapping code a lot easier as we can transparently handle
ee906c9e 593 * mappings on the kernel stack without any special code.
1da177e4 594 */
e7b7e0c3 595struct thread_info *alloc_thread_info_node(struct task_struct *tsk, int node)
1da177e4
LT		 596{
597 struct thread_info *ret;
598
599 ret = (struct thread_info *)__get_free_pages(GFP_KERNEL,
600 THREAD_INFO_ORDER);
601#ifdef CONFIG_DEBUG_STACK_USAGE
602 if (ret)
603 memset(ret, 0, PAGE_SIZE << THREAD_INFO_ORDER);
604#endif /* DEBUG_STACK_USAGE */
605
606 return ret;
607}
608
e7b7e0c3 609void free_thread_info(struct thread_info *ti)
1da177e4
LT		 610{
611 free_pages((unsigned long)ti, THREAD_INFO_ORDER);
612}
613
614/* tsunami.S */
615extern void tsunami_flush_cache_all(void);
616extern void tsunami_flush_cache_mm(struct mm_struct *mm);
617extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
618extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
619extern void tsunami_flush_page_to_ram(unsigned long page);
620extern void tsunami_flush_page_for_dma(unsigned long page);
621extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
622extern void tsunami_flush_tlb_all(void);
623extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
624extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
625extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
626extern void tsunami_setup_blockops(void);
627
628/*
629 * Workaround, until we find what's going on with Swift. When low on memory,
630 * it sometimes loops in fault/handle_mm_fault incl. flush_tlb_page to find
631 * out it is already in page tables/ fault again on the same instruction.
632 * I really don't understand it, have checked it and contexts
633 * are right, flush_tlb_all is done as well, and it faults again...
634 * Strange. -jj
635 *
636 * The following code is a deadwood that may be necessary when
637 * we start to make precise page flushes again. --zaitcev
638 */
4b3073e1 639static void swift_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t *ptep)
1da177e4
LT		 640{
641#if 0
642 static unsigned long last;
643 unsigned int val;
644 /* unsigned int n; */
645
646 if (address == last) {
647 val = srmmu_hwprobe(address);
4b3073e1 648 if (val != 0 && pte_val(*ptep) != val) {
1da177e4 649 printk("swift_update_mmu_cache: "
e9b57cca 650 "addr %lx put %08x probed %08x from %pf\n",
4b3073e1 651 address, pte_val(*ptep), val,
1da177e4
LT		 652 __builtin_return_address(0));
653 srmmu_flush_whole_tlb();
654 }
655 }
656 last = address;
657#endif
658}
659
660/* swift.S */
661extern void swift_flush_cache_all(void);
662extern void swift_flush_cache_mm(struct mm_struct *mm);
663extern void swift_flush_cache_range(struct vm_area_struct *vma,
664 unsigned long start, unsigned long end);
665extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
666extern void swift_flush_page_to_ram(unsigned long page);
667extern void swift_flush_page_for_dma(unsigned long page);
668extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
669extern void swift_flush_tlb_all(void);
670extern void swift_flush_tlb_mm(struct mm_struct *mm);
671extern void swift_flush_tlb_range(struct vm_area_struct *vma,
672 unsigned long start, unsigned long end);
673extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
674
675#if 0 /* P3: deadwood to debug precise flushes on Swift. */
676void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
677{
678 int cctx, ctx1;
679
680 page &= PAGE_MASK;
681 if ((ctx1 = vma->vm_mm->context) != -1) {
682 cctx = srmmu_get_context();
683/* Is context # ever different from current context? P3 */
684 if (cctx != ctx1) {
685 printk("flush ctx %02x curr %02x\n", ctx1, cctx);
686 srmmu_set_context(ctx1);
687 swift_flush_page(page);
688 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
689 "r" (page), "i" (ASI_M_FLUSH_PROBE));
690 srmmu_set_context(cctx);
691 } else {
692 /* Rm. prot. bits from virt. c. */
693 /* swift_flush_cache_all(); */
694 /* swift_flush_cache_page(vma, page); */
695 swift_flush_page(page);
696
697 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
698 "r" (page), "i" (ASI_M_FLUSH_PROBE));
699 /* same as above: srmmu_flush_tlb_page() */
700 }
701 }
702}
703#endif
704
705/*
706 * The following are all MBUS based SRMMU modules, and therefore could
707 * be found in a multiprocessor configuration. On the whole, these
708 * chips seems to be much more touchy about DVMA and page tables
709 * with respect to cache coherency.
710 */
711
712/* Cypress flushes. */
713static void cypress_flush_cache_all(void)
714{
715 volatile unsigned long cypress_sucks;
716 unsigned long faddr, tagval;
717
718 flush_user_windows();
719 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
720 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
721 "=r" (tagval) :
722 "r" (faddr), "r" (0x40000),
723 "i" (ASI_M_DATAC_TAG));
724
725 /* If modified and valid, kick it. */
726 if((tagval & 0x60) == 0x60)
727 cypress_sucks = *(unsigned long *)(0xf0020000 + faddr);
728 }
729}
730
731static void cypress_flush_cache_mm(struct mm_struct *mm)
732{
733 register unsigned long a, b, c, d, e, f, g;
734 unsigned long flags, faddr;
735 int octx;
736
737 FLUSH_BEGIN(mm)
738 flush_user_windows();
739 local_irq_save(flags);
740 octx = srmmu_get_context();
741 srmmu_set_context(mm->context);
742 a = 0x20; b = 0x40; c = 0x60;
743 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
744
745 faddr = (0x10000 - 0x100);
746 goto inside;
747 do {
748 faddr -= 0x100;
749 inside:
750 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
751 "sta %%g0, [%0 + %2] %1\n\t"
752 "sta %%g0, [%0 + %3] %1\n\t"
753 "sta %%g0, [%0 + %4] %1\n\t"
754 "sta %%g0, [%0 + %5] %1\n\t"
755 "sta %%g0, [%0 + %6] %1\n\t"
756 "sta %%g0, [%0 + %7] %1\n\t"
757 "sta %%g0, [%0 + %8] %1\n\t" : :
758 "r" (faddr), "i" (ASI_M_FLUSH_CTX),
759 "r" (a), "r" (b), "r" (c), "r" (d),
760 "r" (e), "r" (f), "r" (g));
761 } while(faddr);
762 srmmu_set_context(octx);
763 local_irq_restore(flags);
764 FLUSH_END
765}
766
767static void cypress_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
768{
769 struct mm_struct *mm = vma->vm_mm;
770 register unsigned long a, b, c, d, e, f, g;
771 unsigned long flags, faddr;
772 int octx;
773
774 FLUSH_BEGIN(mm)
775 flush_user_windows();
776 local_irq_save(flags);
777 octx = srmmu_get_context();
778 srmmu_set_context(mm->context);
779 a = 0x20; b = 0x40; c = 0x60;
780 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
781
782 start &= SRMMU_REAL_PMD_MASK;
783 while(start < end) {
784 faddr = (start + (0x10000 - 0x100));
785 goto inside;
786 do {
787 faddr -= 0x100;
788 inside:
789 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
790 "sta %%g0, [%0 + %2] %1\n\t"
791 "sta %%g0, [%0 + %3] %1\n\t"
792 "sta %%g0, [%0 + %4] %1\n\t"
793 "sta %%g0, [%0 + %5] %1\n\t"
794 "sta %%g0, [%0 + %6] %1\n\t"
795 "sta %%g0, [%0 + %7] %1\n\t"
796 "sta %%g0, [%0 + %8] %1\n\t" : :
797 "r" (faddr),
798 "i" (ASI_M_FLUSH_SEG),
799 "r" (a), "r" (b), "r" (c), "r" (d),
800 "r" (e), "r" (f), "r" (g));
801 } while (faddr != start);
802 start += SRMMU_REAL_PMD_SIZE;
803 }
804 srmmu_set_context(octx);
805 local_irq_restore(flags);
806 FLUSH_END
807}
808
809static void cypress_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
810{
811 register unsigned long a, b, c, d, e, f, g;
812 struct mm_struct *mm = vma->vm_mm;
813 unsigned long flags, line;
814 int octx;
815
816 FLUSH_BEGIN(mm)
817 flush_user_windows();
818 local_irq_save(flags);
819 octx = srmmu_get_context();
820 srmmu_set_context(mm->context);
821 a = 0x20; b = 0x40; c = 0x60;
822 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
823
824 page &= PAGE_MASK;
825 line = (page + PAGE_SIZE) - 0x100;
826 goto inside;
827 do {
828 line -= 0x100;
829 inside:
830 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
831 "sta %%g0, [%0 + %2] %1\n\t"
832 "sta %%g0, [%0 + %3] %1\n\t"
833 "sta %%g0, [%0 + %4] %1\n\t"
834 "sta %%g0, [%0 + %5] %1\n\t"
835 "sta %%g0, [%0 + %6] %1\n\t"
836 "sta %%g0, [%0 + %7] %1\n\t"
837 "sta %%g0, [%0 + %8] %1\n\t" : :
838 "r" (line),
839 "i" (ASI_M_FLUSH_PAGE),
840 "r" (a), "r" (b), "r" (c), "r" (d),
841 "r" (e), "r" (f), "r" (g));
842 } while(line != page);
843 srmmu_set_context(octx);
844 local_irq_restore(flags);
845 FLUSH_END
846}
847
848/* Cypress is copy-back, at least that is how we configure it. */
849static void cypress_flush_page_to_ram(unsigned long page)
850{
851 register unsigned long a, b, c, d, e, f, g;
852 unsigned long line;
853
854 a = 0x20; b = 0x40; c = 0x60; d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
855 page &= PAGE_MASK;
856 line = (page + PAGE_SIZE) - 0x100;
857 goto inside;
858 do {
859 line -= 0x100;
860 inside:
861 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
862 "sta %%g0, [%0 + %2] %1\n\t"
863 "sta %%g0, [%0 + %3] %1\n\t"
864 "sta %%g0, [%0 + %4] %1\n\t"
865 "sta %%g0, [%0 + %5] %1\n\t"
866 "sta %%g0, [%0 + %6] %1\n\t"
867 "sta %%g0, [%0 + %7] %1\n\t"
868 "sta %%g0, [%0 + %8] %1\n\t" : :
869 "r" (line),
870 "i" (ASI_M_FLUSH_PAGE),
871 "r" (a), "r" (b), "r" (c), "r" (d),
872 "r" (e), "r" (f), "r" (g));
873 } while(line != page);
874}
875
876/* Cypress is also IO cache coherent. */
877static void cypress_flush_page_for_dma(unsigned long page)
878{
879}
880
881/* Cypress has unified L2 VIPT, from which both instructions and data
882 * are stored. It does not have an onboard icache of any sort, therefore
883 * no flush is necessary.
884 */
885static void cypress_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
886{
887}
888
889static void cypress_flush_tlb_all(void)
890{
891 srmmu_flush_whole_tlb();
892}
893
894static void cypress_flush_tlb_mm(struct mm_struct *mm)
895{
896 FLUSH_BEGIN(mm)
897 __asm__ __volatile__(
898 "lda [%0] %3, %%g5\n\t"
899 "sta %2, [%0] %3\n\t"
900 "sta %%g0, [%1] %4\n\t"
901 "sta %%g5, [%0] %3\n"
902 : /* no outputs */
903 : "r" (SRMMU_CTX_REG), "r" (0x300), "r" (mm->context),
904 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
905 : "g5");
906 FLUSH_END
907}
908
909static void cypress_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
910{
911 struct mm_struct *mm = vma->vm_mm;
912 unsigned long size;
913
914 FLUSH_BEGIN(mm)
915 start &= SRMMU_PGDIR_MASK;
916 size = SRMMU_PGDIR_ALIGN(end) - start;
917 __asm__ __volatile__(
918 "lda [%0] %5, %%g5\n\t"
919 "sta %1, [%0] %5\n"
920 "1:\n\t"
921 "subcc %3, %4, %3\n\t"
922 "bne 1b\n\t"
923 " sta %%g0, [%2 + %3] %6\n\t"
924 "sta %%g5, [%0] %5\n"
925 : /* no outputs */
926 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (start | 0x200),
927 "r" (size), "r" (SRMMU_PGDIR_SIZE), "i" (ASI_M_MMUREGS),
928 "i" (ASI_M_FLUSH_PROBE)
929 : "g5", "cc");
930 FLUSH_END
931}
932
933static void cypress_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
934{
935 struct mm_struct *mm = vma->vm_mm;
936
937 FLUSH_BEGIN(mm)
938 __asm__ __volatile__(
939 "lda [%0] %3, %%g5\n\t"
940 "sta %1, [%0] %3\n\t"
941 "sta %%g0, [%2] %4\n\t"
942 "sta %%g5, [%0] %3\n"
943 : /* no outputs */
944 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (page & PAGE_MASK),
945 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
946 : "g5");
947 FLUSH_END
948}
949
950/* viking.S */
951extern void viking_flush_cache_all(void);
952extern void viking_flush_cache_mm(struct mm_struct *mm);
953extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start,
954 unsigned long end);
955extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
956extern void viking_flush_page_to_ram(unsigned long page);
957extern void viking_flush_page_for_dma(unsigned long page);
958extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
959extern void viking_flush_page(unsigned long page);
960extern void viking_mxcc_flush_page(unsigned long page);
961extern void viking_flush_tlb_all(void);
962extern void viking_flush_tlb_mm(struct mm_struct *mm);
963extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
964 unsigned long end);
965extern void viking_flush_tlb_page(struct vm_area_struct *vma,
966 unsigned long page);
967extern void sun4dsmp_flush_tlb_all(void);
968extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
969extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
970 unsigned long end);
971extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
972 unsigned long page);
973
974/* hypersparc.S */
975extern void hypersparc_flush_cache_all(void);
976extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
977extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
978extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
979extern void hypersparc_flush_page_to_ram(unsigned long page);
980extern void hypersparc_flush_page_for_dma(unsigned long page);
981extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
982extern void hypersparc_flush_tlb_all(void);
983extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
984extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
985extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
986extern void hypersparc_setup_blockops(void);
987
988/*
989 * NOTE: All of this startup code assumes the low 16mb (approx.) of
990 * kernel mappings are done with one single contiguous chunk of
991 * ram. On small ram machines (classics mainly) we only get
992 * around 8mb mapped for us.
993 */
994
50215d65 995static void __init early_pgtable_allocfail(char *type)
1da177e4
LT		 996{
997 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
998 prom_halt();
999}
1000
50215d65
AB		 1001static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
1002 unsigned long end)
1da177e4
LT		 1003{
1004 pgd_t *pgdp;
1005 pmd_t *pmdp;
1006 pte_t *ptep;
1007
1008 while(start < end) {
1009 pgdp = pgd_offset_k(start);
1010 if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
1011 pmdp = (pmd_t *) __srmmu_get_nocache(
1012 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1013 if (pmdp == NULL)
1014 early_pgtable_allocfail("pmd");
1015 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
1016 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
1017 }
1018 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
1019 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
1020 ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
1021 if (ptep == NULL)
1022 early_pgtable_allocfail("pte");
1023 memset(__nocache_fix(ptep), 0, PTE_SIZE);
1024 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
1025 }
1026 if (start > (0xffffffffUL - PMD_SIZE))
1027 break;
1028 start = (start + PMD_SIZE) & PMD_MASK;
1029 }
1030}
1031
50215d65
AB		 1032static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
1033 unsigned long end)
1da177e4
LT		 1034{
1035 pgd_t *pgdp;
1036 pmd_t *pmdp;
1037 pte_t *ptep;
1038
1039 while(start < end) {
1040 pgdp = pgd_offset_k(start);
1041 if(srmmu_pgd_none(*pgdp)) {
1042 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1043 if (pmdp == NULL)
1044 early_pgtable_allocfail("pmd");
1045 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
1046 srmmu_pgd_set(pgdp, pmdp);
1047 }
1048 pmdp = srmmu_pmd_offset(pgdp, start);
1049 if(srmmu_pmd_none(*pmdp)) {
1050 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
1051 PTE_SIZE);
1052 if (ptep == NULL)
1053 early_pgtable_allocfail("pte");
1054 memset(ptep, 0, PTE_SIZE);
1055 srmmu_pmd_set(pmdp, ptep);
1056 }
1057 if (start > (0xffffffffUL - PMD_SIZE))
1058 break;
1059 start = (start + PMD_SIZE) & PMD_MASK;
1060 }
1061}
1062
1063/*
1064 * This is much cleaner than poking around physical address space
1065 * looking at the prom's page table directly which is what most
1066 * other OS's do. Yuck... this is much better.
1067 */
50215d65
AB		 1068static void __init srmmu_inherit_prom_mappings(unsigned long start,
1069 unsigned long end)
1da177e4
LT		 1070{
1071 pgd_t *pgdp;
1072 pmd_t *pmdp;
1073 pte_t *ptep;
1074 int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
1075 unsigned long prompte;
1076
1077 while(start <= end) {
1078 if (start == 0)
1079 break; /* probably wrap around */
1080 if(start == 0xfef00000)
1081 start = KADB_DEBUGGER_BEGVM;
1082 if(!(prompte = srmmu_hwprobe(start))) {
1083 start += PAGE_SIZE;
1084 continue;
1085 }
1086
1087 /* A red snapper, see what it really is. */
1088 what = 0;
1089
1090 if(!(start & ~(SRMMU_REAL_PMD_MASK))) {
1091 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte)
1092 what = 1;
1093 }
1094
1095 if(!(start & ~(SRMMU_PGDIR_MASK))) {
1096 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) ==
1097 prompte)
1098 what = 2;
1099 }
1100
1101 pgdp = pgd_offset_k(start);
1102 if(what == 2) {
1103 *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte);
1104 start += SRMMU_PGDIR_SIZE;
1105 continue;
1106 }
1107 if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
1108 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1109 if (pmdp == NULL)
1110 early_pgtable_allocfail("pmd");
1111 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
1112 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
1113 }
1114 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
1115 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
1116 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
1117 PTE_SIZE);
1118 if (ptep == NULL)
1119 early_pgtable_allocfail("pte");
1120 memset(__nocache_fix(ptep), 0, PTE_SIZE);
1121 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
1122 }
1123 if(what == 1) {
1124 /*
1125 * We bend the rule where all 16 PTPs in a pmd_t point
1126 * inside the same PTE page, and we leak a perfectly
1127 * good hardware PTE piece. Alternatives seem worse.
1128 */
1129 unsigned int x; /* Index of HW PMD in soft cluster */
1130 x = (start >> PMD_SHIFT) & 15;
1131 *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte;
1132 start += SRMMU_REAL_PMD_SIZE;
1133 continue;
1134 }
1135 ptep = srmmu_pte_offset(__nocache_fix(pmdp), start);
1136 *(pte_t *)__nocache_fix(ptep) = __pte(prompte);
1137 start += PAGE_SIZE;
1138 }
1139}
1140
1141#define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
1142
1143/* Create a third-level SRMMU 16MB page mapping. */
1144static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
1145{
1146 pgd_t *pgdp = pgd_offset_k(vaddr);
1147 unsigned long big_pte;
1148
1149 big_pte = KERNEL_PTE(phys_base >> 4);
1150 *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
1151}
1152
1153/* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
1154static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
1155{
1156 unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
1157 unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
1158 unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
1159 /* Map "low" memory only */
1160 const unsigned long min_vaddr = PAGE_OFFSET;
1161 const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
1162
1163 if (vstart < min_vaddr || vstart >= max_vaddr)
1164 return vstart;
1165
1166 if (vend > max_vaddr || vend < min_vaddr)
1167 vend = max_vaddr;
1168
1169 while(vstart < vend) {
1170 do_large_mapping(vstart, pstart);
1171 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
1172 }
1173 return vstart;
1174}
1175
1176static inline void memprobe_error(char *msg)
1177{
1178 prom_printf(msg);
1179 prom_printf("Halting now...\n");
1180 prom_halt();
1181}
1182
1183static inline void map_kernel(void)
1184{
1185 int i;
1186
1187 if (phys_base > 0) {
1188 do_large_mapping(PAGE_OFFSET, phys_base);
1189 }
1190
1191 for (i = 0; sp_banks[i].num_bytes != 0; i++) {
1192 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
1193 }
1da177e4
LT		 1194}
1195
1196/* Paging initialization on the Sparc Reference MMU. */
1197extern void sparc_context_init(int);
1198
409832f5 1199void (*poke_srmmu)(void) __cpuinitdata = NULL;
1da177e4
LT		 1200
1201extern unsigned long bootmem_init(unsigned long *pages_avail);
1202
1203void __init srmmu_paging_init(void)
1204{
8d125562
AS		 1205 int i;
1206 phandle cpunode;
1da177e4
LT		 1207 char node_str[128];
1208 pgd_t *pgd;
1209 pmd_t *pmd;
1210 pte_t *pte;
1211 unsigned long pages_avail;
1212
1213 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
1214
1215 if (sparc_cpu_model == sun4d)
1216 num_contexts = 65536; /* We know it is Viking */
1217 else {
1218 /* Find the number of contexts on the srmmu. */
1219 cpunode = prom_getchild(prom_root_node);
1220 num_contexts = 0;
1221 while(cpunode != 0) {
1222 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1223 if(!strcmp(node_str, "cpu")) {
1224 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
1225 break;
1226 }
1227 cpunode = prom_getsibling(cpunode);
1228 }
1229 }
1230
1231 if(!num_contexts) {
1232 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
1233 prom_halt();
1234 }
1235
1236 pages_avail = 0;
1237 last_valid_pfn = bootmem_init(&pages_avail);
1238
1239 srmmu_nocache_calcsize();
1240 srmmu_nocache_init();
1241 srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE));
1242 map_kernel();
1243
1244 /* ctx table has to be physically aligned to its size */
1245 srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t));
1246 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
1247
1248 for(i = 0; i < num_contexts; i++)
1249 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
1250
1251 flush_cache_all();
1252 srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
a54123e2
BB		 1253#ifdef CONFIG_SMP
1254 /* Stop from hanging here... */
1255 local_flush_tlb_all();
1256#else
1da177e4 1257 flush_tlb_all();
a54123e2 1258#endif
1da177e4
LT		 1259 poke_srmmu();
1260
1da177e4
LT		 1261 srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
1262 srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
1da177e4
LT		 1263
1264 srmmu_allocate_ptable_skeleton(
1265 __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP);
1266 srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END);
1267
1268 pgd = pgd_offset_k(PKMAP_BASE);
1269 pmd = srmmu_pmd_offset(pgd, PKMAP_BASE);
1270 pte = srmmu_pte_offset(pmd, PKMAP_BASE);
1271 pkmap_page_table = pte;
1272
1273 flush_cache_all();
1274 flush_tlb_all();
1275
1276 sparc_context_init(num_contexts);
1277
1278 kmap_init();
1279
1280 {
1281 unsigned long zones_size[MAX_NR_ZONES];
1282 unsigned long zholes_size[MAX_NR_ZONES];
1283 unsigned long npages;
1284 int znum;
1285
1286 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1287 zones_size[znum] = zholes_size[znum] = 0;
1288
1289 npages = max_low_pfn - pfn_base;
1290
1291 zones_size[ZONE_DMA] = npages;
1292 zholes_size[ZONE_DMA] = npages - pages_avail;
1293
1294 npages = highend_pfn - max_low_pfn;
1295 zones_size[ZONE_HIGHMEM] = npages;
1296 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
1297
9109fb7b 1298 free_area_init_node(0, zones_size, pfn_base, zholes_size);
1da177e4
LT		 1299 }
1300}
1301
1302static void srmmu_mmu_info(struct seq_file *m)
1303{
1304 seq_printf(m,
1305 "MMU type\t: %s\n"
1306 "contexts\t: %d\n"
1307 "nocache total\t: %ld\n"
1308 "nocache used\t: %d\n",
1309 srmmu_name,
1310 num_contexts,
1311 srmmu_nocache_size,
1312 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
1313}
1314
1315static void srmmu_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte)
1316{
1317}
1318
1319static void srmmu_destroy_context(struct mm_struct *mm)
1320{
1321
1322 if(mm->context != NO_CONTEXT) {
1323 flush_cache_mm(mm);
1324 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
1325 flush_tlb_mm(mm);
1326 spin_lock(&srmmu_context_spinlock);
1327 free_context(mm->context);
1328 spin_unlock(&srmmu_context_spinlock);
1329 mm->context = NO_CONTEXT;
1330 }
1331}
1332
1333/* Init various srmmu chip types. */
1334static void __init srmmu_is_bad(void)
1335{
1336 prom_printf("Could not determine SRMMU chip type.\n");
1337 prom_halt();
1338}
1339
1340static void __init init_vac_layout(void)
1341{
8d125562
AS		 1342 phandle nd;
1343 int cache_lines;
1da177e4
LT		 1344 char node_str[128];
1345#ifdef CONFIG_SMP
1346 int cpu = 0;
1347 unsigned long max_size = 0;
1348 unsigned long min_line_size = 0x10000000;
1349#endif
1350
1351 nd = prom_getchild(prom_root_node);
1352 while((nd = prom_getsibling(nd)) != 0) {
1353 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
1354 if(!strcmp(node_str, "cpu")) {
1355 vac_line_size = prom_getint(nd, "cache-line-size");
1356 if (vac_line_size == -1) {
1357 prom_printf("can't determine cache-line-size, "
1358 "halting.\n");
1359 prom_halt();
1360 }
1361 cache_lines = prom_getint(nd, "cache-nlines");
1362 if (cache_lines == -1) {
1363 prom_printf("can't determine cache-nlines, halting.\n");
1364 prom_halt();
1365 }
1366
1367 vac_cache_size = cache_lines * vac_line_size;
1368#ifdef CONFIG_SMP
1369 if(vac_cache_size > max_size)
1370 max_size = vac_cache_size;
1371 if(vac_line_size < min_line_size)
1372 min_line_size = vac_line_size;
a54123e2 1373 //FIXME: cpus not contiguous!!
1da177e4 1374 cpu++;
ec7c14bd 1375 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
1da177e4
LT		 1376 break;
1377#else
1378 break;
1379#endif
1380 }
1381 }
1382 if(nd == 0) {
1383 prom_printf("No CPU nodes found, halting.\n");
1384 prom_halt();
1385 }
1386#ifdef CONFIG_SMP
1387 vac_cache_size = max_size;
1388 vac_line_size = min_line_size;
1389#endif
1390 printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1391 (int)vac_cache_size, (int)vac_line_size);
1392}
1393
409832f5 1394static void __cpuinit poke_hypersparc(void)
1da177e4
LT		 1395{
1396 volatile unsigned long clear;
1397 unsigned long mreg = srmmu_get_mmureg();
1398
1399 hyper_flush_unconditional_combined();
1400
1401 mreg &= ~(HYPERSPARC_CWENABLE);
1402 mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1403 mreg |= (HYPERSPARC_CMODE);
1404
1405 srmmu_set_mmureg(mreg);
1406
1407#if 0 /* XXX I think this is bad news... -DaveM */
1408 hyper_clear_all_tags();
1409#endif
1410
1411 put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1412 hyper_flush_whole_icache();
1413 clear = srmmu_get_faddr();
1414 clear = srmmu_get_fstatus();
1415}
1416
1417static void __init init_hypersparc(void)
1418{
1419 srmmu_name = "ROSS HyperSparc";
1420 srmmu_modtype = HyperSparc;
1421
1422 init_vac_layout();
1423
1424 is_hypersparc = 1;
1425
1da177e4
LT		 1426 BTFIXUPSET_CALL(flush_cache_all, hypersparc_flush_cache_all, BTFIXUPCALL_NORM);
1427 BTFIXUPSET_CALL(flush_cache_mm, hypersparc_flush_cache_mm, BTFIXUPCALL_NORM);
1428 BTFIXUPSET_CALL(flush_cache_range, hypersparc_flush_cache_range, BTFIXUPCALL_NORM);
1429 BTFIXUPSET_CALL(flush_cache_page, hypersparc_flush_cache_page, BTFIXUPCALL_NORM);
1430
1431 BTFIXUPSET_CALL(flush_tlb_all, hypersparc_flush_tlb_all, BTFIXUPCALL_NORM);
1432 BTFIXUPSET_CALL(flush_tlb_mm, hypersparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1433 BTFIXUPSET_CALL(flush_tlb_range, hypersparc_flush_tlb_range, BTFIXUPCALL_NORM);
1434 BTFIXUPSET_CALL(flush_tlb_page, hypersparc_flush_tlb_page, BTFIXUPCALL_NORM);
1435
1436 BTFIXUPSET_CALL(__flush_page_to_ram, hypersparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1437 BTFIXUPSET_CALL(flush_sig_insns, hypersparc_flush_sig_insns, BTFIXUPCALL_NORM);
1438 BTFIXUPSET_CALL(flush_page_for_dma, hypersparc_flush_page_for_dma, BTFIXUPCALL_NOP);
1439
1440
1441 poke_srmmu = poke_hypersparc;
1442
1443 hypersparc_setup_blockops();
1444}
1445
409832f5 1446static void __cpuinit poke_cypress(void)
1da177e4
LT		 1447{
1448 unsigned long mreg = srmmu_get_mmureg();
1449 unsigned long faddr, tagval;
1450 volatile unsigned long cypress_sucks;
1451 volatile unsigned long clear;
1452
1453 clear = srmmu_get_faddr();
1454 clear = srmmu_get_fstatus();
1455
1456 if (!(mreg & CYPRESS_CENABLE)) {
1457 for(faddr = 0x0; faddr < 0x10000; faddr += 20) {
1458 __asm__ __volatile__("sta %%g0, [%0 + %1] %2\n\t"
1459 "sta %%g0, [%0] %2\n\t" : :
1460 "r" (faddr), "r" (0x40000),
1461 "i" (ASI_M_DATAC_TAG));
1462 }
1463 } else {
1464 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
1465 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
1466 "=r" (tagval) :
1467 "r" (faddr), "r" (0x40000),
1468 "i" (ASI_M_DATAC_TAG));
1469
1470 /* If modified and valid, kick it. */
1471 if((tagval & 0x60) == 0x60)
1472 cypress_sucks = *(unsigned long *)
1473 (0xf0020000 + faddr);
1474 }
1475 }
1476
1477 /* And one more, for our good neighbor, Mr. Broken Cypress. */
1478 clear = srmmu_get_faddr();
1479 clear = srmmu_get_fstatus();
1480
1481 mreg |= (CYPRESS_CENABLE | CYPRESS_CMODE);
1482 srmmu_set_mmureg(mreg);
1483}
1484
1485static void __init init_cypress_common(void)
1486{
1487 init_vac_layout();
1488
1da177e4
LT		 1489 BTFIXUPSET_CALL(flush_cache_all, cypress_flush_cache_all, BTFIXUPCALL_NORM);
1490 BTFIXUPSET_CALL(flush_cache_mm, cypress_flush_cache_mm, BTFIXUPCALL_NORM);
1491 BTFIXUPSET_CALL(flush_cache_range, cypress_flush_cache_range, BTFIXUPCALL_NORM);
1492 BTFIXUPSET_CALL(flush_cache_page, cypress_flush_cache_page, BTFIXUPCALL_NORM);
1493
1494 BTFIXUPSET_CALL(flush_tlb_all, cypress_flush_tlb_all, BTFIXUPCALL_NORM);
1495 BTFIXUPSET_CALL(flush_tlb_mm, cypress_flush_tlb_mm, BTFIXUPCALL_NORM);
1496 BTFIXUPSET_CALL(flush_tlb_page, cypress_flush_tlb_page, BTFIXUPCALL_NORM);
1497 BTFIXUPSET_CALL(flush_tlb_range, cypress_flush_tlb_range, BTFIXUPCALL_NORM);
1498
1499
1500 BTFIXUPSET_CALL(__flush_page_to_ram, cypress_flush_page_to_ram, BTFIXUPCALL_NORM);
1501 BTFIXUPSET_CALL(flush_sig_insns, cypress_flush_sig_insns, BTFIXUPCALL_NOP);
1502 BTFIXUPSET_CALL(flush_page_for_dma, cypress_flush_page_for_dma, BTFIXUPCALL_NOP);
1503
1504 poke_srmmu = poke_cypress;
1505}
1506
1507static void __init init_cypress_604(void)
1508{
1509 srmmu_name = "ROSS Cypress-604(UP)";
1510 srmmu_modtype = Cypress;
1511 init_cypress_common();
1512}
1513
1514static void __init init_cypress_605(unsigned long mrev)
1515{
1516 srmmu_name = "ROSS Cypress-605(MP)";
1517 if(mrev == 0xe) {
1518 srmmu_modtype = Cypress_vE;
1519 hwbug_bitmask |= HWBUG_COPYBACK_BROKEN;
1520 } else {
1521 if(mrev == 0xd) {
1522 srmmu_modtype = Cypress_vD;
1523 hwbug_bitmask |= HWBUG_ASIFLUSH_BROKEN;
1524 } else {
1525 srmmu_modtype = Cypress;
1526 }
1527 }
1528 init_cypress_common();
1529}
1530
409832f5 1531static void __cpuinit poke_swift(void)
1da177e4
LT		 1532{
1533 unsigned long mreg;
1534
1535 /* Clear any crap from the cache or else... */
1536 swift_flush_cache_all();
1537
1538 /* Enable I & D caches */
1539 mreg = srmmu_get_mmureg();
1540 mreg |= (SWIFT_IE | SWIFT_DE);
1541 /*
1542 * The Swift branch folding logic is completely broken. At
1543 * trap time, if things are just right, if can mistakenly
1544 * think that a trap is coming from kernel mode when in fact
1545 * it is coming from user mode (it mis-executes the branch in
1546 * the trap code). So you see things like crashme completely
1547 * hosing your machine which is completely unacceptable. Turn
1548 * this shit off... nice job Fujitsu.
1549 */
1550 mreg &= ~(SWIFT_BF);
1551 srmmu_set_mmureg(mreg);
1552}
1553
1554#define SWIFT_MASKID_ADDR 0x10003018
1555static void __init init_swift(void)
1556{
1557 unsigned long swift_rev;
1558
1559 __asm__ __volatile__("lda [%1] %2, %0\n\t"
1560 "srl %0, 0x18, %0\n\t" :
1561 "=r" (swift_rev) :
1562 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1563 srmmu_name = "Fujitsu Swift";
1564 switch(swift_rev) {
1565 case 0x11:
1566 case 0x20:
1567 case 0x23:
1568 case 0x30:
1569 srmmu_modtype = Swift_lots_o_bugs;
1570 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1571 /*
1572 * Gee george, I wonder why Sun is so hush hush about
1573 * this hardware bug... really braindamage stuff going
1574 * on here. However I think we can find a way to avoid
1575 * all of the workaround overhead under Linux. Basically,
1576 * any page fault can cause kernel pages to become user
1577 * accessible (the mmu gets confused and clears some of
1578 * the ACC bits in kernel ptes). Aha, sounds pretty
1579 * horrible eh? But wait, after extensive testing it appears
1580 * that if you use pgd_t level large kernel pte's (like the
1581 * 4MB pages on the Pentium) the bug does not get tripped
1582 * at all. This avoids almost all of the major overhead.
1583 * Welcome to a world where your vendor tells you to,
1584 * "apply this kernel patch" instead of "sorry for the
1585 * broken hardware, send it back and we'll give you
1586 * properly functioning parts"
1587 */
1588 break;
1589 case 0x25:
1590 case 0x31:
1591 srmmu_modtype = Swift_bad_c;
1592 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1593 /*
1594 * You see Sun allude to this hardware bug but never
1595 * admit things directly, they'll say things like,
1596 * "the Swift chip cache problems" or similar.
1597 */
1598 break;
1599 default:
1600 srmmu_modtype = Swift_ok;
1601 break;
6cb79b3f 1602 }
1da177e4
LT		 1603
1604 BTFIXUPSET_CALL(flush_cache_all, swift_flush_cache_all, BTFIXUPCALL_NORM);
1605 BTFIXUPSET_CALL(flush_cache_mm, swift_flush_cache_mm, BTFIXUPCALL_NORM);
1606 BTFIXUPSET_CALL(flush_cache_page, swift_flush_cache_page, BTFIXUPCALL_NORM);
1607 BTFIXUPSET_CALL(flush_cache_range, swift_flush_cache_range, BTFIXUPCALL_NORM);
1608
1609
1610 BTFIXUPSET_CALL(flush_tlb_all, swift_flush_tlb_all, BTFIXUPCALL_NORM);
1611 BTFIXUPSET_CALL(flush_tlb_mm, swift_flush_tlb_mm, BTFIXUPCALL_NORM);
1612 BTFIXUPSET_CALL(flush_tlb_page, swift_flush_tlb_page, BTFIXUPCALL_NORM);
1613 BTFIXUPSET_CALL(flush_tlb_range, swift_flush_tlb_range, BTFIXUPCALL_NORM);
1614
1615 BTFIXUPSET_CALL(__flush_page_to_ram, swift_flush_page_to_ram, BTFIXUPCALL_NORM);
1616 BTFIXUPSET_CALL(flush_sig_insns, swift_flush_sig_insns, BTFIXUPCALL_NORM);
1617 BTFIXUPSET_CALL(flush_page_for_dma, swift_flush_page_for_dma, BTFIXUPCALL_NORM);
1618
1619 BTFIXUPSET_CALL(update_mmu_cache, swift_update_mmu_cache, BTFIXUPCALL_NORM);
1620
1621 flush_page_for_dma_global = 0;
1622
1623 /*
1624 * Are you now convinced that the Swift is one of the
1625 * biggest VLSI abortions of all time? Bravo Fujitsu!
1626 * Fujitsu, the !#?!%$'d up processor people. I bet if
1627 * you examined the microcode of the Swift you'd find
1628 * XXX's all over the place.
1629 */
1630 poke_srmmu = poke_swift;
1631}
1632
1633static void turbosparc_flush_cache_all(void)
1634{
1635 flush_user_windows();
1636 turbosparc_idflash_clear();
1637}
1638
1639static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1640{
1641 FLUSH_BEGIN(mm)
1642 flush_user_windows();
1643 turbosparc_idflash_clear();
1644 FLUSH_END
1645}
1646
1647static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1648{
1649 FLUSH_BEGIN(vma->vm_mm)
1650 flush_user_windows();
1651 turbosparc_idflash_clear();
1652 FLUSH_END
1653}
1654
1655static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1656{
1657 FLUSH_BEGIN(vma->vm_mm)
1658 flush_user_windows();
1659 if (vma->vm_flags & VM_EXEC)
1660 turbosparc_flush_icache();
1661 turbosparc_flush_dcache();
1662 FLUSH_END
1663}
1664
1665/* TurboSparc is copy-back, if we turn it on, but this does not work. */
1666static void turbosparc_flush_page_to_ram(unsigned long page)
1667{
1668#ifdef TURBOSPARC_WRITEBACK
1669 volatile unsigned long clear;
1670
1671 if (srmmu_hwprobe(page))
1672 turbosparc_flush_page_cache(page);
1673 clear = srmmu_get_fstatus();
1674#endif
1675}
1676
1677static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1678{
1679}
1680
1681static void turbosparc_flush_page_for_dma(unsigned long page)
1682{
1683 turbosparc_flush_dcache();
1684}
1685
1686static void turbosparc_flush_tlb_all(void)
1687{
1688 srmmu_flush_whole_tlb();
1689}
1690
1691static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1692{
1693 FLUSH_BEGIN(mm)
1694 srmmu_flush_whole_tlb();
1695 FLUSH_END
1696}
1697
1698static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
1699{
1700 FLUSH_BEGIN(vma->vm_mm)
1701 srmmu_flush_whole_tlb();
1702 FLUSH_END
1703}
1704
1705static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1706{
1707 FLUSH_BEGIN(vma->vm_mm)
1708 srmmu_flush_whole_tlb();
1709 FLUSH_END
1710}
1711
1712
409832f5 1713static void __cpuinit poke_turbosparc(void)
1da177e4
LT		 1714{
1715 unsigned long mreg = srmmu_get_mmureg();
1716 unsigned long ccreg;
1717
1718 /* Clear any crap from the cache or else... */
1719 turbosparc_flush_cache_all();
1720 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */
1721 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
1722 srmmu_set_mmureg(mreg);
1723
1724 ccreg = turbosparc_get_ccreg();
1725
1726#ifdef TURBOSPARC_WRITEBACK
1727 ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */
1728 ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1729 /* Write-back D-cache, emulate VLSI
1730 * abortion number three, not number one */
1731#else
1732 /* For now let's play safe, optimize later */
1733 ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1734 /* Do DVMA snooping in Dcache, Write-thru D-cache */
1735 ccreg &= ~(TURBOSPARC_uS2);
1736 /* Emulate VLSI abortion number three, not number one */
1737#endif
1738
1739 switch (ccreg & 7) {
1740 case 0: /* No SE cache */
1741 case 7: /* Test mode */
1742 break;
1743 default:
1744 ccreg |= (TURBOSPARC_SCENABLE);
1745 }
1746 turbosparc_set_ccreg (ccreg);
1747
1748 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1749 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
1750 srmmu_set_mmureg(mreg);
1751}
1752
1753static void __init init_turbosparc(void)
1754{
1755 srmmu_name = "Fujitsu TurboSparc";
1756 srmmu_modtype = TurboSparc;
1757
1758 BTFIXUPSET_CALL(flush_cache_all, turbosparc_flush_cache_all, BTFIXUPCALL_NORM);
1759 BTFIXUPSET_CALL(flush_cache_mm, turbosparc_flush_cache_mm, BTFIXUPCALL_NORM);
1760 BTFIXUPSET_CALL(flush_cache_page, turbosparc_flush_cache_page, BTFIXUPCALL_NORM);
1761 BTFIXUPSET_CALL(flush_cache_range, turbosparc_flush_cache_range, BTFIXUPCALL_NORM);
1762
1763 BTFIXUPSET_CALL(flush_tlb_all, turbosparc_flush_tlb_all, BTFIXUPCALL_NORM);
1764 BTFIXUPSET_CALL(flush_tlb_mm, turbosparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1765 BTFIXUPSET_CALL(flush_tlb_page, turbosparc_flush_tlb_page, BTFIXUPCALL_NORM);
1766 BTFIXUPSET_CALL(flush_tlb_range, turbosparc_flush_tlb_range, BTFIXUPCALL_NORM);
1767
1768 BTFIXUPSET_CALL(__flush_page_to_ram, turbosparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1769
1770 BTFIXUPSET_CALL(flush_sig_insns, turbosparc_flush_sig_insns, BTFIXUPCALL_NOP);
1771 BTFIXUPSET_CALL(flush_page_for_dma, turbosparc_flush_page_for_dma, BTFIXUPCALL_NORM);
1772
1773 poke_srmmu = poke_turbosparc;
1774}
1775
409832f5 1776static void __cpuinit poke_tsunami(void)
1da177e4
LT		 1777{
1778 unsigned long mreg = srmmu_get_mmureg();
1779
1780 tsunami_flush_icache();
1781 tsunami_flush_dcache();
1782 mreg &= ~TSUNAMI_ITD;
1783 mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1784 srmmu_set_mmureg(mreg);
1785}
1786
1787static void __init init_tsunami(void)
1788{
1789 /*
1790 * Tsunami's pretty sane, Sun and TI actually got it
1791 * somewhat right this time. Fujitsu should have
1792 * taken some lessons from them.
1793 */
1794
1795 srmmu_name = "TI Tsunami";
1796 srmmu_modtype = Tsunami;
1797
1798 BTFIXUPSET_CALL(flush_cache_all, tsunami_flush_cache_all, BTFIXUPCALL_NORM);
1799 BTFIXUPSET_CALL(flush_cache_mm, tsunami_flush_cache_mm, BTFIXUPCALL_NORM);
1800 BTFIXUPSET_CALL(flush_cache_page, tsunami_flush_cache_page, BTFIXUPCALL_NORM);
1801 BTFIXUPSET_CALL(flush_cache_range, tsunami_flush_cache_range, BTFIXUPCALL_NORM);
1802
1803
1804 BTFIXUPSET_CALL(flush_tlb_all, tsunami_flush_tlb_all, BTFIXUPCALL_NORM);
1805 BTFIXUPSET_CALL(flush_tlb_mm, tsunami_flush_tlb_mm, BTFIXUPCALL_NORM);
1806 BTFIXUPSET_CALL(flush_tlb_page, tsunami_flush_tlb_page, BTFIXUPCALL_NORM);
1807 BTFIXUPSET_CALL(flush_tlb_range, tsunami_flush_tlb_range, BTFIXUPCALL_NORM);
1808
1809 BTFIXUPSET_CALL(__flush_page_to_ram, tsunami_flush_page_to_ram, BTFIXUPCALL_NOP);
1810 BTFIXUPSET_CALL(flush_sig_insns, tsunami_flush_sig_insns, BTFIXUPCALL_NORM);
1811 BTFIXUPSET_CALL(flush_page_for_dma, tsunami_flush_page_for_dma, BTFIXUPCALL_NORM);
1812
1813 poke_srmmu = poke_tsunami;
1814
1815 tsunami_setup_blockops();
1816}
1817
409832f5 1818static void __cpuinit poke_viking(void)
1da177e4
LT		 1819{
1820 unsigned long mreg = srmmu_get_mmureg();
1821 static int smp_catch;
1822
1823 if(viking_mxcc_present) {
1824 unsigned long mxcc_control = mxcc_get_creg();
1825
1826 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1827 mxcc_control &= ~(MXCC_CTL_RRC);
1828 mxcc_set_creg(mxcc_control);
1829
1830 /*
1831 * We don't need memory parity checks.
1832 * XXX This is a mess, have to dig out later. ecd.
1833 viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1834 */
1835
1836 /* We do cache ptables on MXCC. */
1837 mreg |= VIKING_TCENABLE;
1838 } else {
1839 unsigned long bpreg;
1840
1841 mreg &= ~(VIKING_TCENABLE);
1842 if(smp_catch++) {
1843 /* Must disable mixed-cmd mode here for other cpu's. */
1844 bpreg = viking_get_bpreg();
1845 bpreg &= ~(VIKING_ACTION_MIX);
1846 viking_set_bpreg(bpreg);
1847
1848 /* Just in case PROM does something funny. */
1849 msi_set_sync();
1850 }
1851 }
1852
1853 mreg |= VIKING_SPENABLE;
1854 mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1855 mreg |= VIKING_SBENABLE;
1856 mreg &= ~(VIKING_ACENABLE);
1857 srmmu_set_mmureg(mreg);
1da177e4
LT		 1858}
1859
1860static void __init init_viking(void)
1861{
1862 unsigned long mreg = srmmu_get_mmureg();
1863
1864 /* Ahhh, the viking. SRMMU VLSI abortion number two... */
1865 if(mreg & VIKING_MMODE) {
1866 srmmu_name = "TI Viking";
1867 viking_mxcc_present = 0;
1868 msi_set_sync();
1869
1da177e4
LT		 1870 /*
1871 * We need this to make sure old viking takes no hits
1872 * on it's cache for dma snoops to workaround the
1873 * "load from non-cacheable memory" interrupt bug.
1874 * This is only necessary because of the new way in
1875 * which we use the IOMMU.
1876 */
1877 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page, BTFIXUPCALL_NORM);
1878
1879 flush_page_for_dma_global = 0;
1880 } else {
1881 srmmu_name = "TI Viking/MXCC";
1882 viking_mxcc_present = 1;
1883
1884 srmmu_cache_pagetables = 1;
1885
1886 /* MXCC vikings lack the DMA snooping bug. */
1887 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page_for_dma, BTFIXUPCALL_NOP);
1888 }
1889
1890 BTFIXUPSET_CALL(flush_cache_all, viking_flush_cache_all, BTFIXUPCALL_NORM);
1891 BTFIXUPSET_CALL(flush_cache_mm, viking_flush_cache_mm, BTFIXUPCALL_NORM);
1892 BTFIXUPSET_CALL(flush_cache_page, viking_flush_cache_page, BTFIXUPCALL_NORM);
1893 BTFIXUPSET_CALL(flush_cache_range, viking_flush_cache_range, BTFIXUPCALL_NORM);
1894
1895#ifdef CONFIG_SMP
1896 if (sparc_cpu_model == sun4d) {
1897 BTFIXUPSET_CALL(flush_tlb_all, sun4dsmp_flush_tlb_all, BTFIXUPCALL_NORM);
1898 BTFIXUPSET_CALL(flush_tlb_mm, sun4dsmp_flush_tlb_mm, BTFIXUPCALL_NORM);
1899 BTFIXUPSET_CALL(flush_tlb_page, sun4dsmp_flush_tlb_page, BTFIXUPCALL_NORM);
1900 BTFIXUPSET_CALL(flush_tlb_range, sun4dsmp_flush_tlb_range, BTFIXUPCALL_NORM);
1901 } else
1902#endif
1903 {
1904 BTFIXUPSET_CALL(flush_tlb_all, viking_flush_tlb_all, BTFIXUPCALL_NORM);
1905 BTFIXUPSET_CALL(flush_tlb_mm, viking_flush_tlb_mm, BTFIXUPCALL_NORM);
1906 BTFIXUPSET_CALL(flush_tlb_page, viking_flush_tlb_page, BTFIXUPCALL_NORM);
1907 BTFIXUPSET_CALL(flush_tlb_range, viking_flush_tlb_range, BTFIXUPCALL_NORM);
1908 }
1909
1910 BTFIXUPSET_CALL(__flush_page_to_ram, viking_flush_page_to_ram, BTFIXUPCALL_NOP);
1911 BTFIXUPSET_CALL(flush_sig_insns, viking_flush_sig_insns, BTFIXUPCALL_NOP);
1912
1913 poke_srmmu = poke_viking;
1914}
1915
75d9e346
KE		 1916#ifdef CONFIG_SPARC_LEON
1917
1918void __init poke_leonsparc(void)
1919{
1920}
1921
1922void __init init_leon(void)
1923{
1924
c803ba90 1925 srmmu_name = "LEON";
75d9e346
KE		 1926
1927 BTFIXUPSET_CALL(flush_cache_all, leon_flush_cache_all,
1928 BTFIXUPCALL_NORM);
1929 BTFIXUPSET_CALL(flush_cache_mm, leon_flush_cache_all,
1930 BTFIXUPCALL_NORM);
1931 BTFIXUPSET_CALL(flush_cache_page, leon_flush_pcache_all,
1932 BTFIXUPCALL_NORM);
1933 BTFIXUPSET_CALL(flush_cache_range, leon_flush_cache_all,
1934 BTFIXUPCALL_NORM);
1935 BTFIXUPSET_CALL(flush_page_for_dma, leon_flush_dcache_all,
1936 BTFIXUPCALL_NORM);
1937
1938 BTFIXUPSET_CALL(flush_tlb_all, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1939 BTFIXUPSET_CALL(flush_tlb_mm, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1940 BTFIXUPSET_CALL(flush_tlb_page, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1941 BTFIXUPSET_CALL(flush_tlb_range, leon_flush_tlb_all, BTFIXUPCALL_NORM);
1942
1943 BTFIXUPSET_CALL(__flush_page_to_ram, leon_flush_cache_all,
1944 BTFIXUPCALL_NOP);
1945 BTFIXUPSET_CALL(flush_sig_insns, leon_flush_cache_all, BTFIXUPCALL_NOP);
1946
1947 poke_srmmu = poke_leonsparc;
1948
1949 srmmu_cache_pagetables = 0;
1950
1951 leon_flush_during_switch = leon_flush_needed();
1952}
1953#endif
1954
1da177e4
LT		 1955/* Probe for the srmmu chip version. */
1956static void __init get_srmmu_type(void)
1957{
1958 unsigned long mreg, psr;
1959 unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1960
1961 srmmu_modtype = SRMMU_INVAL_MOD;
1962 hwbug_bitmask = 0;
1963
1964 mreg = srmmu_get_mmureg(); psr = get_psr();
1965 mod_typ = (mreg & 0xf0000000) >> 28;
1966 mod_rev = (mreg & 0x0f000000) >> 24;
1967 psr_typ = (psr >> 28) & 0xf;
1968 psr_vers = (psr >> 24) & 0xf;
1969
75d9e346
KE		 1970 /* First, check for sparc-leon. */
1971 if (sparc_cpu_model == sparc_leon) {
75d9e346
KE		 1972 init_leon();
1973 return;
1974 }
1975
1976 /* Second, check for HyperSparc or Cypress. */
1da177e4
LT		 1977 if(mod_typ == 1) {
1978 switch(mod_rev) {
1979 case 7:
1980 /* UP or MP Hypersparc */
1981 init_hypersparc();
1982 break;
1983 case 0:
1984 case 2:
1985 /* Uniprocessor Cypress */
1986 init_cypress_604();
1987 break;
1988 case 10:
1989 case 11:
1990 case 12:
1991 /* _REALLY OLD_ Cypress MP chips... */
1992 case 13:
1993 case 14:
1994 case 15:
1995 /* MP Cypress mmu/cache-controller */
1996 init_cypress_605(mod_rev);
1997 break;
1998 default:
1999 /* Some other Cypress revision, assume a 605. */
2000 init_cypress_605(mod_rev);
2001 break;
6cb79b3f 2002 }
1da177e4
LT		 2003 return;
2004 }
2005
2006 /*
2007 * Now Fujitsu TurboSparc. It might happen that it is
2008 * in Swift emulation mode, so we will check later...
2009 */
2010 if (psr_typ == 0 && psr_vers == 5) {
2011 init_turbosparc();
2012 return;
2013 }
2014
2015 /* Next check for Fujitsu Swift. */
2016 if(psr_typ == 0 && psr_vers == 4) {
8d125562 2017 phandle cpunode;
1da177e4
LT		 2018 char node_str[128];
2019
2020 /* Look if it is not a TurboSparc emulating Swift... */
2021 cpunode = prom_getchild(prom_root_node);
2022 while((cpunode = prom_getsibling(cpunode)) != 0) {
2023 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
2024 if(!strcmp(node_str, "cpu")) {
2025 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
2026 prom_getintdefault(cpunode, "psr-version", 1) == 5) {
2027 init_turbosparc();
2028 return;
2029 }
2030 break;
2031 }
2032 }
2033
2034 init_swift();
2035 return;
2036 }
2037
2038 /* Now the Viking family of srmmu. */
2039 if(psr_typ == 4 &&
2040 ((psr_vers == 0) ||
2041 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
2042 init_viking();
2043 return;
2044 }
2045
2046 /* Finally the Tsunami. */
2047 if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
2048 init_tsunami();
2049 return;
2050 }
2051
2052 /* Oh well */
2053 srmmu_is_bad();
2054}
2055
1da177e4
LT		 2056extern unsigned long spwin_mmu_patchme, fwin_mmu_patchme,
2057 tsetup_mmu_patchme, rtrap_mmu_patchme;
2058
2059extern unsigned long spwin_srmmu_stackchk, srmmu_fwin_stackchk,
2060 tsetup_srmmu_stackchk, srmmu_rett_stackchk;
2061
1da177e4
LT		 2062#ifdef CONFIG_SMP
2063/* Local cross-calls. */
2064static void smp_flush_page_for_dma(unsigned long page)
2065{
2066 xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_for_dma), page);
2067 local_flush_page_for_dma(page);
2068}
2069
2070#endif
2071
1da177e4
LT		 2072/* Load up routines and constants for sun4m and sun4d mmu */
2073void __init ld_mmu_srmmu(void)
2074{
2075 extern void ld_mmu_iommu(void);
2076 extern void ld_mmu_iounit(void);
2077 extern void ___xchg32_sun4md(void);
2078
1da177e4
LT		 2079 /* Functions */
2080#ifndef CONFIG_SMP
2081 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4md, BTFIXUPCALL_SWAPG1G2);
2082#endif
1da177e4
LT		 2083
2084 BTFIXUPSET_CALL(set_pte, srmmu_set_pte, BTFIXUPCALL_SWAPO0O1);
1da177e4 2085
46a82b2d 2086 BTFIXUPSET_CALL(pgd_page_vaddr, srmmu_pgd_page, BTFIXUPCALL_NORM);
1da177e4 2087
1da177e4 2088 BTFIXUPSET_CALL(pte_present, srmmu_pte_present, BTFIXUPCALL_NORM);
1da177e4
LT		 2089
2090 BTFIXUPSET_CALL(pmd_bad, srmmu_pmd_bad, BTFIXUPCALL_NORM);
2091 BTFIXUPSET_CALL(pmd_present, srmmu_pmd_present, BTFIXUPCALL_NORM);
1da177e4
LT		 2092
2093 BTFIXUPSET_CALL(pgd_none, srmmu_pgd_none, BTFIXUPCALL_NORM);
2094 BTFIXUPSET_CALL(pgd_bad, srmmu_pgd_bad, BTFIXUPCALL_NORM);
2095 BTFIXUPSET_CALL(pgd_present, srmmu_pgd_present, BTFIXUPCALL_NORM);
1da177e4
LT		 2096
2097 BTFIXUPSET_CALL(mk_pte, srmmu_mk_pte, BTFIXUPCALL_NORM);
2098 BTFIXUPSET_CALL(mk_pte_phys, srmmu_mk_pte_phys, BTFIXUPCALL_NORM);
2099 BTFIXUPSET_CALL(mk_pte_io, srmmu_mk_pte_io, BTFIXUPCALL_NORM);
2100 BTFIXUPSET_CALL(pgd_set, srmmu_pgd_set, BTFIXUPCALL_NORM);
2101 BTFIXUPSET_CALL(pmd_set, srmmu_pmd_set, BTFIXUPCALL_NORM);
2102 BTFIXUPSET_CALL(pmd_populate, srmmu_pmd_populate, BTFIXUPCALL_NORM);
2103
2104 BTFIXUPSET_INT(pte_modify_mask, SRMMU_CHG_MASK);
2105 BTFIXUPSET_CALL(pmd_offset, srmmu_pmd_offset, BTFIXUPCALL_NORM);
2106 BTFIXUPSET_CALL(pte_offset_kernel, srmmu_pte_offset, BTFIXUPCALL_NORM);
2107
2108 BTFIXUPSET_CALL(free_pte_fast, srmmu_free_pte_fast, BTFIXUPCALL_NORM);
2109 BTFIXUPSET_CALL(pte_free, srmmu_pte_free, BTFIXUPCALL_NORM);
2110 BTFIXUPSET_CALL(pte_alloc_one_kernel, srmmu_pte_alloc_one_kernel, BTFIXUPCALL_NORM);
2111 BTFIXUPSET_CALL(pte_alloc_one, srmmu_pte_alloc_one, BTFIXUPCALL_NORM);
2112 BTFIXUPSET_CALL(free_pmd_fast, srmmu_pmd_free, BTFIXUPCALL_NORM);
2113 BTFIXUPSET_CALL(pmd_alloc_one, srmmu_pmd_alloc_one, BTFIXUPCALL_NORM);
2114 BTFIXUPSET_CALL(free_pgd_fast, srmmu_free_pgd_fast, BTFIXUPCALL_NORM);
2115 BTFIXUPSET_CALL(get_pgd_fast, srmmu_get_pgd_fast, BTFIXUPCALL_NORM);
2116
2117 BTFIXUPSET_HALF(pte_writei, SRMMU_WRITE);
2118 BTFIXUPSET_HALF(pte_dirtyi, SRMMU_DIRTY);
2119 BTFIXUPSET_HALF(pte_youngi, SRMMU_REF);
2120 BTFIXUPSET_HALF(pte_filei, SRMMU_FILE);
2121 BTFIXUPSET_HALF(pte_wrprotecti, SRMMU_WRITE);
2122 BTFIXUPSET_HALF(pte_mkcleani, SRMMU_DIRTY);
2123 BTFIXUPSET_HALF(pte_mkoldi, SRMMU_REF);
2124 BTFIXUPSET_CALL(pte_mkwrite, srmmu_pte_mkwrite, BTFIXUPCALL_ORINT(SRMMU_WRITE));
2125 BTFIXUPSET_CALL(pte_mkdirty, srmmu_pte_mkdirty, BTFIXUPCALL_ORINT(SRMMU_DIRTY));
2126 BTFIXUPSET_CALL(pte_mkyoung, srmmu_pte_mkyoung, BTFIXUPCALL_ORINT(SRMMU_REF));
2127 BTFIXUPSET_CALL(update_mmu_cache, srmmu_update_mmu_cache, BTFIXUPCALL_NOP);
2128 BTFIXUPSET_CALL(destroy_context, srmmu_destroy_context, BTFIXUPCALL_NORM);
2129
2130 BTFIXUPSET_CALL(sparc_mapiorange, srmmu_mapiorange, BTFIXUPCALL_NORM);
2131 BTFIXUPSET_CALL(sparc_unmapiorange, srmmu_unmapiorange, BTFIXUPCALL_NORM);
2132
2133 BTFIXUPSET_CALL(__swp_type, srmmu_swp_type, BTFIXUPCALL_NORM);
2134 BTFIXUPSET_CALL(__swp_offset, srmmu_swp_offset, BTFIXUPCALL_NORM);
2135 BTFIXUPSET_CALL(__swp_entry, srmmu_swp_entry, BTFIXUPCALL_NORM);
2136
2137 BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM);
2138
1da177e4 2139 get_srmmu_type();
1da177e4
LT		 2140
2141#ifdef CONFIG_SMP
2142 /* El switcheroo... */
2143
2144 BTFIXUPCOPY_CALL(local_flush_cache_all, flush_cache_all);
2145 BTFIXUPCOPY_CALL(local_flush_cache_mm, flush_cache_mm);
2146 BTFIXUPCOPY_CALL(local_flush_cache_range, flush_cache_range);
2147 BTFIXUPCOPY_CALL(local_flush_cache_page, flush_cache_page);
2148 BTFIXUPCOPY_CALL(local_flush_tlb_all, flush_tlb_all);
2149 BTFIXUPCOPY_CALL(local_flush_tlb_mm, flush_tlb_mm);
2150 BTFIXUPCOPY_CALL(local_flush_tlb_range, flush_tlb_range);
2151 BTFIXUPCOPY_CALL(local_flush_tlb_page, flush_tlb_page);
2152 BTFIXUPCOPY_CALL(local_flush_page_to_ram, __flush_page_to_ram);
2153 BTFIXUPCOPY_CALL(local_flush_sig_insns, flush_sig_insns);
2154 BTFIXUPCOPY_CALL(local_flush_page_for_dma, flush_page_for_dma);
2155
2156 BTFIXUPSET_CALL(flush_cache_all, smp_flush_cache_all, BTFIXUPCALL_NORM);
2157 BTFIXUPSET_CALL(flush_cache_mm, smp_flush_cache_mm, BTFIXUPCALL_NORM);
2158 BTFIXUPSET_CALL(flush_cache_range, smp_flush_cache_range, BTFIXUPCALL_NORM);
2159 BTFIXUPSET_CALL(flush_cache_page, smp_flush_cache_page, BTFIXUPCALL_NORM);
8401707f
KE		 2160 if (sparc_cpu_model != sun4d &&
2161 sparc_cpu_model != sparc_leon) {
1da177e4
LT		 2162 BTFIXUPSET_CALL(flush_tlb_all, smp_flush_tlb_all, BTFIXUPCALL_NORM);
2163 BTFIXUPSET_CALL(flush_tlb_mm, smp_flush_tlb_mm, BTFIXUPCALL_NORM);
2164 BTFIXUPSET_CALL(flush_tlb_range, smp_flush_tlb_range, BTFIXUPCALL_NORM);
2165 BTFIXUPSET_CALL(flush_tlb_page, smp_flush_tlb_page, BTFIXUPCALL_NORM);
2166 }
2167 BTFIXUPSET_CALL(__flush_page_to_ram, smp_flush_page_to_ram, BTFIXUPCALL_NORM);
2168 BTFIXUPSET_CALL(flush_sig_insns, smp_flush_sig_insns, BTFIXUPCALL_NORM);
2169 BTFIXUPSET_CALL(flush_page_for_dma, smp_flush_page_for_dma, BTFIXUPCALL_NORM);
64273d08
DM		 2170
2171 if (poke_srmmu == poke_viking) {
2172 /* Avoid unnecessary cross calls. */
2173 BTFIXUPCOPY_CALL(flush_cache_all, local_flush_cache_all);
2174 BTFIXUPCOPY_CALL(flush_cache_mm, local_flush_cache_mm);
2175 BTFIXUPCOPY_CALL(flush_cache_range, local_flush_cache_range);
2176 BTFIXUPCOPY_CALL(flush_cache_page, local_flush_cache_page);
2177 BTFIXUPCOPY_CALL(__flush_page_to_ram, local_flush_page_to_ram);
2178 BTFIXUPCOPY_CALL(flush_sig_insns, local_flush_sig_insns);
2179 BTFIXUPCOPY_CALL(flush_page_for_dma, local_flush_page_for_dma);
2180 }
1da177e4
LT		 2181#endif
2182
2183 if (sparc_cpu_model == sun4d)
2184 ld_mmu_iounit();
2185 else
2186 ld_mmu_iommu();
2187#ifdef CONFIG_SMP
2188 if (sparc_cpu_model == sun4d)
2189 sun4d_init_smp();
8401707f
KE		 2190 else if (sparc_cpu_model == sparc_leon)
2191 leon_init_smp();
1da177e4
LT		 2192 else
2193 sun4m_init_smp();
2194#endif
2195}
Linux 6.x block layer and io_uring tree(s)