Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* arch/sparc64/mm/tlb.c |
2 | * | |
3 | * Copyright (C) 2004 David S. Miller <davem@redhat.com> | |
4 | */ | |
5 | ||
6 | #include <linux/kernel.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/percpu.h> | |
9 | #include <linux/mm.h> | |
10 | #include <linux/swap.h> | |
c9f2946f | 11 | #include <linux/preempt.h> |
1da177e4 LT |
12 | |
13 | #include <asm/pgtable.h> | |
14 | #include <asm/pgalloc.h> | |
15 | #include <asm/tlbflush.h> | |
16 | #include <asm/cacheflush.h> | |
17 | #include <asm/mmu_context.h> | |
18 | #include <asm/tlb.h> | |
19 | ||
20 | /* Heavily inspired by the ppc64 code. */ | |
21 | ||
90f08e39 | 22 | static DEFINE_PER_CPU(struct tlb_batch, tlb_batch); |
1da177e4 LT |
23 | |
24 | void flush_tlb_pending(void) | |
25 | { | |
90f08e39 | 26 | struct tlb_batch *tb = &get_cpu_var(tlb_batch); |
1da177e4 | 27 | |
90f08e39 PZ |
28 | if (tb->tlb_nr) { |
29 | flush_tsb_user(tb); | |
74bf4312 | 30 | |
90f08e39 | 31 | if (CTX_VALID(tb->mm->context)) { |
1da177e4 | 32 | #ifdef CONFIG_SMP |
90f08e39 PZ |
33 | smp_flush_tlb_pending(tb->mm, tb->tlb_nr, |
34 | &tb->vaddrs[0]); | |
1da177e4 | 35 | #else |
90f08e39 PZ |
36 | __flush_tlb_pending(CTX_HWBITS(tb->mm->context), |
37 | tb->tlb_nr, &tb->vaddrs[0]); | |
1da177e4 LT |
38 | #endif |
39 | } | |
90f08e39 | 40 | tb->tlb_nr = 0; |
1da177e4 | 41 | } |
c9f2946f | 42 | |
90f08e39 | 43 | put_cpu_var(tlb_batch); |
1da177e4 LT |
44 | } |
45 | ||
9e695d2e DM |
46 | static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, |
47 | bool exec) | |
1da177e4 | 48 | { |
90f08e39 | 49 | struct tlb_batch *tb = &get_cpu_var(tlb_batch); |
1da177e4 LT |
50 | unsigned long nr; |
51 | ||
52 | vaddr &= PAGE_MASK; | |
9e695d2e | 53 | if (exec) |
1da177e4 LT |
54 | vaddr |= 0x1UL; |
55 | ||
9e695d2e DM |
56 | nr = tb->tlb_nr; |
57 | ||
58 | if (unlikely(nr != 0 && mm != tb->mm)) { | |
59 | flush_tlb_pending(); | |
60 | nr = 0; | |
61 | } | |
62 | ||
63 | if (nr == 0) | |
64 | tb->mm = mm; | |
65 | ||
66 | tb->vaddrs[nr] = vaddr; | |
67 | tb->tlb_nr = ++nr; | |
68 | if (nr >= TLB_BATCH_NR) | |
69 | flush_tlb_pending(); | |
70 | ||
71 | put_cpu_var(tlb_batch); | |
72 | } | |
73 | ||
74 | void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, | |
75 | pte_t *ptep, pte_t orig, int fullmm) | |
76 | { | |
7a591cfe DM |
77 | if (tlb_type != hypervisor && |
78 | pte_dirty(orig)) { | |
1da177e4 LT |
79 | unsigned long paddr, pfn = pte_pfn(orig); |
80 | struct address_space *mapping; | |
81 | struct page *page; | |
82 | ||
83 | if (!pfn_valid(pfn)) | |
84 | goto no_cache_flush; | |
85 | ||
86 | page = pfn_to_page(pfn); | |
87 | if (PageReserved(page)) | |
88 | goto no_cache_flush; | |
89 | ||
90 | /* A real file page? */ | |
91 | mapping = page_mapping(page); | |
92 | if (!mapping) | |
93 | goto no_cache_flush; | |
94 | ||
95 | paddr = (unsigned long) page_address(page); | |
96 | if ((paddr ^ vaddr) & (1 << 13)) | |
97 | flush_dcache_page_all(mm, page); | |
98 | } | |
99 | ||
100 | no_cache_flush: | |
9e695d2e DM |
101 | if (!fullmm) |
102 | tlb_batch_add_one(mm, vaddr, pte_exec(orig)); | |
103 | } | |
104 | ||
105 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
106 | static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, | |
107 | pmd_t pmd, bool exec) | |
108 | { | |
109 | unsigned long end; | |
110 | pte_t *pte; | |
111 | ||
112 | pte = pte_offset_map(&pmd, vaddr); | |
113 | end = vaddr + HPAGE_SIZE; | |
114 | while (vaddr < end) { | |
115 | if (pte_val(*pte) & _PAGE_VALID) | |
116 | tlb_batch_add_one(mm, vaddr, exec); | |
117 | pte++; | |
118 | vaddr += PAGE_SIZE; | |
119 | } | |
120 | pte_unmap(pte); | |
121 | } | |
1da177e4 | 122 | |
9e695d2e DM |
123 | void set_pmd_at(struct mm_struct *mm, unsigned long addr, |
124 | pmd_t *pmdp, pmd_t pmd) | |
125 | { | |
126 | pmd_t orig = *pmdp; | |
127 | ||
128 | *pmdp = pmd; | |
129 | ||
130 | if (mm == &init_mm) | |
1da177e4 | 131 | return; |
9e695d2e DM |
132 | |
133 | if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) { | |
134 | if (pmd_val(pmd) & PMD_ISHUGE) | |
135 | mm->context.huge_pte_count++; | |
136 | else | |
137 | mm->context.huge_pte_count--; | |
138 | if (mm->context.huge_pte_count == 1) | |
139 | hugetlb_setup(mm); | |
90f08e39 | 140 | } |
1da177e4 | 141 | |
9e695d2e DM |
142 | if (!pmd_none(orig)) { |
143 | bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0); | |
1da177e4 | 144 | |
9e695d2e DM |
145 | addr &= HPAGE_MASK; |
146 | if (pmd_val(orig) & PMD_ISHUGE) | |
147 | tlb_batch_add_one(mm, addr, exec); | |
148 | else | |
149 | tlb_batch_pmd_scan(mm, addr, orig, exec); | |
1da177e4 | 150 | } |
9e695d2e | 151 | } |
1da177e4 | 152 | |
9e695d2e DM |
153 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) |
154 | { | |
155 | struct list_head *lh = (struct list_head *) pgtable; | |
1da177e4 | 156 | |
9e695d2e | 157 | assert_spin_locked(&mm->page_table_lock); |
90f08e39 | 158 | |
9e695d2e DM |
159 | /* FIFO */ |
160 | if (!mm->pmd_huge_pte) | |
161 | INIT_LIST_HEAD(lh); | |
162 | else | |
163 | list_add(lh, (struct list_head *) mm->pmd_huge_pte); | |
164 | mm->pmd_huge_pte = pgtable; | |
165 | } | |
166 | ||
167 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm) | |
168 | { | |
169 | struct list_head *lh; | |
170 | pgtable_t pgtable; | |
171 | ||
172 | assert_spin_locked(&mm->page_table_lock); | |
173 | ||
174 | /* FIFO */ | |
175 | pgtable = mm->pmd_huge_pte; | |
176 | lh = (struct list_head *) pgtable; | |
177 | if (list_empty(lh)) | |
178 | mm->pmd_huge_pte = NULL; | |
179 | else { | |
180 | mm->pmd_huge_pte = (pgtable_t) lh->next; | |
181 | list_del(lh); | |
182 | } | |
183 | pte_val(pgtable[0]) = 0; | |
184 | pte_val(pgtable[1]) = 0; | |
185 | ||
186 | return pgtable; | |
1da177e4 | 187 | } |
9e695d2e | 188 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |