Commit | Line | Data |
---|---|---|
6656920b CZ |
1 | /* |
2 | * arch/xtensa/mm/cache.c | |
3 | * | |
4 | * This file is subject to the terms and conditions of the GNU General Public | |
5 | * License. See the file "COPYING" in the main directory of this archive | |
6 | * for more details. | |
7 | * | |
8 | * Copyright (C) 2001-2006 Tensilica Inc. | |
9 | * | |
10 | * Chris Zankel <chris@zankel.net> | |
11 | * Joe Taylor | |
12 | * Marc Gauthier | |
13 | * | |
14 | */ | |
15 | ||
16 | #include <linux/init.h> | |
17 | #include <linux/signal.h> | |
18 | #include <linux/sched.h> | |
19 | #include <linux/kernel.h> | |
20 | #include <linux/errno.h> | |
21 | #include <linux/string.h> | |
22 | #include <linux/types.h> | |
23 | #include <linux/ptrace.h> | |
57c8a661 | 24 | #include <linux/memblock.h> |
6656920b CZ |
25 | #include <linux/swap.h> |
26 | #include <linux/pagemap.h> | |
65fddcfc | 27 | #include <linux/pgtable.h> |
6656920b | 28 | |
6656920b CZ |
29 | #include <asm/bootparam.h> |
30 | #include <asm/mmu_context.h> | |
31 | #include <asm/tlb.h> | |
32 | #include <asm/tlbflush.h> | |
33 | #include <asm/page.h> | |
6656920b | 34 | |
6656920b CZ |
35 | /* |
36 | * Note: | |
37 | * The kernel provides one architecture bit PG_arch_1 in the page flags that | |
38 | * can be used for cache coherency. | |
39 | * | |
40 | * I$-D$ coherency. | |
41 | * | |
42 | * The Xtensa architecture doesn't keep the instruction cache coherent with | |
43 | * the data cache. We use the architecture bit to indicate if the caches | |
44 | * are coherent. The kernel clears this bit whenever a page is added to the | |
45 | * page cache. At that time, the caches might not be in sync. We, therefore, | |
46 | * define this flag as 'clean' if set. | |
47 | * | |
48 | * D-cache aliasing. | |
49 | * | |
50 | * With cache aliasing, we have to always flush the cache when pages are | |
51 | * unmapped (see tlb_start_vma(). So, we use this flag to indicate a dirty | |
52 | * page. | |
53 | * | |
54 | * | |
55 | * | |
56 | */ | |
57 | ||
a91902db MF |
58 | #if (DCACHE_WAY_SIZE > PAGE_SIZE) |
59 | static inline void kmap_invalidate_coherent(struct page *page, | |
60 | unsigned long vaddr) | |
61 | { | |
62 | if (!DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { | |
63 | unsigned long kvaddr; | |
64 | ||
65 | if (!PageHighMem(page)) { | |
66 | kvaddr = (unsigned long)page_to_virt(page); | |
67 | ||
68 | __invalidate_dcache_page(kvaddr); | |
69 | } else { | |
70 | kvaddr = TLBTEMP_BASE_1 + | |
71 | (page_to_phys(page) & DCACHE_ALIAS_MASK); | |
72 | ||
3a860d16 | 73 | preempt_disable(); |
a91902db MF |
74 | __invalidate_dcache_page_alias(kvaddr, |
75 | page_to_phys(page)); | |
3a860d16 | 76 | preempt_enable(); |
a91902db MF |
77 | } |
78 | } | |
79 | } | |
80 | ||
81 | static inline void *coherent_kvaddr(struct page *page, unsigned long base, | |
82 | unsigned long vaddr, unsigned long *paddr) | |
83 | { | |
84 | if (PageHighMem(page) || !DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { | |
85 | *paddr = page_to_phys(page); | |
86 | return (void *)(base + (vaddr & DCACHE_ALIAS_MASK)); | |
87 | } else { | |
88 | *paddr = 0; | |
89 | return page_to_virt(page); | |
90 | } | |
91 | } | |
92 | ||
93 | void clear_user_highpage(struct page *page, unsigned long vaddr) | |
94 | { | |
95 | unsigned long paddr; | |
96 | void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr); | |
97 | ||
a67cc9aa | 98 | preempt_disable(); |
a91902db MF |
99 | kmap_invalidate_coherent(page, vaddr); |
100 | set_bit(PG_arch_1, &page->flags); | |
101 | clear_page_alias(kvaddr, paddr); | |
a67cc9aa | 102 | preempt_enable(); |
a91902db | 103 | } |
bc652eb6 | 104 | EXPORT_SYMBOL(clear_user_highpage); |
a91902db MF |
105 | |
106 | void copy_user_highpage(struct page *dst, struct page *src, | |
107 | unsigned long vaddr, struct vm_area_struct *vma) | |
108 | { | |
109 | unsigned long dst_paddr, src_paddr; | |
110 | void *dst_vaddr = coherent_kvaddr(dst, TLBTEMP_BASE_1, vaddr, | |
111 | &dst_paddr); | |
112 | void *src_vaddr = coherent_kvaddr(src, TLBTEMP_BASE_2, vaddr, | |
113 | &src_paddr); | |
114 | ||
a67cc9aa | 115 | preempt_disable(); |
a91902db MF |
116 | kmap_invalidate_coherent(dst, vaddr); |
117 | set_bit(PG_arch_1, &dst->flags); | |
118 | copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr); | |
a67cc9aa | 119 | preempt_enable(); |
a91902db | 120 | } |
bc652eb6 | 121 | EXPORT_SYMBOL(copy_user_highpage); |
a91902db | 122 | |
6656920b CZ |
123 | /* |
124 | * Any time the kernel writes to a user page cache page, or it is about to | |
125 | * read from a page cache page this routine is called. | |
126 | * | |
127 | */ | |
128 | ||
129 | void flush_dcache_page(struct page *page) | |
130 | { | |
cb9f753a | 131 | struct address_space *mapping = page_mapping_file(page); |
6656920b CZ |
132 | |
133 | /* | |
134 | * If we have a mapping but the page is not mapped to user-space | |
135 | * yet, we simply mark this page dirty and defer flushing the | |
136 | * caches until update_mmu(). | |
137 | */ | |
138 | ||
139 | if (mapping && !mapping_mapped(mapping)) { | |
140 | if (!test_bit(PG_arch_1, &page->flags)) | |
141 | set_bit(PG_arch_1, &page->flags); | |
142 | return; | |
143 | ||
144 | } else { | |
145 | ||
146 | unsigned long phys = page_to_phys(page); | |
147 | unsigned long temp = page->index << PAGE_SHIFT; | |
148 | unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys)); | |
149 | unsigned long virt; | |
150 | ||
151 | /* | |
152 | * Flush the page in kernel space and user space. | |
153 | * Note that we can omit that step if aliasing is not | |
154 | * an issue, but we do have to synchronize I$ and D$ | |
155 | * if we have a mapping. | |
156 | */ | |
157 | ||
158 | if (!alias && !mapping) | |
159 | return; | |
160 | ||
3a860d16 | 161 | preempt_disable(); |
270eec76 MF |
162 | virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); |
163 | __flush_invalidate_dcache_page_alias(virt, phys); | |
6656920b CZ |
164 | |
165 | virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK); | |
166 | ||
167 | if (alias) | |
168 | __flush_invalidate_dcache_page_alias(virt, phys); | |
169 | ||
170 | if (mapping) | |
171 | __invalidate_icache_page_alias(virt, phys); | |
3a860d16 | 172 | preempt_enable(); |
6656920b CZ |
173 | } |
174 | ||
175 | /* There shouldn't be an entry in the cache for this page anymore. */ | |
176 | } | |
bc652eb6 | 177 | EXPORT_SYMBOL(flush_dcache_page); |
6656920b CZ |
178 | |
179 | /* | |
180 | * For now, flush the whole cache. FIXME?? | |
181 | */ | |
182 | ||
f615136c | 183 | void local_flush_cache_range(struct vm_area_struct *vma, |
6656920b CZ |
184 | unsigned long start, unsigned long end) |
185 | { | |
186 | __flush_invalidate_dcache_all(); | |
187 | __invalidate_icache_all(); | |
188 | } | |
bc652eb6 | 189 | EXPORT_SYMBOL(local_flush_cache_range); |
6656920b CZ |
190 | |
191 | /* | |
192 | * Remove any entry in the cache for this page. | |
193 | * | |
194 | * Note that this function is only called for user pages, so use the | |
195 | * alias versions of the cache flush functions. | |
196 | */ | |
197 | ||
f615136c | 198 | void local_flush_cache_page(struct vm_area_struct *vma, unsigned long address, |
c4c4594b | 199 | unsigned long pfn) |
6656920b CZ |
200 | { |
201 | /* Note that we have to use the 'alias' address to avoid multi-hit */ | |
202 | ||
203 | unsigned long phys = page_to_phys(pfn_to_page(pfn)); | |
204 | unsigned long virt = TLBTEMP_BASE_1 + (address & DCACHE_ALIAS_MASK); | |
205 | ||
3a860d16 | 206 | preempt_disable(); |
6656920b CZ |
207 | __flush_invalidate_dcache_page_alias(virt, phys); |
208 | __invalidate_icache_page_alias(virt, phys); | |
3a860d16 | 209 | preempt_enable(); |
6656920b | 210 | } |
bc652eb6 | 211 | EXPORT_SYMBOL(local_flush_cache_page); |
6656920b | 212 | |
6d0f581d | 213 | #endif /* DCACHE_WAY_SIZE > PAGE_SIZE */ |
6656920b CZ |
214 | |
215 | void | |
4b3073e1 | 216 | update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t *ptep) |
6656920b | 217 | { |
4b3073e1 | 218 | unsigned long pfn = pte_pfn(*ptep); |
6656920b CZ |
219 | struct page *page; |
220 | ||
221 | if (!pfn_valid(pfn)) | |
222 | return; | |
223 | ||
224 | page = pfn_to_page(pfn); | |
225 | ||
226 | /* Invalidate old entry in TLBs */ | |
227 | ||
f615136c | 228 | flush_tlb_page(vma, addr); |
6656920b | 229 | |
6d0f581d | 230 | #if (DCACHE_WAY_SIZE > PAGE_SIZE) |
6656920b CZ |
231 | |
232 | if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) { | |
6656920b | 233 | unsigned long phys = page_to_phys(page); |
270eec76 | 234 | unsigned long tmp; |
6656920b | 235 | |
3a860d16 | 236 | preempt_disable(); |
270eec76 MF |
237 | tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); |
238 | __flush_invalidate_dcache_page_alias(tmp, phys); | |
239 | tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK); | |
c4c4594b CZ |
240 | __flush_invalidate_dcache_page_alias(tmp, phys); |
241 | __invalidate_icache_page_alias(tmp, phys); | |
3a860d16 | 242 | preempt_enable(); |
6656920b CZ |
243 | |
244 | clear_bit(PG_arch_1, &page->flags); | |
245 | } | |
246 | #else | |
247 | if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags) | |
248 | && (vma->vm_flags & VM_EXEC) != 0) { | |
65559100 | 249 | unsigned long paddr = (unsigned long)kmap_atomic(page); |
b67360db CZ |
250 | __flush_dcache_page(paddr); |
251 | __invalidate_icache_page(paddr); | |
6656920b | 252 | set_bit(PG_arch_1, &page->flags); |
65559100 | 253 | kunmap_atomic((void *)paddr); |
6656920b CZ |
254 | } |
255 | #endif | |
256 | } | |
257 | ||
258 | /* | |
259 | * access_process_vm() has called get_user_pages(), which has done a | |
260 | * flush_dcache_page() on the page. | |
261 | */ | |
262 | ||
6d0f581d | 263 | #if (DCACHE_WAY_SIZE > PAGE_SIZE) |
6656920b | 264 | |
c4c4594b | 265 | void copy_to_user_page(struct vm_area_struct *vma, struct page *page, |
6656920b CZ |
266 | unsigned long vaddr, void *dst, const void *src, |
267 | unsigned long len) | |
268 | { | |
269 | unsigned long phys = page_to_phys(page); | |
270 | unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); | |
271 | ||
272 | /* Flush and invalidate user page if aliased. */ | |
273 | ||
274 | if (alias) { | |
c4c4594b | 275 | unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); |
3a860d16 | 276 | preempt_disable(); |
c4c4594b | 277 | __flush_invalidate_dcache_page_alias(t, phys); |
3a860d16 | 278 | preempt_enable(); |
6656920b CZ |
279 | } |
280 | ||
281 | /* Copy data */ | |
282 | ||
283 | memcpy(dst, src, len); | |
284 | ||
285 | /* | |
286 | * Flush and invalidate kernel page if aliased and synchronize | |
287 | * data and instruction caches for executable pages. | |
288 | */ | |
289 | ||
290 | if (alias) { | |
c4c4594b | 291 | unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); |
6656920b | 292 | |
3a860d16 | 293 | preempt_disable(); |
6656920b | 294 | __flush_invalidate_dcache_range((unsigned long) dst, len); |
c4c4594b CZ |
295 | if ((vma->vm_flags & VM_EXEC) != 0) |
296 | __invalidate_icache_page_alias(t, phys); | |
3a860d16 | 297 | preempt_enable(); |
6656920b CZ |
298 | |
299 | } else if ((vma->vm_flags & VM_EXEC) != 0) { | |
300 | __flush_dcache_range((unsigned long)dst,len); | |
301 | __invalidate_icache_range((unsigned long) dst, len); | |
302 | } | |
303 | } | |
304 | ||
305 | extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page, | |
306 | unsigned long vaddr, void *dst, const void *src, | |
307 | unsigned long len) | |
308 | { | |
309 | unsigned long phys = page_to_phys(page); | |
310 | unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); | |
311 | ||
312 | /* | |
313 | * Flush user page if aliased. | |
314 | * (Note: a simply flush would be sufficient) | |
315 | */ | |
316 | ||
317 | if (alias) { | |
c4c4594b | 318 | unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); |
3a860d16 | 319 | preempt_disable(); |
c4c4594b | 320 | __flush_invalidate_dcache_page_alias(t, phys); |
3a860d16 | 321 | preempt_enable(); |
6656920b CZ |
322 | } |
323 | ||
324 | memcpy(dst, src, len); | |
325 | } | |
326 | ||
327 | #endif |