Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * arch/sh/mm/cache-sh4.c | |
3 | * | |
4 | * Copyright (C) 1999, 2000, 2002 Niibe Yutaka | |
deaef20e | 5 | * Copyright (C) 2001 - 2009 Paul Mundt |
1da177e4 | 6 | * Copyright (C) 2003 Richard Curnow |
09b5a10c | 7 | * Copyright (c) 2007 STMicroelectronics (R&D) Ltd. |
1da177e4 LT |
8 | * |
9 | * This file is subject to the terms and conditions of the GNU General Public | |
10 | * License. See the file "COPYING" in the main directory of this archive | |
11 | * for more details. | |
12 | */ | |
1da177e4 | 13 | #include <linux/init.h> |
1da177e4 | 14 | #include <linux/mm.h> |
52e27782 PM |
15 | #include <linux/io.h> |
16 | #include <linux/mutex.h> | |
2277ab4a | 17 | #include <linux/fs.h> |
deaef20e PM |
18 | #include <linux/highmem.h> |
19 | #include <asm/pgtable.h> | |
1da177e4 LT |
20 | #include <asm/mmu_context.h> |
21 | #include <asm/cacheflush.h> | |
22 | ||
28ccf7f9 PM |
23 | /* |
24 | * The maximum number of pages we support up to when doing ranged dcache | |
25 | * flushing. Anything exceeding this will simply flush the dcache in its | |
26 | * entirety. | |
27 | */ | |
09b5a10c | 28 | #define MAX_ICACHE_PAGES 32 |
28ccf7f9 | 29 | |
a7a7c0e1 | 30 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
a252710f | 31 | unsigned long exec_offset); |
b638d0b9 | 32 | |
1da177e4 LT |
33 | /* |
34 | * Write back the range of D-cache, and purge the I-cache. | |
35 | * | |
09b5a10c CS |
36 | * Called from kernel/module.c:sys_init_module and routine for a.out format, |
37 | * signal handler code and kprobes code | |
1da177e4 | 38 | */ |
a6325247 | 39 | static void __uses_jump_to_uncached sh4_flush_icache_range(void *args) |
1da177e4 | 40 | { |
f26b2a56 | 41 | struct flusher_data *data = args; |
f26b2a56 | 42 | unsigned long start, end; |
983f4c51 | 43 | unsigned long flags, v; |
1da177e4 LT |
44 | int i; |
45 | ||
f26b2a56 PM |
46 | start = data->addr1; |
47 | end = data->addr2; | |
48 | ||
682f88ab PM |
49 | /* If there are too many pages then just blow away the caches */ |
50 | if (((end - start) >> PAGE_SHIFT) >= MAX_ICACHE_PAGES) { | |
51 | local_flush_cache_all(NULL); | |
52 | return; | |
53 | } | |
54 | ||
55 | /* | |
56 | * Selectively flush d-cache then invalidate the i-cache. | |
57 | * This is inefficient, so only use this for small ranges. | |
58 | */ | |
59 | start &= ~(L1_CACHE_BYTES-1); | |
60 | end += L1_CACHE_BYTES-1; | |
61 | end &= ~(L1_CACHE_BYTES-1); | |
983f4c51 | 62 | |
682f88ab PM |
63 | local_irq_save(flags); |
64 | jump_to_uncached(); | |
983f4c51 | 65 | |
682f88ab PM |
66 | for (v = start; v < end; v += L1_CACHE_BYTES) { |
67 | unsigned long icacheaddr; | |
a9d244a2 | 68 | int j, n; |
983f4c51 | 69 | |
682f88ab | 70 | __ocbwb(v); |
983f4c51 | 71 | |
682f88ab PM |
72 | icacheaddr = CACHE_IC_ADDRESS_ARRAY | (v & |
73 | cpu_data->icache.entry_mask); | |
09b5a10c | 74 | |
682f88ab | 75 | /* Clear i-cache line valid-bit */ |
a9d244a2 | 76 | n = boot_cpu_data.icache.n_aliases; |
682f88ab | 77 | for (i = 0; i < cpu_data->icache.ways; i++) { |
a9d244a2 MF |
78 | for (j = 0; j < n; j++) |
79 | __raw_writel(0, icacheaddr + (j * PAGE_SIZE)); | |
682f88ab PM |
80 | icacheaddr += cpu_data->icache.way_incr; |
81 | } | |
09b5a10c | 82 | } |
682f88ab PM |
83 | |
84 | back_to_cached(); | |
85 | local_irq_restore(flags); | |
1da177e4 LT |
86 | } |
87 | ||
a7a7c0e1 | 88 | static inline void flush_cache_one(unsigned long start, unsigned long phys) |
1da177e4 | 89 | { |
983f4c51 | 90 | unsigned long flags, exec_offset = 0; |
33573c0e | 91 | |
1da177e4 | 92 | /* |
1f69b6af MF |
93 | * All types of SH-4 require PC to be uncached to operate on the I-cache. |
94 | * Some types of SH-4 require PC to be uncached to operate on the D-cache. | |
1da177e4 | 95 | */ |
7ec9d6f8 | 96 | if ((boot_cpu_data.flags & CPU_HAS_P2_FLUSH_BUG) || |
33573c0e | 97 | (start < CACHE_OC_ADDRESS_ARRAY)) |
1f69b6af | 98 | exec_offset = cached_to_uncached; |
33573c0e | 99 | |
983f4c51 | 100 | local_irq_save(flags); |
a781d1e5 | 101 | __flush_cache_one(start, phys, exec_offset); |
983f4c51 | 102 | local_irq_restore(flags); |
1da177e4 LT |
103 | } |
104 | ||
105 | /* | |
106 | * Write back & invalidate the D-cache of the page. | |
107 | * (To avoid "alias" issues) | |
108 | */ | |
e76a0136 | 109 | static void sh4_flush_dcache_page(void *arg) |
1da177e4 | 110 | { |
e76a0136 | 111 | struct page *page = arg; |
c139a595 | 112 | #ifndef CONFIG_SMP |
2277ab4a PM |
113 | struct address_space *mapping = page_mapping(page); |
114 | ||
2277ab4a PM |
115 | if (mapping && !mapping_mapped(mapping)) |
116 | set_bit(PG_dcache_dirty, &page->flags); | |
117 | else | |
118 | #endif | |
119 | { | |
31c9efde | 120 | unsigned long phys = page_to_phys(page); |
b638d0b9 RC |
121 | unsigned long addr = CACHE_OC_ADDRESS_ARRAY; |
122 | int i, n; | |
1da177e4 LT |
123 | |
124 | /* Loop all the D-cache */ | |
7ec9d6f8 | 125 | n = boot_cpu_data.dcache.n_aliases; |
a781d1e5 | 126 | for (i = 0; i < n; i++, addr += PAGE_SIZE) |
a7a7c0e1 | 127 | flush_cache_one(addr, phys); |
1da177e4 | 128 | } |
fdfc74f9 PM |
129 | |
130 | wmb(); | |
1da177e4 LT |
131 | } |
132 | ||
28ccf7f9 | 133 | /* TODO: Selective icache invalidation through IC address array.. */ |
205a3b43 | 134 | static void __uses_jump_to_uncached flush_icache_all(void) |
1da177e4 | 135 | { |
983f4c51 | 136 | unsigned long flags, ccr; |
1da177e4 | 137 | |
983f4c51 | 138 | local_irq_save(flags); |
cbaa118e | 139 | jump_to_uncached(); |
1da177e4 LT |
140 | |
141 | /* Flush I-cache */ | |
142 | ccr = ctrl_inl(CCR); | |
143 | ccr |= CCR_CACHE_ICI; | |
144 | ctrl_outl(ccr, CCR); | |
145 | ||
29847622 | 146 | /* |
cbaa118e | 147 | * back_to_cached() will take care of the barrier for us, don't add |
29847622 PM |
148 | * another one! |
149 | */ | |
983f4c51 | 150 | |
cbaa118e | 151 | back_to_cached(); |
983f4c51 | 152 | local_irq_restore(flags); |
1da177e4 LT |
153 | } |
154 | ||
bd6df574 | 155 | static void flush_dcache_all(void) |
1da177e4 | 156 | { |
bd6df574 PM |
157 | unsigned long addr, end_addr, entry_offset; |
158 | ||
159 | end_addr = CACHE_OC_ADDRESS_ARRAY + | |
160 | (current_cpu_data.dcache.sets << | |
161 | current_cpu_data.dcache.entry_shift) * | |
162 | current_cpu_data.dcache.ways; | |
163 | ||
164 | entry_offset = 1 << current_cpu_data.dcache.entry_shift; | |
165 | ||
166 | for (addr = CACHE_OC_ADDRESS_ARRAY; addr < end_addr; ) { | |
167 | __raw_writel(0, addr); addr += entry_offset; | |
168 | __raw_writel(0, addr); addr += entry_offset; | |
169 | __raw_writel(0, addr); addr += entry_offset; | |
170 | __raw_writel(0, addr); addr += entry_offset; | |
171 | __raw_writel(0, addr); addr += entry_offset; | |
172 | __raw_writel(0, addr); addr += entry_offset; | |
173 | __raw_writel(0, addr); addr += entry_offset; | |
174 | __raw_writel(0, addr); addr += entry_offset; | |
175 | } | |
a252710f PM |
176 | } |
177 | ||
f26b2a56 | 178 | static void sh4_flush_cache_all(void *unused) |
a252710f PM |
179 | { |
180 | flush_dcache_all(); | |
1da177e4 LT |
181 | flush_icache_all(); |
182 | } | |
183 | ||
28ccf7f9 PM |
184 | /* |
185 | * Note : (RPC) since the caches are physically tagged, the only point | |
186 | * of flush_cache_mm for SH-4 is to get rid of aliases from the | |
187 | * D-cache. The assumption elsewhere, e.g. flush_cache_range, is that | |
188 | * lines can stay resident so long as the virtual address they were | |
189 | * accessed with (hence cache set) is in accord with the physical | |
654d364e | 190 | * address (i.e. tag). It's no different here. |
28ccf7f9 PM |
191 | * |
192 | * Caller takes mm->mmap_sem. | |
193 | */ | |
f26b2a56 | 194 | static void sh4_flush_cache_mm(void *arg) |
1da177e4 | 195 | { |
f26b2a56 PM |
196 | struct mm_struct *mm = arg; |
197 | ||
e7b8b7f1 PM |
198 | if (cpu_context(smp_processor_id(), mm) == NO_CONTEXT) |
199 | return; | |
200 | ||
654d364e | 201 | flush_dcache_all(); |
1da177e4 LT |
202 | } |
203 | ||
204 | /* | |
205 | * Write back and invalidate I/D-caches for the page. | |
206 | * | |
207 | * ADDR: Virtual Address (U0 address) | |
208 | * PFN: Physical page number | |
209 | */ | |
f26b2a56 | 210 | static void sh4_flush_cache_page(void *args) |
1da177e4 | 211 | { |
f26b2a56 PM |
212 | struct flusher_data *data = args; |
213 | struct vm_area_struct *vma; | |
deaef20e | 214 | struct page *page; |
f26b2a56 | 215 | unsigned long address, pfn, phys; |
deaef20e PM |
216 | int map_coherent = 0; |
217 | pgd_t *pgd; | |
218 | pud_t *pud; | |
219 | pmd_t *pmd; | |
220 | pte_t *pte; | |
221 | void *vaddr; | |
b638d0b9 | 222 | |
f26b2a56 | 223 | vma = data->vma; |
abeaf33a | 224 | address = data->addr1 & PAGE_MASK; |
f26b2a56 PM |
225 | pfn = data->addr2; |
226 | phys = pfn << PAGE_SHIFT; | |
deaef20e | 227 | page = pfn_to_page(pfn); |
f26b2a56 | 228 | |
e7b8b7f1 PM |
229 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
230 | return; | |
231 | ||
deaef20e PM |
232 | pgd = pgd_offset(vma->vm_mm, address); |
233 | pud = pud_offset(pgd, address); | |
234 | pmd = pmd_offset(pud, address); | |
235 | pte = pte_offset_kernel(pmd, address); | |
236 | ||
237 | /* If the page isn't present, there is nothing to do here. */ | |
238 | if (!(pte_val(*pte) & _PAGE_PRESENT)) | |
239 | return; | |
1da177e4 | 240 | |
deaef20e PM |
241 | if ((vma->vm_mm == current->active_mm)) |
242 | vaddr = NULL; | |
243 | else { | |
b638d0b9 | 244 | /* |
deaef20e PM |
245 | * Use kmap_coherent or kmap_atomic to do flushes for |
246 | * another ASID than the current one. | |
b638d0b9 | 247 | */ |
deaef20e PM |
248 | map_coherent = (current_cpu_data.dcache.n_aliases && |
249 | !test_bit(PG_dcache_dirty, &page->flags) && | |
250 | page_mapped(page)); | |
251 | if (map_coherent) | |
252 | vaddr = kmap_coherent(page, address); | |
253 | else | |
254 | vaddr = kmap_atomic(page, KM_USER0); | |
255 | ||
256 | address = (unsigned long)vaddr; | |
257 | } | |
258 | ||
259 | if (pages_do_alias(address, phys)) | |
abeaf33a | 260 | flush_cache_one(CACHE_OC_ADDRESS_ARRAY | |
deaef20e PM |
261 | (address & shm_align_mask), phys); |
262 | ||
263 | if (vma->vm_flags & VM_EXEC) | |
264 | flush_icache_all(); | |
265 | ||
266 | if (vaddr) { | |
267 | if (map_coherent) | |
268 | kunmap_coherent(vaddr); | |
269 | else | |
270 | kunmap_atomic(vaddr, KM_USER0); | |
b638d0b9 | 271 | } |
1da177e4 LT |
272 | } |
273 | ||
274 | /* | |
275 | * Write back and invalidate D-caches. | |
276 | * | |
277 | * START, END: Virtual Address (U0 address) | |
278 | * | |
279 | * NOTE: We need to flush the _physical_ page entry. | |
280 | * Flushing the cache lines for U0 only isn't enough. | |
281 | * We need to flush for P1 too, which may contain aliases. | |
282 | */ | |
f26b2a56 | 283 | static void sh4_flush_cache_range(void *args) |
1da177e4 | 284 | { |
f26b2a56 PM |
285 | struct flusher_data *data = args; |
286 | struct vm_area_struct *vma; | |
287 | unsigned long start, end; | |
288 | ||
289 | vma = data->vma; | |
290 | start = data->addr1; | |
291 | end = data->addr2; | |
292 | ||
e7b8b7f1 PM |
293 | if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) |
294 | return; | |
295 | ||
b638d0b9 RC |
296 | /* |
297 | * If cache is only 4k-per-way, there are never any 'aliases'. Since | |
298 | * the cache is physically tagged, the data can just be left in there. | |
299 | */ | |
7ec9d6f8 | 300 | if (boot_cpu_data.dcache.n_aliases == 0) |
b638d0b9 RC |
301 | return; |
302 | ||
654d364e | 303 | flush_dcache_all(); |
b638d0b9 | 304 | |
654d364e | 305 | if (vma->vm_flags & VM_EXEC) |
1da177e4 LT |
306 | flush_icache_all(); |
307 | } | |
308 | ||
b638d0b9 | 309 | /** |
a7a7c0e1 | 310 | * __flush_cache_one |
b638d0b9 RC |
311 | * |
312 | * @addr: address in memory mapped cache array | |
313 | * @phys: P1 address to flush (has to match tags if addr has 'A' bit | |
314 | * set i.e. associative write) | |
315 | * @exec_offset: set to 0x20000000 if flush has to be executed from P2 | |
316 | * region else 0x0 | |
317 | * | |
318 | * The offset into the cache array implied by 'addr' selects the | |
319 | * 'colour' of the virtual address range that will be flushed. The | |
320 | * operation (purge/write-back) is selected by the lower 2 bits of | |
321 | * 'phys'. | |
322 | */ | |
a7a7c0e1 | 323 | static void __flush_cache_one(unsigned long addr, unsigned long phys, |
b638d0b9 RC |
324 | unsigned long exec_offset) |
325 | { | |
326 | int way_count; | |
327 | unsigned long base_addr = addr; | |
328 | struct cache_info *dcache; | |
329 | unsigned long way_incr; | |
330 | unsigned long a, ea, p; | |
331 | unsigned long temp_pc; | |
332 | ||
7ec9d6f8 | 333 | dcache = &boot_cpu_data.dcache; |
b638d0b9 RC |
334 | /* Write this way for better assembly. */ |
335 | way_count = dcache->ways; | |
336 | way_incr = dcache->way_incr; | |
337 | ||
338 | /* | |
339 | * Apply exec_offset (i.e. branch to P2 if required.). | |
340 | * | |
341 | * FIXME: | |
342 | * | |
343 | * If I write "=r" for the (temp_pc), it puts this in r6 hence | |
344 | * trashing exec_offset before it's been added on - why? Hence | |
345 | * "=&r" as a 'workaround' | |
346 | */ | |
347 | asm volatile("mov.l 1f, %0\n\t" | |
348 | "add %1, %0\n\t" | |
349 | "jmp @%0\n\t" | |
350 | "nop\n\t" | |
351 | ".balign 4\n\t" | |
352 | "1: .long 2f\n\t" | |
353 | "2:\n" : "=&r" (temp_pc) : "r" (exec_offset)); | |
354 | ||
355 | /* | |
356 | * We know there will be >=1 iteration, so write as do-while to avoid | |
357 | * pointless nead-of-loop check for 0 iterations. | |
358 | */ | |
359 | do { | |
360 | ea = base_addr + PAGE_SIZE; | |
361 | a = base_addr; | |
362 | p = phys; | |
363 | ||
364 | do { | |
365 | *(volatile unsigned long *)a = p; | |
366 | /* | |
367 | * Next line: intentionally not p+32, saves an add, p | |
368 | * will do since only the cache tag bits need to | |
369 | * match. | |
370 | */ | |
371 | *(volatile unsigned long *)(a+32) = p; | |
372 | a += 64; | |
373 | p += 64; | |
374 | } while (a < ea); | |
375 | ||
376 | base_addr += way_incr; | |
377 | } while (--way_count != 0); | |
378 | } | |
379 | ||
37443ef3 PM |
380 | extern void __weak sh4__flush_region_init(void); |
381 | ||
382 | /* | |
383 | * SH-4 has virtually indexed and physically tagged cache. | |
384 | */ | |
385 | void __init sh4_cache_init(void) | |
386 | { | |
387 | printk("PVR=%08x CVR=%08x PRR=%08x\n", | |
388 | ctrl_inl(CCN_PVR), | |
389 | ctrl_inl(CCN_CVR), | |
390 | ctrl_inl(CCN_PRR)); | |
391 | ||
f26b2a56 PM |
392 | local_flush_icache_range = sh4_flush_icache_range; |
393 | local_flush_dcache_page = sh4_flush_dcache_page; | |
394 | local_flush_cache_all = sh4_flush_cache_all; | |
395 | local_flush_cache_mm = sh4_flush_cache_mm; | |
396 | local_flush_cache_dup_mm = sh4_flush_cache_mm; | |
397 | local_flush_cache_page = sh4_flush_cache_page; | |
398 | local_flush_cache_range = sh4_flush_cache_range; | |
37443ef3 PM |
399 | |
400 | sh4__flush_region_init(); | |
401 | } |