Commit | Line | Data |
---|---|---|
1a472c9d AK |
1 | /* |
2 | * TLB flush routines for radix kernels. | |
3 | * | |
4 | * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/mm.h> | |
13 | #include <linux/hugetlb.h> | |
14 | #include <linux/memblock.h> | |
15 | ||
a25bd72b | 16 | #include <asm/ppc-opcode.h> |
1a472c9d AK |
17 | #include <asm/tlb.h> |
18 | #include <asm/tlbflush.h> | |
0428491c | 19 | #include <asm/trace.h> |
a25bd72b | 20 | #include <asm/cputhreads.h> |
1a472c9d | 21 | |
36194812 AK |
22 | #define RIC_FLUSH_TLB 0 |
23 | #define RIC_FLUSH_PWC 1 | |
24 | #define RIC_FLUSH_ALL 2 | |
25 | ||
d4748276 NP |
26 | /* |
27 | * tlbiel instruction for radix, set invalidation | |
28 | * i.e., r=1 and is=01 or is=10 or is=11 | |
29 | */ | |
30 | static inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, | |
31 | unsigned int pid, | |
32 | unsigned int ric, unsigned int prs) | |
33 | { | |
34 | unsigned long rb; | |
35 | unsigned long rs; | |
36 | unsigned int r = 1; /* radix format */ | |
37 | ||
38 | rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); | |
39 | rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); | |
40 | ||
41 | asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) | |
42 | : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r) | |
43 | : "memory"); | |
44 | } | |
45 | ||
46 | static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) | |
47 | { | |
48 | unsigned int set; | |
49 | ||
50 | asm volatile("ptesync": : :"memory"); | |
51 | ||
52 | /* | |
53 | * Flush the first set of the TLB, and the entire Page Walk Cache | |
54 | * and partition table entries. Then flush the remaining sets of the | |
55 | * TLB. | |
56 | */ | |
57 | tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); | |
58 | for (set = 1; set < num_sets; set++) | |
59 | tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); | |
60 | ||
61 | /* Do the same for process scoped entries. */ | |
62 | tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); | |
63 | for (set = 1; set < num_sets; set++) | |
64 | tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); | |
65 | ||
66 | asm volatile("ptesync": : :"memory"); | |
67 | } | |
68 | ||
69 | void radix__tlbiel_all(unsigned int action) | |
70 | { | |
71 | unsigned int is; | |
72 | ||
73 | switch (action) { | |
74 | case TLB_INVAL_SCOPE_GLOBAL: | |
75 | is = 3; | |
76 | break; | |
77 | case TLB_INVAL_SCOPE_LPID: | |
78 | is = 2; | |
79 | break; | |
80 | default: | |
81 | BUG(); | |
82 | } | |
83 | ||
84 | if (early_cpu_has_feature(CPU_FTR_ARCH_300)) | |
85 | tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); | |
86 | else | |
87 | WARN(1, "%s called on pre-POWER9 CPU\n", __func__); | |
88 | ||
89 | asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); | |
90 | } | |
91 | ||
36194812 AK |
92 | static inline void __tlbiel_pid(unsigned long pid, int set, |
93 | unsigned long ric) | |
1a472c9d | 94 | { |
36194812 | 95 | unsigned long rb,rs,prs,r; |
1a472c9d AK |
96 | |
97 | rb = PPC_BIT(53); /* IS = 1 */ | |
98 | rb |= set << PPC_BITLSHIFT(51); | |
99 | rs = ((unsigned long)pid) << PPC_BITLSHIFT(31); | |
100 | prs = 1; /* process scoped */ | |
101 | r = 1; /* raidx format */ | |
1a472c9d | 102 | |
8cd6d3c2 | 103 | asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) |
1a472c9d | 104 | : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); |
0428491c | 105 | trace_tlbie(0, 1, rb, rs, ric, prs, r); |
1a472c9d AK |
106 | } |
107 | ||
0b2f5a8a NP |
108 | static inline void __tlbie_pid(unsigned long pid, unsigned long ric) |
109 | { | |
110 | unsigned long rb,rs,prs,r; | |
111 | ||
112 | rb = PPC_BIT(53); /* IS = 1 */ | |
113 | rs = pid << PPC_BITLSHIFT(31); | |
114 | prs = 1; /* process scoped */ | |
115 | r = 1; /* raidx format */ | |
116 | ||
117 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
118 | : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); | |
119 | trace_tlbie(0, 0, rb, rs, ric, prs, r); | |
120 | } | |
121 | ||
243fee32 AK |
122 | static inline void __tlbiel_va(unsigned long va, unsigned long pid, |
123 | unsigned long ap, unsigned long ric) | |
124 | { | |
125 | unsigned long rb,rs,prs,r; | |
126 | ||
127 | rb = va & ~(PPC_BITMASK(52, 63)); | |
128 | rb |= ap << PPC_BITLSHIFT(58); | |
129 | rs = pid << PPC_BITLSHIFT(31); | |
130 | prs = 1; /* process scoped */ | |
131 | r = 1; /* raidx format */ | |
132 | ||
133 | asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) | |
134 | : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); | |
135 | trace_tlbie(0, 1, rb, rs, ric, prs, r); | |
136 | } | |
137 | ||
138 | static inline void __tlbie_va(unsigned long va, unsigned long pid, | |
139 | unsigned long ap, unsigned long ric) | |
140 | { | |
141 | unsigned long rb,rs,prs,r; | |
142 | ||
143 | rb = va & ~(PPC_BITMASK(52, 63)); | |
144 | rb |= ap << PPC_BITLSHIFT(58); | |
145 | rs = pid << PPC_BITLSHIFT(31); | |
146 | prs = 1; /* process scoped */ | |
147 | r = 1; /* raidx format */ | |
148 | ||
149 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
150 | : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); | |
151 | trace_tlbie(0, 0, rb, rs, ric, prs, r); | |
152 | } | |
153 | ||
1a472c9d AK |
154 | /* |
155 | * We use 128 set in radix mode and 256 set in hpt mode. | |
156 | */ | |
36194812 | 157 | static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) |
1a472c9d AK |
158 | { |
159 | int set; | |
160 | ||
f7327e0b | 161 | asm volatile("ptesync": : :"memory"); |
a5998fcb AK |
162 | |
163 | /* | |
164 | * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL, | |
165 | * also flush the entire Page Walk Cache. | |
166 | */ | |
167 | __tlbiel_pid(pid, 0, ric); | |
168 | ||
5ce5fe14 BH |
169 | /* For PWC, only one flush is needed */ |
170 | if (ric == RIC_FLUSH_PWC) { | |
171 | asm volatile("ptesync": : :"memory"); | |
172 | return; | |
173 | } | |
a5998fcb | 174 | |
5ce5fe14 | 175 | /* For the remaining sets, just flush the TLB */ |
a5998fcb | 176 | for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++) |
5ce5fe14 | 177 | __tlbiel_pid(pid, set, RIC_FLUSH_TLB); |
a5998fcb | 178 | |
f7327e0b | 179 | asm volatile("ptesync": : :"memory"); |
90c1e3c2 | 180 | asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); |
1a472c9d AK |
181 | } |
182 | ||
36194812 | 183 | static inline void _tlbie_pid(unsigned long pid, unsigned long ric) |
1a472c9d | 184 | { |
1a472c9d | 185 | asm volatile("ptesync": : :"memory"); |
80a4ae20 BH |
186 | |
187 | /* | |
188 | * Workaround the fact that the "ric" argument to __tlbie_pid | |
189 | * must be a compile-time contraint to match the "i" constraint | |
190 | * in the asm statement. | |
191 | */ | |
192 | switch (ric) { | |
193 | case RIC_FLUSH_TLB: | |
194 | __tlbie_pid(pid, RIC_FLUSH_TLB); | |
195 | break; | |
196 | case RIC_FLUSH_PWC: | |
197 | __tlbie_pid(pid, RIC_FLUSH_PWC); | |
198 | break; | |
199 | case RIC_FLUSH_ALL: | |
200 | default: | |
201 | __tlbie_pid(pid, RIC_FLUSH_ALL); | |
202 | } | |
1a472c9d AK |
203 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
204 | } | |
205 | ||
cbf09c83 NP |
206 | static inline void __tlbiel_va_range(unsigned long start, unsigned long end, |
207 | unsigned long pid, unsigned long page_size, | |
208 | unsigned long psize) | |
209 | { | |
210 | unsigned long addr; | |
211 | unsigned long ap = mmu_get_ap(psize); | |
212 | ||
213 | for (addr = start; addr < end; addr += page_size) | |
214 | __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); | |
215 | } | |
216 | ||
14001c60 | 217 | static inline void _tlbiel_va(unsigned long va, unsigned long pid, |
d665767e | 218 | unsigned long psize, unsigned long ric) |
14001c60 | 219 | { |
d665767e NP |
220 | unsigned long ap = mmu_get_ap(psize); |
221 | ||
14001c60 NP |
222 | asm volatile("ptesync": : :"memory"); |
223 | __tlbiel_va(va, pid, ap, ric); | |
224 | asm volatile("ptesync": : :"memory"); | |
225 | } | |
226 | ||
d665767e NP |
227 | static inline void _tlbiel_va_range(unsigned long start, unsigned long end, |
228 | unsigned long pid, unsigned long page_size, | |
0b2f5a8a | 229 | unsigned long psize, bool also_pwc) |
d665767e | 230 | { |
d665767e | 231 | asm volatile("ptesync": : :"memory"); |
0b2f5a8a NP |
232 | if (also_pwc) |
233 | __tlbiel_pid(pid, 0, RIC_FLUSH_PWC); | |
cbf09c83 | 234 | __tlbiel_va_range(start, end, pid, page_size, psize); |
d665767e NP |
235 | asm volatile("ptesync": : :"memory"); |
236 | } | |
237 | ||
cbf09c83 NP |
238 | static inline void __tlbie_va_range(unsigned long start, unsigned long end, |
239 | unsigned long pid, unsigned long page_size, | |
240 | unsigned long psize) | |
241 | { | |
242 | unsigned long addr; | |
243 | unsigned long ap = mmu_get_ap(psize); | |
244 | ||
245 | for (addr = start; addr < end; addr += page_size) | |
246 | __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); | |
247 | } | |
248 | ||
14001c60 | 249 | static inline void _tlbie_va(unsigned long va, unsigned long pid, |
d665767e | 250 | unsigned long psize, unsigned long ric) |
14001c60 | 251 | { |
d665767e NP |
252 | unsigned long ap = mmu_get_ap(psize); |
253 | ||
14001c60 NP |
254 | asm volatile("ptesync": : :"memory"); |
255 | __tlbie_va(va, pid, ap, ric); | |
256 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); | |
257 | } | |
258 | ||
d665767e NP |
259 | static inline void _tlbie_va_range(unsigned long start, unsigned long end, |
260 | unsigned long pid, unsigned long page_size, | |
0b2f5a8a | 261 | unsigned long psize, bool also_pwc) |
d665767e | 262 | { |
d665767e | 263 | asm volatile("ptesync": : :"memory"); |
0b2f5a8a NP |
264 | if (also_pwc) |
265 | __tlbie_pid(pid, RIC_FLUSH_PWC); | |
cbf09c83 | 266 | __tlbie_va_range(start, end, pid, page_size, psize); |
d665767e NP |
267 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
268 | } | |
14001c60 | 269 | |
1a472c9d AK |
270 | /* |
271 | * Base TLB flushing operations: | |
272 | * | |
273 | * - flush_tlb_mm(mm) flushes the specified mm context TLB's | |
274 | * - flush_tlb_page(vma, vmaddr) flushes one page | |
275 | * - flush_tlb_range(vma, start, end) flushes a range of pages | |
276 | * - flush_tlb_kernel_range(start, end) flushes kernel pages | |
277 | * | |
278 | * - local_* variants of page and mm only apply to the current | |
279 | * processor | |
280 | */ | |
281 | void radix__local_flush_tlb_mm(struct mm_struct *mm) | |
282 | { | |
9690c157 | 283 | unsigned long pid; |
1a472c9d AK |
284 | |
285 | preempt_disable(); | |
286 | pid = mm->context.id; | |
287 | if (pid != MMU_NO_CONTEXT) | |
a46cc7a9 | 288 | _tlbiel_pid(pid, RIC_FLUSH_TLB); |
1a472c9d AK |
289 | preempt_enable(); |
290 | } | |
291 | EXPORT_SYMBOL(radix__local_flush_tlb_mm); | |
292 | ||
a46cc7a9 | 293 | #ifndef CONFIG_SMP |
6110236b | 294 | void radix__local_flush_all_mm(struct mm_struct *mm) |
a145abf1 AK |
295 | { |
296 | unsigned long pid; | |
a145abf1 AK |
297 | |
298 | preempt_disable(); | |
a145abf1 AK |
299 | pid = mm->context.id; |
300 | if (pid != MMU_NO_CONTEXT) | |
a46cc7a9 | 301 | _tlbiel_pid(pid, RIC_FLUSH_ALL); |
a145abf1 AK |
302 | preempt_enable(); |
303 | } | |
6110236b | 304 | EXPORT_SYMBOL(radix__local_flush_all_mm); |
a46cc7a9 | 305 | #endif /* CONFIG_SMP */ |
a145abf1 | 306 | |
f22dfc91 | 307 | void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, |
fbfa26d8 | 308 | int psize) |
1a472c9d | 309 | { |
9690c157 | 310 | unsigned long pid; |
1a472c9d AK |
311 | |
312 | preempt_disable(); | |
67730272 | 313 | pid = mm->context.id; |
1a472c9d | 314 | if (pid != MMU_NO_CONTEXT) |
d665767e | 315 | _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); |
1a472c9d AK |
316 | preempt_enable(); |
317 | } | |
318 | ||
319 | void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) | |
320 | { | |
48483760 AK |
321 | #ifdef CONFIG_HUGETLB_PAGE |
322 | /* need the return fix for nohash.c */ | |
67730272 ME |
323 | if (is_vm_hugetlb_page(vma)) |
324 | return radix__local_flush_hugetlb_page(vma, vmaddr); | |
48483760 | 325 | #endif |
67730272 | 326 | radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); |
1a472c9d AK |
327 | } |
328 | EXPORT_SYMBOL(radix__local_flush_tlb_page); | |
329 | ||
80a4ae20 BH |
330 | static bool mm_needs_flush_escalation(struct mm_struct *mm) |
331 | { | |
332 | /* | |
333 | * P9 nest MMU has issues with the page walk cache | |
334 | * caching PTEs and not flushing them properly when | |
335 | * RIC = 0 for a PID/LPID invalidate | |
336 | */ | |
337 | return atomic_read(&mm->context.copros) != 0; | |
338 | } | |
339 | ||
1a472c9d | 340 | #ifdef CONFIG_SMP |
1a472c9d AK |
341 | void radix__flush_tlb_mm(struct mm_struct *mm) |
342 | { | |
9690c157 | 343 | unsigned long pid; |
1a472c9d | 344 | |
1a472c9d AK |
345 | pid = mm->context.id; |
346 | if (unlikely(pid == MMU_NO_CONTEXT)) | |
dffe8449 | 347 | return; |
1a472c9d | 348 | |
dffe8449 | 349 | preempt_disable(); |
80a4ae20 BH |
350 | if (!mm_is_thread_local(mm)) { |
351 | if (mm_needs_flush_escalation(mm)) | |
352 | _tlbie_pid(pid, RIC_FLUSH_ALL); | |
353 | else | |
354 | _tlbie_pid(pid, RIC_FLUSH_TLB); | |
355 | } else | |
a46cc7a9 | 356 | _tlbiel_pid(pid, RIC_FLUSH_TLB); |
1a472c9d AK |
357 | preempt_enable(); |
358 | } | |
359 | EXPORT_SYMBOL(radix__flush_tlb_mm); | |
360 | ||
6110236b | 361 | void radix__flush_all_mm(struct mm_struct *mm) |
a145abf1 AK |
362 | { |
363 | unsigned long pid; | |
a145abf1 | 364 | |
a145abf1 AK |
365 | pid = mm->context.id; |
366 | if (unlikely(pid == MMU_NO_CONTEXT)) | |
dffe8449 | 367 | return; |
a145abf1 | 368 | |
dffe8449 | 369 | preempt_disable(); |
3c9ac2bc | 370 | if (!mm_is_thread_local(mm)) |
a46cc7a9 | 371 | _tlbie_pid(pid, RIC_FLUSH_ALL); |
3c9ac2bc | 372 | else |
a46cc7a9 | 373 | _tlbiel_pid(pid, RIC_FLUSH_ALL); |
a145abf1 AK |
374 | preempt_enable(); |
375 | } | |
6110236b | 376 | EXPORT_SYMBOL(radix__flush_all_mm); |
a46cc7a9 BH |
377 | |
378 | void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) | |
379 | { | |
380 | tlb->need_flush_all = 1; | |
381 | } | |
a145abf1 AK |
382 | EXPORT_SYMBOL(radix__flush_tlb_pwc); |
383 | ||
f22dfc91 | 384 | void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, |
fbfa26d8 | 385 | int psize) |
1a472c9d | 386 | { |
9690c157 | 387 | unsigned long pid; |
1a472c9d | 388 | |
67730272 | 389 | pid = mm->context.id; |
1a472c9d | 390 | if (unlikely(pid == MMU_NO_CONTEXT)) |
dffe8449 NP |
391 | return; |
392 | ||
393 | preempt_disable(); | |
3c9ac2bc | 394 | if (!mm_is_thread_local(mm)) |
d665767e | 395 | _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); |
3c9ac2bc | 396 | else |
d665767e | 397 | _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); |
1a472c9d AK |
398 | preempt_enable(); |
399 | } | |
400 | ||
401 | void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) | |
402 | { | |
48483760 | 403 | #ifdef CONFIG_HUGETLB_PAGE |
67730272 ME |
404 | if (is_vm_hugetlb_page(vma)) |
405 | return radix__flush_hugetlb_page(vma, vmaddr); | |
48483760 | 406 | #endif |
67730272 | 407 | radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize); |
1a472c9d AK |
408 | } |
409 | EXPORT_SYMBOL(radix__flush_tlb_page); | |
410 | ||
a46cc7a9 BH |
411 | #else /* CONFIG_SMP */ |
412 | #define radix__flush_all_mm radix__local_flush_all_mm | |
1a472c9d AK |
413 | #endif /* CONFIG_SMP */ |
414 | ||
415 | void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) | |
416 | { | |
36194812 | 417 | _tlbie_pid(0, RIC_FLUSH_ALL); |
1a472c9d AK |
418 | } |
419 | EXPORT_SYMBOL(radix__flush_tlb_kernel_range); | |
420 | ||
cbf09c83 NP |
421 | #define TLB_FLUSH_ALL -1UL |
422 | ||
1a472c9d | 423 | /* |
cbf09c83 NP |
424 | * Number of pages above which we invalidate the entire PID rather than |
425 | * flush individual pages, for local and global flushes respectively. | |
426 | * | |
427 | * tlbie goes out to the interconnect and individual ops are more costly. | |
428 | * It also does not iterate over sets like the local tlbiel variant when | |
429 | * invalidating a full PID, so it has a far lower threshold to change from | |
430 | * individual page flushes to full-pid flushes. | |
1a472c9d | 431 | */ |
cbf09c83 | 432 | static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; |
f6f27951 | 433 | static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2; |
cbf09c83 | 434 | |
1a472c9d AK |
435 | void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
436 | unsigned long end) | |
437 | ||
438 | { | |
439 | struct mm_struct *mm = vma->vm_mm; | |
cbf09c83 NP |
440 | unsigned long pid; |
441 | unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; | |
442 | unsigned long page_size = 1UL << page_shift; | |
443 | unsigned long nr_pages = (end - start) >> page_shift; | |
444 | bool local, full; | |
a46cc7a9 | 445 | |
cbf09c83 NP |
446 | #ifdef CONFIG_HUGETLB_PAGE |
447 | if (is_vm_hugetlb_page(vma)) | |
448 | return radix__flush_hugetlb_tlb_range(vma, start, end); | |
449 | #endif | |
450 | ||
451 | pid = mm->context.id; | |
452 | if (unlikely(pid == MMU_NO_CONTEXT)) | |
453 | return; | |
454 | ||
455 | preempt_disable(); | |
f6f27951 NP |
456 | if (mm_is_thread_local(mm)) { |
457 | local = true; | |
458 | full = (end == TLB_FLUSH_ALL || | |
459 | nr_pages > tlb_local_single_page_flush_ceiling); | |
460 | } else { | |
461 | local = false; | |
462 | full = (end == TLB_FLUSH_ALL || | |
463 | nr_pages > tlb_single_page_flush_ceiling); | |
464 | } | |
cbf09c83 NP |
465 | |
466 | if (full) { | |
80a4ae20 | 467 | if (local) { |
cbf09c83 | 468 | _tlbiel_pid(pid, RIC_FLUSH_TLB); |
80a4ae20 BH |
469 | } else { |
470 | if (mm_needs_flush_escalation(mm)) | |
471 | _tlbie_pid(pid, RIC_FLUSH_ALL); | |
472 | else | |
473 | _tlbie_pid(pid, RIC_FLUSH_TLB); | |
474 | } | |
cbf09c83 NP |
475 | } else { |
476 | bool hflush = false; | |
477 | unsigned long hstart, hend; | |
478 | ||
479 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
480 | hstart = (start + HPAGE_PMD_SIZE - 1) >> HPAGE_PMD_SHIFT; | |
481 | hend = end >> HPAGE_PMD_SHIFT; | |
482 | if (hstart < hend) { | |
483 | hstart <<= HPAGE_PMD_SHIFT; | |
484 | hend <<= HPAGE_PMD_SHIFT; | |
485 | hflush = true; | |
486 | } | |
487 | #endif | |
488 | ||
489 | asm volatile("ptesync": : :"memory"); | |
490 | if (local) { | |
491 | __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); | |
492 | if (hflush) | |
493 | __tlbiel_va_range(hstart, hend, pid, | |
494 | HPAGE_PMD_SIZE, MMU_PAGE_2M); | |
495 | asm volatile("ptesync": : :"memory"); | |
496 | } else { | |
497 | __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); | |
498 | if (hflush) | |
499 | __tlbie_va_range(hstart, hend, pid, | |
500 | HPAGE_PMD_SIZE, MMU_PAGE_2M); | |
501 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); | |
502 | } | |
503 | } | |
504 | preempt_enable(); | |
1a472c9d AK |
505 | } |
506 | EXPORT_SYMBOL(radix__flush_tlb_range); | |
507 | ||
912cc87a AK |
508 | static int radix_get_mmu_psize(int page_size) |
509 | { | |
510 | int psize; | |
511 | ||
512 | if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift)) | |
513 | psize = mmu_virtual_psize; | |
514 | else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift)) | |
515 | psize = MMU_PAGE_2M; | |
516 | else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift)) | |
517 | psize = MMU_PAGE_1G; | |
518 | else | |
519 | return -1; | |
520 | return psize; | |
521 | } | |
1a472c9d | 522 | |
0b2f5a8a NP |
523 | static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, |
524 | unsigned long end, int psize); | |
525 | ||
1a472c9d AK |
526 | void radix__tlb_flush(struct mmu_gather *tlb) |
527 | { | |
8cb8140c | 528 | int psize = 0; |
1a472c9d | 529 | struct mm_struct *mm = tlb->mm; |
8cb8140c AK |
530 | int page_size = tlb->page_size; |
531 | ||
8cb8140c AK |
532 | /* |
533 | * if page size is not something we understand, do a full mm flush | |
30b49ec7 NP |
534 | * |
535 | * A "fullmm" flush must always do a flush_all_mm (RIC=2) flush | |
536 | * that flushes the process table entry cache upon process teardown. | |
537 | * See the comment for radix in arch_exit_mmap(). | |
8cb8140c | 538 | */ |
0b2f5a8a | 539 | if (tlb->fullmm) { |
a46cc7a9 | 540 | radix__flush_all_mm(mm); |
0b2f5a8a NP |
541 | } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) { |
542 | if (!tlb->need_flush_all) | |
543 | radix__flush_tlb_mm(mm); | |
544 | else | |
545 | radix__flush_all_mm(mm); | |
546 | } else { | |
547 | unsigned long start = tlb->start; | |
548 | unsigned long end = tlb->end; | |
549 | ||
550 | if (!tlb->need_flush_all) | |
551 | radix__flush_tlb_range_psize(mm, start, end, psize); | |
552 | else | |
553 | radix__flush_tlb_pwc_range_psize(mm, start, end, psize); | |
554 | } | |
555 | tlb->need_flush_all = 0; | |
8cb8140c AK |
556 | } |
557 | ||
0b2f5a8a NP |
558 | static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, |
559 | unsigned long start, unsigned long end, | |
560 | int psize, bool also_pwc) | |
8cb8140c AK |
561 | { |
562 | unsigned long pid; | |
cbf09c83 NP |
563 | unsigned int page_shift = mmu_psize_defs[psize].shift; |
564 | unsigned long page_size = 1UL << page_shift; | |
565 | unsigned long nr_pages = (end - start) >> page_shift; | |
566 | bool local, full; | |
8cb8140c | 567 | |
67730272 | 568 | pid = mm->context.id; |
8cb8140c | 569 | if (unlikely(pid == MMU_NO_CONTEXT)) |
dffe8449 | 570 | return; |
8cb8140c | 571 | |
dffe8449 | 572 | preempt_disable(); |
f6f27951 NP |
573 | if (mm_is_thread_local(mm)) { |
574 | local = true; | |
575 | full = (end == TLB_FLUSH_ALL || | |
576 | nr_pages > tlb_local_single_page_flush_ceiling); | |
577 | } else { | |
578 | local = false; | |
579 | full = (end == TLB_FLUSH_ALL || | |
580 | nr_pages > tlb_single_page_flush_ceiling); | |
581 | } | |
cbf09c83 NP |
582 | |
583 | if (full) { | |
80a4ae20 BH |
584 | if (!local && mm_needs_flush_escalation(mm)) |
585 | also_pwc = true; | |
586 | ||
8cb8140c | 587 | if (local) |
0b2f5a8a | 588 | _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); |
8cb8140c | 589 | else |
0b2f5a8a | 590 | _tlbie_pid(pid, also_pwc ? RIC_FLUSH_ALL: RIC_FLUSH_TLB); |
dffe8449 | 591 | } else { |
14001c60 | 592 | if (local) |
0b2f5a8a | 593 | _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); |
14001c60 | 594 | else |
0b2f5a8a | 595 | _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); |
8cb8140c | 596 | } |
8cb8140c | 597 | preempt_enable(); |
1a472c9d | 598 | } |
912cc87a | 599 | |
0b2f5a8a NP |
600 | void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, |
601 | unsigned long end, int psize) | |
602 | { | |
603 | return __radix__flush_tlb_range_psize(mm, start, end, psize, false); | |
604 | } | |
605 | ||
606 | static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, | |
607 | unsigned long end, int psize) | |
608 | { | |
609 | __radix__flush_tlb_range_psize(mm, start, end, psize, true); | |
610 | } | |
611 | ||
424de9c6 BH |
612 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
613 | void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) | |
614 | { | |
424de9c6 | 615 | unsigned long pid, end; |
424de9c6 | 616 | |
67730272 | 617 | pid = mm->context.id; |
424de9c6 | 618 | if (unlikely(pid == MMU_NO_CONTEXT)) |
dffe8449 | 619 | return; |
424de9c6 BH |
620 | |
621 | /* 4k page size, just blow the world */ | |
622 | if (PAGE_SIZE == 0x1000) { | |
623 | radix__flush_all_mm(mm); | |
624 | return; | |
625 | } | |
626 | ||
cbf09c83 NP |
627 | end = addr + HPAGE_PMD_SIZE; |
628 | ||
629 | /* Otherwise first do the PWC, then iterate the pages. */ | |
dffe8449 | 630 | preempt_disable(); |
424de9c6 | 631 | |
cbf09c83 | 632 | if (mm_is_thread_local(mm)) { |
0b2f5a8a | 633 | _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); |
cbf09c83 | 634 | } else { |
0b2f5a8a | 635 | _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); |
cbf09c83 | 636 | } |
14001c60 | 637 | |
424de9c6 BH |
638 | preempt_enable(); |
639 | } | |
640 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
641 | ||
d8e91e93 AK |
642 | void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, |
643 | unsigned long start, unsigned long end) | |
644 | { | |
645 | radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); | |
646 | } | |
647 | EXPORT_SYMBOL(radix__flush_pmd_tlb_range); | |
be34d300 AK |
648 | |
649 | void radix__flush_tlb_all(void) | |
650 | { | |
651 | unsigned long rb,prs,r,rs; | |
652 | unsigned long ric = RIC_FLUSH_ALL; | |
653 | ||
654 | rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */ | |
655 | prs = 0; /* partition scoped */ | |
656 | r = 1; /* raidx format */ | |
657 | rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */ | |
658 | ||
659 | asm volatile("ptesync": : :"memory"); | |
660 | /* | |
661 | * now flush guest entries by passing PRS = 1 and LPID != 0 | |
662 | */ | |
663 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
664 | : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory"); | |
665 | /* | |
666 | * now flush host entires by passing PRS = 0 and LPID == 0 | |
667 | */ | |
668 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
669 | : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory"); | |
670 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); | |
671 | } | |
6d3a0379 AK |
672 | |
673 | void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, | |
674 | unsigned long address) | |
675 | { | |
676 | /* | |
677 | * We track page size in pte only for DD1, So we can | |
678 | * call this only on DD1. | |
679 | */ | |
680 | if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) { | |
681 | VM_WARN_ON(1); | |
682 | return; | |
683 | } | |
684 | ||
ddb014b6 | 685 | if (old_pte & R_PAGE_LARGE) |
6d3a0379 AK |
686 | radix__flush_tlb_page_psize(mm, address, MMU_PAGE_2M); |
687 | else | |
688 | radix__flush_tlb_page_psize(mm, address, mmu_virtual_psize); | |
689 | } | |
a25bd72b BH |
690 | |
691 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
692 | extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) | |
693 | { | |
694 | unsigned int pid = mm->context.id; | |
695 | ||
696 | if (unlikely(pid == MMU_NO_CONTEXT)) | |
697 | return; | |
698 | ||
699 | /* | |
700 | * If this context hasn't run on that CPU before and KVM is | |
701 | * around, there's a slim chance that the guest on another | |
702 | * CPU just brought in obsolete translation into the TLB of | |
703 | * this CPU due to a bad prefetch using the guest PID on | |
704 | * the way into the hypervisor. | |
705 | * | |
706 | * We work around this here. If KVM is possible, we check if | |
707 | * any sibling thread is in KVM. If it is, the window may exist | |
708 | * and thus we flush that PID from the core. | |
709 | * | |
710 | * A potential future improvement would be to mark which PIDs | |
711 | * have never been used on the system and avoid it if the PID | |
712 | * is new and the process has no other cpumask bit set. | |
713 | */ | |
714 | if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { | |
715 | int cpu = smp_processor_id(); | |
716 | int sib = cpu_first_thread_sibling(cpu); | |
717 | bool flush = false; | |
718 | ||
719 | for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { | |
720 | if (sib == cpu) | |
721 | continue; | |
722 | if (paca[sib].kvm_hstate.kvm_vcpu) | |
723 | flush = true; | |
724 | } | |
725 | if (flush) | |
726 | _tlbiel_pid(pid, RIC_FLUSH_ALL); | |
727 | } | |
728 | } | |
729 | EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); | |
730 | #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ |