Commit | Line | Data |
---|---|---|
ac41aaee | 1 | // SPDX-License-Identifier: GPL-2.0 |
3610cce8 | 2 | /* |
a53c8fab | 3 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
4 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
5 | */ | |
6 | ||
7 | #include <linux/sched.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/errno.h> | |
5a0e3ad6 | 10 | #include <linux/gfp.h> |
3610cce8 MS |
11 | #include <linux/mm.h> |
12 | #include <linux/swap.h> | |
13 | #include <linux/smp.h> | |
3610cce8 | 14 | #include <linux/spinlock.h> |
80217147 | 15 | #include <linux/rcupdate.h> |
e5992f2e | 16 | #include <linux/slab.h> |
b31288fa | 17 | #include <linux/swapops.h> |
0b46e0a3 | 18 | #include <linux/sysctl.h> |
3ac8e380 DD |
19 | #include <linux/ksm.h> |
20 | #include <linux/mman.h> | |
3610cce8 | 21 | |
3610cce8 MS |
22 | #include <asm/pgtable.h> |
23 | #include <asm/pgalloc.h> | |
24 | #include <asm/tlb.h> | |
25 | #include <asm/tlbflush.h> | |
6252d702 | 26 | #include <asm/mmu_context.h> |
2d42f947 | 27 | #include <asm/page-states.h> |
3610cce8 | 28 | |
118bd31b | 29 | static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr, |
cd774b90 | 30 | pte_t *ptep, int nodat) |
118bd31b MS |
31 | { |
32 | unsigned long opt, asce; | |
33 | ||
34 | if (MACHINE_HAS_TLB_GUEST) { | |
35 | opt = 0; | |
36 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 37 | if (asce == 0UL || nodat) |
118bd31b | 38 | opt |= IPTE_NODAT; |
28c807e5 MS |
39 | if (asce != -1UL) { |
40 | asce = asce ? : mm->context.asce; | |
41 | opt |= IPTE_GUEST_ASCE; | |
42 | } | |
43 | __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL); | |
118bd31b | 44 | } else { |
28c807e5 | 45 | __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL); |
118bd31b MS |
46 | } |
47 | } | |
48 | ||
49 | static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr, | |
cd774b90 | 50 | pte_t *ptep, int nodat) |
118bd31b MS |
51 | { |
52 | unsigned long opt, asce; | |
53 | ||
54 | if (MACHINE_HAS_TLB_GUEST) { | |
55 | opt = 0; | |
56 | asce = READ_ONCE(mm->context.gmap_asce); | |
cd774b90 | 57 | if (asce == 0UL || nodat) |
118bd31b | 58 | opt |= IPTE_NODAT; |
28c807e5 MS |
59 | if (asce != -1UL) { |
60 | asce = asce ? : mm->context.asce; | |
61 | opt |= IPTE_GUEST_ASCE; | |
62 | } | |
63 | __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL); | |
118bd31b | 64 | } else { |
28c807e5 | 65 | __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); |
118bd31b MS |
66 | } |
67 | } | |
68 | ||
ebde765c | 69 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
cd774b90 MS |
70 | unsigned long addr, pte_t *ptep, |
71 | int nodat) | |
ebde765c | 72 | { |
ebde765c MS |
73 | pte_t old; |
74 | ||
75 | old = *ptep; | |
76 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
77 | return old; | |
64f31d58 MS |
78 | atomic_inc(&mm->context.flush_count); |
79 | if (MACHINE_HAS_TLB_LC && | |
ebde765c | 80 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
cd774b90 | 81 | ptep_ipte_local(mm, addr, ptep, nodat); |
ebde765c | 82 | else |
cd774b90 | 83 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 84 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
85 | return old; |
86 | } | |
87 | ||
88 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
cd774b90 MS |
89 | unsigned long addr, pte_t *ptep, |
90 | int nodat) | |
ebde765c | 91 | { |
ebde765c MS |
92 | pte_t old; |
93 | ||
94 | old = *ptep; | |
95 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
96 | return old; | |
64f31d58 MS |
97 | atomic_inc(&mm->context.flush_count); |
98 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
99 | cpumask_of(smp_processor_id()))) { | |
ebde765c MS |
100 | pte_val(*ptep) |= _PAGE_INVALID; |
101 | mm->context.flush_mm = 1; | |
102 | } else | |
cd774b90 | 103 | ptep_ipte_global(mm, addr, ptep, nodat); |
64f31d58 | 104 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
105 | return old; |
106 | } | |
107 | ||
1e133ab2 MS |
108 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
109 | { | |
110 | unsigned long new = 0; | |
111 | #ifdef CONFIG_PGSTE | |
112 | unsigned long old; | |
113 | ||
1e133ab2 MS |
114 | asm( |
115 | " lg %0,%2\n" | |
116 | "0: lgr %1,%0\n" | |
117 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
118 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
119 | " csg %0,%1,%2\n" | |
120 | " jl 0b\n" | |
121 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
122 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
123 | #endif | |
124 | return __pgste(new); | |
125 | } | |
126 | ||
127 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
128 | { | |
129 | #ifdef CONFIG_PGSTE | |
130 | asm( | |
131 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
132 | " stg %1,%0\n" | |
133 | : "=Q" (ptep[PTRS_PER_PTE]) | |
134 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
135 | : "cc", "memory"); | |
1e133ab2 MS |
136 | #endif |
137 | } | |
138 | ||
139 | static inline pgste_t pgste_get(pte_t *ptep) | |
140 | { | |
141 | unsigned long pgste = 0; | |
142 | #ifdef CONFIG_PGSTE | |
143 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
144 | #endif | |
145 | return __pgste(pgste); | |
146 | } | |
147 | ||
148 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
149 | { | |
150 | #ifdef CONFIG_PGSTE | |
151 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
152 | #endif | |
153 | } | |
154 | ||
ebde765c MS |
155 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
156 | struct mm_struct *mm) | |
157 | { | |
158 | #ifdef CONFIG_PGSTE | |
159 | unsigned long address, bits, skey; | |
160 | ||
55531b74 | 161 | if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID) |
ebde765c MS |
162 | return pgste; |
163 | address = pte_val(pte) & PAGE_MASK; | |
164 | skey = (unsigned long) page_get_storage_key(address); | |
165 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
166 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
167 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
168 | /* Copy page access key and fetch protection bit to pgste */ | |
169 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
170 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
171 | #endif | |
172 | return pgste; | |
173 | ||
174 | } | |
175 | ||
176 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
177 | struct mm_struct *mm) | |
178 | { | |
179 | #ifdef CONFIG_PGSTE | |
180 | unsigned long address; | |
181 | unsigned long nkey; | |
182 | ||
55531b74 | 183 | if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID) |
ebde765c MS |
184 | return; |
185 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
186 | address = pte_val(entry) & PAGE_MASK; | |
187 | /* | |
188 | * Set page access key and fetch protection bit from pgste. | |
189 | * The guest C/R information is still in the PGSTE, set real | |
190 | * key C/R to 0. | |
191 | */ | |
192 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
193 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
194 | page_set_storage_key(address, nkey, 0); | |
195 | #endif | |
196 | } | |
197 | ||
198 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
199 | { | |
200 | #ifdef CONFIG_PGSTE | |
201 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
202 | (pte_val(entry) & _PAGE_WRITE) && | |
203 | !(pte_val(entry) & _PAGE_INVALID)) { | |
204 | if (!MACHINE_HAS_ESOP) { | |
205 | /* | |
206 | * Without enhanced suppression-on-protection force | |
207 | * the dirty bit on for all writable ptes. | |
208 | */ | |
209 | pte_val(entry) |= _PAGE_DIRTY; | |
210 | pte_val(entry) &= ~_PAGE_PROTECT; | |
211 | } | |
212 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
213 | /* This pte allows write access, set user-dirty */ | |
214 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
215 | } | |
216 | #endif | |
217 | *ptep = entry; | |
218 | return pgste; | |
219 | } | |
220 | ||
b2d73b2a MS |
221 | static inline pgste_t pgste_pte_notify(struct mm_struct *mm, |
222 | unsigned long addr, | |
223 | pte_t *ptep, pgste_t pgste) | |
ebde765c MS |
224 | { |
225 | #ifdef CONFIG_PGSTE | |
4be130a0 MS |
226 | unsigned long bits; |
227 | ||
228 | bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); | |
229 | if (bits) { | |
230 | pgste_val(pgste) ^= bits; | |
231 | ptep_notify(mm, addr, ptep, bits); | |
ebde765c MS |
232 | } |
233 | #endif | |
234 | return pgste; | |
235 | } | |
236 | ||
ebde765c MS |
237 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
238 | unsigned long addr, pte_t *ptep) | |
239 | { | |
240 | pgste_t pgste = __pgste(0); | |
241 | ||
242 | if (mm_has_pgste(mm)) { | |
243 | pgste = pgste_get_lock(ptep); | |
b2d73b2a | 244 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
ebde765c MS |
245 | } |
246 | return pgste; | |
247 | } | |
248 | ||
0d6da872 | 249 | static inline pte_t ptep_xchg_commit(struct mm_struct *mm, |
ebde765c MS |
250 | unsigned long addr, pte_t *ptep, |
251 | pgste_t pgste, pte_t old, pte_t new) | |
252 | { | |
253 | if (mm_has_pgste(mm)) { | |
254 | if (pte_val(old) & _PAGE_INVALID) | |
255 | pgste_set_key(ptep, pgste, new, mm); | |
256 | if (pte_val(new) & _PAGE_INVALID) { | |
257 | pgste = pgste_update_all(old, pgste, mm); | |
258 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
259 | _PGSTE_GPS_USAGE_UNUSED) | |
260 | pte_val(old) |= _PAGE_UNUSED; | |
261 | } | |
262 | pgste = pgste_set_pte(ptep, pgste, new); | |
263 | pgste_set_unlock(ptep, pgste); | |
264 | } else { | |
265 | *ptep = new; | |
266 | } | |
0d6da872 | 267 | return old; |
ebde765c MS |
268 | } |
269 | ||
270 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
271 | pte_t *ptep, pte_t new) | |
272 | { | |
273 | pgste_t pgste; | |
274 | pte_t old; | |
cd774b90 | 275 | int nodat; |
ebde765c | 276 | |
a9809407 | 277 | preempt_disable(); |
ebde765c | 278 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
279 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
280 | old = ptep_flush_direct(mm, addr, ptep, nodat); | |
0d6da872 | 281 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 282 | preempt_enable(); |
ebde765c MS |
283 | return old; |
284 | } | |
285 | EXPORT_SYMBOL(ptep_xchg_direct); | |
286 | ||
287 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
288 | pte_t *ptep, pte_t new) | |
289 | { | |
290 | pgste_t pgste; | |
291 | pte_t old; | |
cd774b90 | 292 | int nodat; |
ebde765c | 293 | |
a9809407 | 294 | preempt_disable(); |
ebde765c | 295 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
296 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
297 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
0d6da872 | 298 | old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new); |
a9809407 | 299 | preempt_enable(); |
ebde765c MS |
300 | return old; |
301 | } | |
302 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
303 | ||
0cbe3e26 | 304 | pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, |
ebde765c MS |
305 | pte_t *ptep) |
306 | { | |
307 | pgste_t pgste; | |
308 | pte_t old; | |
cd774b90 | 309 | int nodat; |
0cbe3e26 | 310 | struct mm_struct *mm = vma->vm_mm; |
ebde765c | 311 | |
a9809407 | 312 | preempt_disable(); |
ebde765c | 313 | pgste = ptep_xchg_start(mm, addr, ptep); |
cd774b90 MS |
314 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
315 | old = ptep_flush_lazy(mm, addr, ptep, nodat); | |
ebde765c MS |
316 | if (mm_has_pgste(mm)) { |
317 | pgste = pgste_update_all(old, pgste, mm); | |
318 | pgste_set(ptep, pgste); | |
319 | } | |
320 | return old; | |
321 | } | |
ebde765c | 322 | |
0cbe3e26 | 323 | void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, |
04a86453 | 324 | pte_t *ptep, pte_t old_pte, pte_t pte) |
ebde765c MS |
325 | { |
326 | pgste_t pgste; | |
0cbe3e26 | 327 | struct mm_struct *mm = vma->vm_mm; |
ebde765c | 328 | |
57d7f939 MS |
329 | if (!MACHINE_HAS_NX) |
330 | pte_val(pte) &= ~_PAGE_NOEXEC; | |
ebde765c MS |
331 | if (mm_has_pgste(mm)) { |
332 | pgste = pgste_get(ptep); | |
333 | pgste_set_key(ptep, pgste, pte, mm); | |
334 | pgste = pgste_set_pte(ptep, pgste, pte); | |
335 | pgste_set_unlock(ptep, pgste); | |
336 | } else { | |
337 | *ptep = pte; | |
338 | } | |
a9809407 | 339 | preempt_enable(); |
ebde765c | 340 | } |
ebde765c | 341 | |
118bd31b MS |
342 | static inline void pmdp_idte_local(struct mm_struct *mm, |
343 | unsigned long addr, pmd_t *pmdp) | |
344 | { | |
345 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
346 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
347 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 348 | else |
28c807e5 | 349 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL); |
a9e00d83 | 350 | if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) |
6a376277 | 351 | gmap_pmdp_idte_local(mm, addr); |
118bd31b MS |
352 | } |
353 | ||
354 | static inline void pmdp_idte_global(struct mm_struct *mm, | |
355 | unsigned long addr, pmd_t *pmdp) | |
356 | { | |
6a376277 | 357 | if (MACHINE_HAS_TLB_GUEST) { |
28c807e5 MS |
358 | __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, |
359 | mm->context.asce, IDTE_GLOBAL); | |
a9e00d83 | 360 | if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) |
6a376277 JF |
361 | gmap_pmdp_idte_global(mm, addr); |
362 | } else if (MACHINE_HAS_IDTE) { | |
28c807e5 | 363 | __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL); |
a9e00d83 | 364 | if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) |
6a376277 JF |
365 | gmap_pmdp_idte_global(mm, addr); |
366 | } else { | |
118bd31b | 367 | __pmdp_csp(pmdp); |
a9e00d83 | 368 | if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m) |
6a376277 JF |
369 | gmap_pmdp_csp(mm, addr); |
370 | } | |
118bd31b MS |
371 | } |
372 | ||
227be799 MS |
373 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
374 | unsigned long addr, pmd_t *pmdp) | |
375 | { | |
227be799 MS |
376 | pmd_t old; |
377 | ||
378 | old = *pmdp; | |
379 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
380 | return old; | |
64f31d58 MS |
381 | atomic_inc(&mm->context.flush_count); |
382 | if (MACHINE_HAS_TLB_LC && | |
227be799 | 383 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
118bd31b | 384 | pmdp_idte_local(mm, addr, pmdp); |
227be799 | 385 | else |
118bd31b | 386 | pmdp_idte_global(mm, addr, pmdp); |
64f31d58 | 387 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
388 | return old; |
389 | } | |
390 | ||
391 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
392 | unsigned long addr, pmd_t *pmdp) | |
393 | { | |
227be799 MS |
394 | pmd_t old; |
395 | ||
396 | old = *pmdp; | |
397 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
398 | return old; | |
64f31d58 MS |
399 | atomic_inc(&mm->context.flush_count); |
400 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
401 | cpumask_of(smp_processor_id()))) { | |
227be799 MS |
402 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; |
403 | mm->context.flush_mm = 1; | |
6a376277 JF |
404 | if (mm_has_pgste(mm)) |
405 | gmap_pmdp_invalidate(mm, addr); | |
118bd31b MS |
406 | } else { |
407 | pmdp_idte_global(mm, addr, pmdp); | |
408 | } | |
64f31d58 | 409 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
410 | return old; |
411 | } | |
412 | ||
81a8f2be | 413 | #ifdef CONFIG_PGSTE |
637ff9ef JF |
414 | static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr) |
415 | { | |
416 | pgd_t *pgd; | |
417 | p4d_t *p4d; | |
418 | pud_t *pud; | |
419 | pmd_t *pmd; | |
420 | ||
421 | pgd = pgd_offset(mm, addr); | |
422 | p4d = p4d_alloc(mm, pgd, addr); | |
423 | if (!p4d) | |
424 | return NULL; | |
425 | pud = pud_alloc(mm, p4d, addr); | |
426 | if (!pud) | |
427 | return NULL; | |
428 | pmd = pmd_alloc(mm, pud, addr); | |
429 | return pmd; | |
430 | } | |
81a8f2be | 431 | #endif |
637ff9ef | 432 | |
227be799 MS |
433 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, |
434 | pmd_t *pmdp, pmd_t new) | |
435 | { | |
436 | pmd_t old; | |
437 | ||
a9809407 | 438 | preempt_disable(); |
227be799 MS |
439 | old = pmdp_flush_direct(mm, addr, pmdp); |
440 | *pmdp = new; | |
a9809407 | 441 | preempt_enable(); |
227be799 MS |
442 | return old; |
443 | } | |
444 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
445 | ||
446 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
447 | pmd_t *pmdp, pmd_t new) | |
448 | { | |
449 | pmd_t old; | |
450 | ||
a9809407 | 451 | preempt_disable(); |
227be799 MS |
452 | old = pmdp_flush_lazy(mm, addr, pmdp); |
453 | *pmdp = new; | |
a9809407 | 454 | preempt_enable(); |
227be799 MS |
455 | return old; |
456 | } | |
457 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
458 | ||
118bd31b MS |
459 | static inline void pudp_idte_local(struct mm_struct *mm, |
460 | unsigned long addr, pud_t *pudp) | |
d08de8e2 | 461 | { |
118bd31b | 462 | if (MACHINE_HAS_TLB_GUEST) |
28c807e5 MS |
463 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
464 | mm->context.asce, IDTE_LOCAL); | |
118bd31b | 465 | else |
28c807e5 | 466 | __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL); |
118bd31b | 467 | } |
d08de8e2 | 468 | |
118bd31b MS |
469 | static inline void pudp_idte_global(struct mm_struct *mm, |
470 | unsigned long addr, pud_t *pudp) | |
471 | { | |
472 | if (MACHINE_HAS_TLB_GUEST) | |
28c807e5 MS |
473 | __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE, |
474 | mm->context.asce, IDTE_GLOBAL); | |
118bd31b | 475 | else if (MACHINE_HAS_IDTE) |
28c807e5 | 476 | __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL); |
118bd31b | 477 | else |
d08de8e2 GS |
478 | /* |
479 | * Invalid bit position is the same for pmd and pud, so we can | |
480 | * re-use _pmd_csp() here | |
481 | */ | |
482 | __pmdp_csp((pmd_t *) pudp); | |
118bd31b MS |
483 | } |
484 | ||
485 | static inline pud_t pudp_flush_direct(struct mm_struct *mm, | |
486 | unsigned long addr, pud_t *pudp) | |
487 | { | |
488 | pud_t old; | |
489 | ||
490 | old = *pudp; | |
491 | if (pud_val(old) & _REGION_ENTRY_INVALID) | |
d08de8e2 | 492 | return old; |
d08de8e2 GS |
493 | atomic_inc(&mm->context.flush_count); |
494 | if (MACHINE_HAS_TLB_LC && | |
495 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
118bd31b | 496 | pudp_idte_local(mm, addr, pudp); |
d08de8e2 | 497 | else |
118bd31b | 498 | pudp_idte_global(mm, addr, pudp); |
d08de8e2 GS |
499 | atomic_dec(&mm->context.flush_count); |
500 | return old; | |
501 | } | |
502 | ||
503 | pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
504 | pud_t *pudp, pud_t new) | |
505 | { | |
506 | pud_t old; | |
507 | ||
508 | preempt_disable(); | |
509 | old = pudp_flush_direct(mm, addr, pudp); | |
510 | *pudp = new; | |
511 | preempt_enable(); | |
512 | return old; | |
513 | } | |
514 | EXPORT_SYMBOL(pudp_xchg_direct); | |
515 | ||
75077afb | 516 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
517 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
518 | pgtable_t pgtable) | |
9501d09f GS |
519 | { |
520 | struct list_head *lh = (struct list_head *) pgtable; | |
521 | ||
ec66ad66 | 522 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
523 | |
524 | /* FIFO */ | |
c389a250 | 525 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
526 | INIT_LIST_HEAD(lh); |
527 | else | |
c389a250 KS |
528 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
529 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
530 | } |
531 | ||
6b0b50b0 | 532 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
533 | { |
534 | struct list_head *lh; | |
535 | pgtable_t pgtable; | |
536 | pte_t *ptep; | |
537 | ||
ec66ad66 | 538 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
539 | |
540 | /* FIFO */ | |
c389a250 | 541 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
542 | lh = (struct list_head *) pgtable; |
543 | if (list_empty(lh)) | |
c389a250 | 544 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 545 | else { |
c389a250 | 546 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
547 | list_del(lh); |
548 | } | |
549 | ptep = (pte_t *) pgtable; | |
e5098611 | 550 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 551 | ptep++; |
e5098611 | 552 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
553 | return pgtable; |
554 | } | |
75077afb | 555 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
556 | |
557 | #ifdef CONFIG_PGSTE | |
558 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
559 | pte_t *ptep, pte_t entry) | |
560 | { | |
561 | pgste_t pgste; | |
562 | ||
563 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
a9809407 | 564 | preempt_disable(); |
1e133ab2 MS |
565 | pgste = pgste_get_lock(ptep); |
566 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
567 | pgste_set_key(ptep, pgste, entry, mm); | |
568 | pgste = pgste_set_pte(ptep, pgste, entry); | |
569 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 570 | preempt_enable(); |
1e133ab2 MS |
571 | } |
572 | ||
573 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
574 | { | |
575 | pgste_t pgste; | |
576 | ||
a9809407 | 577 | preempt_disable(); |
1e133ab2 MS |
578 | pgste = pgste_get_lock(ptep); |
579 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
580 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 581 | preempt_enable(); |
1e133ab2 MS |
582 | } |
583 | ||
b2d73b2a MS |
584 | /** |
585 | * ptep_force_prot - change access rights of a locked pte | |
586 | * @mm: pointer to the process mm_struct | |
587 | * @addr: virtual address in the guest address space | |
588 | * @ptep: pointer to the page table entry | |
589 | * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE | |
4be130a0 | 590 | * @bit: pgste bit to set (e.g. for notification) |
b2d73b2a MS |
591 | * |
592 | * Returns 0 if the access rights were changed and -EAGAIN if the current | |
593 | * and requested access rights are incompatible. | |
594 | */ | |
595 | int ptep_force_prot(struct mm_struct *mm, unsigned long addr, | |
4be130a0 | 596 | pte_t *ptep, int prot, unsigned long bit) |
b2d73b2a MS |
597 | { |
598 | pte_t entry; | |
599 | pgste_t pgste; | |
cd774b90 | 600 | int pte_i, pte_p, nodat; |
b2d73b2a MS |
601 | |
602 | pgste = pgste_get_lock(ptep); | |
603 | entry = *ptep; | |
604 | /* Check pte entry after all locks have been acquired */ | |
605 | pte_i = pte_val(entry) & _PAGE_INVALID; | |
606 | pte_p = pte_val(entry) & _PAGE_PROTECT; | |
607 | if ((pte_i && (prot != PROT_NONE)) || | |
608 | (pte_p && (prot & PROT_WRITE))) { | |
609 | pgste_set_unlock(ptep, pgste); | |
610 | return -EAGAIN; | |
611 | } | |
4be130a0 | 612 | /* Change access rights and set pgste bit */ |
cd774b90 | 613 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
b2d73b2a | 614 | if (prot == PROT_NONE && !pte_i) { |
cd774b90 | 615 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
616 | pgste = pgste_update_all(entry, pgste, mm); |
617 | pte_val(entry) |= _PAGE_INVALID; | |
618 | } | |
619 | if (prot == PROT_READ && !pte_p) { | |
cd774b90 | 620 | ptep_flush_direct(mm, addr, ptep, nodat); |
b2d73b2a MS |
621 | pte_val(entry) &= ~_PAGE_INVALID; |
622 | pte_val(entry) |= _PAGE_PROTECT; | |
623 | } | |
4be130a0 | 624 | pgste_val(pgste) |= bit; |
b2d73b2a MS |
625 | pgste = pgste_set_pte(ptep, pgste, entry); |
626 | pgste_set_unlock(ptep, pgste); | |
627 | return 0; | |
628 | } | |
629 | ||
4be130a0 | 630 | int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, |
a9d23e71 | 631 | pte_t *sptep, pte_t *tptep, pte_t pte) |
4be130a0 MS |
632 | { |
633 | pgste_t spgste, tpgste; | |
634 | pte_t spte, tpte; | |
635 | int rc = -EAGAIN; | |
636 | ||
a9d23e71 DH |
637 | if (!(pte_val(*tptep) & _PAGE_INVALID)) |
638 | return 0; /* already shadowed */ | |
4be130a0 MS |
639 | spgste = pgste_get_lock(sptep); |
640 | spte = *sptep; | |
641 | if (!(pte_val(spte) & _PAGE_INVALID) && | |
a9d23e71 DH |
642 | !((pte_val(spte) & _PAGE_PROTECT) && |
643 | !(pte_val(pte) & _PAGE_PROTECT))) { | |
4be130a0 MS |
644 | pgste_val(spgste) |= PGSTE_VSIE_BIT; |
645 | tpgste = pgste_get_lock(tptep); | |
646 | pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | | |
a9d23e71 | 647 | (pte_val(pte) & _PAGE_PROTECT); |
4be130a0 MS |
648 | /* don't touch the storage key - it belongs to parent pgste */ |
649 | tpgste = pgste_set_pte(tptep, tpgste, tpte); | |
650 | pgste_set_unlock(tptep, tpgste); | |
a9d23e71 | 651 | rc = 1; |
4be130a0 MS |
652 | } |
653 | pgste_set_unlock(sptep, spgste); | |
654 | return rc; | |
655 | } | |
656 | ||
657 | void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) | |
658 | { | |
659 | pgste_t pgste; | |
cd774b90 | 660 | int nodat; |
4be130a0 MS |
661 | |
662 | pgste = pgste_get_lock(ptep); | |
663 | /* notifier is called by the caller */ | |
cd774b90 MS |
664 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
665 | ptep_flush_direct(mm, saddr, ptep, nodat); | |
4be130a0 MS |
666 | /* don't touch the storage key - it belongs to parent pgste */ |
667 | pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); | |
668 | pgste_set_unlock(ptep, pgste); | |
669 | } | |
670 | ||
1e133ab2 MS |
671 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) |
672 | { | |
673 | if (!non_swap_entry(entry)) | |
674 | dec_mm_counter(mm, MM_SWAPENTS); | |
675 | else if (is_migration_entry(entry)) { | |
676 | struct page *page = migration_entry_to_page(entry); | |
677 | ||
678 | dec_mm_counter(mm, mm_counter(page)); | |
679 | } | |
680 | free_swap_and_cache(entry); | |
681 | } | |
682 | ||
683 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
684 | pte_t *ptep, int reset) | |
685 | { | |
686 | unsigned long pgstev; | |
687 | pgste_t pgste; | |
688 | pte_t pte; | |
689 | ||
690 | /* Zap unused and logically-zero pages */ | |
a9809407 | 691 | preempt_disable(); |
1e133ab2 MS |
692 | pgste = pgste_get_lock(ptep); |
693 | pgstev = pgste_val(pgste); | |
694 | pte = *ptep; | |
1c343f7b | 695 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
696 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
697 | (pgstev & _PGSTE_GPS_ZERO))) { | |
698 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
699 | pte_clear(mm, addr, ptep); | |
700 | } | |
701 | if (reset) | |
702 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
703 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 704 | preempt_enable(); |
1e133ab2 MS |
705 | } |
706 | ||
707 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
708 | { | |
709 | unsigned long ptev; | |
710 | pgste_t pgste; | |
711 | ||
97ca7bfc | 712 | /* Clear storage key ACC and F, but set R/C */ |
a9809407 | 713 | preempt_disable(); |
1e133ab2 | 714 | pgste = pgste_get_lock(ptep); |
97ca7bfc CB |
715 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); |
716 | pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT; | |
1e133ab2 MS |
717 | ptev = pte_val(*ptep); |
718 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
719 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
720 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 721 | preempt_enable(); |
1e133ab2 MS |
722 | } |
723 | ||
724 | /* | |
725 | * Test and reset if a guest page is dirty | |
726 | */ | |
0959e168 JF |
727 | bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr, |
728 | pte_t *ptep) | |
1e133ab2 | 729 | { |
1e133ab2 | 730 | pgste_t pgste; |
1e133ab2 MS |
731 | pte_t pte; |
732 | bool dirty; | |
cd774b90 | 733 | int nodat; |
1e133ab2 | 734 | |
1e133ab2 MS |
735 | pgste = pgste_get_lock(ptep); |
736 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
737 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
738 | pte = *ptep; | |
739 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
b2d73b2a | 740 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
cd774b90 MS |
741 | nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT); |
742 | ptep_ipte_global(mm, addr, ptep, nodat); | |
1e133ab2 MS |
743 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) |
744 | pte_val(pte) |= _PAGE_PROTECT; | |
745 | else | |
746 | pte_val(pte) |= _PAGE_INVALID; | |
747 | *ptep = pte; | |
748 | } | |
749 | pgste_set_unlock(ptep, pgste); | |
1e133ab2 MS |
750 | return dirty; |
751 | } | |
0959e168 | 752 | EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc); |
1e133ab2 MS |
753 | |
754 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
755 | unsigned char key, bool nq) | |
756 | { | |
637ff9ef | 757 | unsigned long keyul, paddr; |
1e133ab2 MS |
758 | spinlock_t *ptl; |
759 | pgste_t old, new; | |
637ff9ef | 760 | pmd_t *pmdp; |
1e133ab2 MS |
761 | pte_t *ptep; |
762 | ||
637ff9ef JF |
763 | pmdp = pmd_alloc_map(mm, addr); |
764 | if (unlikely(!pmdp)) | |
765 | return -EFAULT; | |
766 | ||
767 | ptl = pmd_lock(mm, pmdp); | |
768 | if (!pmd_present(*pmdp)) { | |
769 | spin_unlock(ptl); | |
770 | return -EFAULT; | |
771 | } | |
772 | ||
773 | if (pmd_large(*pmdp)) { | |
774 | paddr = pmd_val(*pmdp) & HPAGE_MASK; | |
775 | paddr |= addr & ~HPAGE_MASK; | |
776 | /* | |
777 | * Huge pmds need quiescing operations, they are | |
778 | * always mapped. | |
779 | */ | |
780 | page_set_storage_key(paddr, key, 1); | |
781 | spin_unlock(ptl); | |
782 | return 0; | |
783 | } | |
784 | spin_unlock(ptl); | |
785 | ||
786 | ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); | |
d3ed1cee | 787 | if (unlikely(!ptep)) |
1e133ab2 | 788 | return -EFAULT; |
1e133ab2 MS |
789 | |
790 | new = old = pgste_get_lock(ptep); | |
791 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
792 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
793 | keyul = (unsigned long) key; | |
794 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
795 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
796 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
637ff9ef | 797 | unsigned long bits, skey; |
1e133ab2 | 798 | |
637ff9ef JF |
799 | paddr = pte_val(*ptep) & PAGE_MASK; |
800 | skey = (unsigned long) page_get_storage_key(paddr); | |
1e133ab2 MS |
801 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); |
802 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
803 | /* Set storage key ACC and FP */ | |
637ff9ef | 804 | page_set_storage_key(paddr, skey, !nq); |
1e133ab2 MS |
805 | /* Merge host changed & referenced into pgste */ |
806 | pgste_val(new) |= bits << 52; | |
807 | } | |
808 | /* changing the guest storage key is considered a change of the page */ | |
809 | if ((pgste_val(new) ^ pgste_val(old)) & | |
810 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
811 | pgste_val(new) |= PGSTE_UC_BIT; | |
812 | ||
813 | pgste_set_unlock(ptep, new); | |
814 | pte_unmap_unlock(ptep, ptl); | |
1e133ab2 MS |
815 | return 0; |
816 | } | |
817 | EXPORT_SYMBOL(set_guest_storage_key); | |
818 | ||
1824c723 DH |
819 | /** |
820 | * Conditionally set a guest storage key (handling csske). | |
821 | * oldkey will be updated when either mr or mc is set and a pointer is given. | |
822 | * | |
823 | * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest | |
824 | * storage key was updated and -EFAULT on access errors. | |
825 | */ | |
826 | int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
827 | unsigned char key, unsigned char *oldkey, | |
828 | bool nq, bool mr, bool mc) | |
829 | { | |
830 | unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; | |
831 | int rc; | |
832 | ||
833 | /* we can drop the pgste lock between getting and setting the key */ | |
834 | if (mr | mc) { | |
835 | rc = get_guest_storage_key(current->mm, addr, &tmp); | |
836 | if (rc) | |
837 | return rc; | |
838 | if (oldkey) | |
839 | *oldkey = tmp; | |
840 | if (!mr) | |
841 | mask |= _PAGE_REFERENCED; | |
842 | if (!mc) | |
843 | mask |= _PAGE_CHANGED; | |
844 | if (!((tmp ^ key) & mask)) | |
845 | return 0; | |
846 | } | |
847 | rc = set_guest_storage_key(current->mm, addr, key, nq); | |
848 | return rc < 0 ? rc : 1; | |
849 | } | |
850 | EXPORT_SYMBOL(cond_set_guest_storage_key); | |
851 | ||
a7e19ab5 DH |
852 | /** |
853 | * Reset a guest reference bit (rrbe), returning the reference and changed bit. | |
854 | * | |
855 | * Returns < 0 in case of error, otherwise the cc to be reported to the guest. | |
856 | */ | |
857 | int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) | |
1e133ab2 | 858 | { |
1e133ab2 | 859 | spinlock_t *ptl; |
637ff9ef | 860 | unsigned long paddr; |
a7e19ab5 | 861 | pgste_t old, new; |
637ff9ef | 862 | pmd_t *pmdp; |
1e133ab2 | 863 | pte_t *ptep; |
a7e19ab5 | 864 | int cc = 0; |
1e133ab2 | 865 | |
637ff9ef JF |
866 | pmdp = pmd_alloc_map(mm, addr); |
867 | if (unlikely(!pmdp)) | |
868 | return -EFAULT; | |
869 | ||
870 | ptl = pmd_lock(mm, pmdp); | |
871 | if (!pmd_present(*pmdp)) { | |
872 | spin_unlock(ptl); | |
873 | return -EFAULT; | |
874 | } | |
875 | ||
876 | if (pmd_large(*pmdp)) { | |
877 | paddr = pmd_val(*pmdp) & HPAGE_MASK; | |
878 | paddr |= addr & ~HPAGE_MASK; | |
879 | cc = page_reset_referenced(paddr); | |
880 | spin_unlock(ptl); | |
881 | return cc; | |
882 | } | |
883 | spin_unlock(ptl); | |
884 | ||
885 | ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); | |
a7e19ab5 | 886 | if (unlikely(!ptep)) |
1e133ab2 | 887 | return -EFAULT; |
1e133ab2 | 888 | |
a7e19ab5 DH |
889 | new = old = pgste_get_lock(ptep); |
890 | /* Reset guest reference bit only */ | |
891 | pgste_val(new) &= ~PGSTE_GR_BIT; | |
1e133ab2 | 892 | |
a7e19ab5 | 893 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { |
637ff9ef JF |
894 | paddr = pte_val(*ptep) & PAGE_MASK; |
895 | cc = page_reset_referenced(paddr); | |
a7e19ab5 DH |
896 | /* Merge real referenced bit into host-set */ |
897 | pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; | |
1e133ab2 | 898 | } |
a7e19ab5 DH |
899 | /* Reflect guest's logical view, not physical */ |
900 | cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; | |
901 | /* Changing the guest storage key is considered a change of the page */ | |
902 | if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) | |
903 | pgste_val(new) |= PGSTE_UC_BIT; | |
904 | ||
905 | pgste_set_unlock(ptep, new); | |
906 | pte_unmap_unlock(ptep, ptl); | |
4bead2a4 | 907 | return cc; |
a7e19ab5 DH |
908 | } |
909 | EXPORT_SYMBOL(reset_guest_reference_bit); | |
910 | ||
154c8c19 DH |
911 | int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, |
912 | unsigned char *key) | |
1e133ab2 | 913 | { |
637ff9ef | 914 | unsigned long paddr; |
1e133ab2 MS |
915 | spinlock_t *ptl; |
916 | pgste_t pgste; | |
637ff9ef | 917 | pmd_t *pmdp; |
1e133ab2 | 918 | pte_t *ptep; |
1e133ab2 | 919 | |
637ff9ef JF |
920 | pmdp = pmd_alloc_map(mm, addr); |
921 | if (unlikely(!pmdp)) | |
922 | return -EFAULT; | |
923 | ||
924 | ptl = pmd_lock(mm, pmdp); | |
925 | if (!pmd_present(*pmdp)) { | |
926 | /* Not yet mapped memory has a zero key */ | |
927 | spin_unlock(ptl); | |
928 | *key = 0; | |
929 | return 0; | |
930 | } | |
931 | ||
932 | if (pmd_large(*pmdp)) { | |
933 | paddr = pmd_val(*pmdp) & HPAGE_MASK; | |
934 | paddr |= addr & ~HPAGE_MASK; | |
935 | *key = page_get_storage_key(paddr); | |
936 | spin_unlock(ptl); | |
937 | return 0; | |
938 | } | |
939 | spin_unlock(ptl); | |
940 | ||
941 | ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl); | |
d3ed1cee | 942 | if (unlikely(!ptep)) |
1e133ab2 | 943 | return -EFAULT; |
1e133ab2 | 944 | |
d3ed1cee | 945 | pgste = pgste_get_lock(ptep); |
154c8c19 | 946 | *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; |
637ff9ef | 947 | paddr = pte_val(*ptep) & PAGE_MASK; |
8d6037a7 | 948 | if (!(pte_val(*ptep) & _PAGE_INVALID)) |
637ff9ef | 949 | *key = page_get_storage_key(paddr); |
8d6037a7 | 950 | /* Reflect guest's logical view, not physical */ |
154c8c19 | 951 | *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; |
1e133ab2 MS |
952 | pgste_set_unlock(ptep, pgste); |
953 | pte_unmap_unlock(ptep, ptl); | |
154c8c19 | 954 | return 0; |
1e133ab2 MS |
955 | } |
956 | EXPORT_SYMBOL(get_guest_storage_key); | |
2d42f947 CI |
957 | |
958 | /** | |
959 | * pgste_perform_essa - perform ESSA actions on the PGSTE. | |
960 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
961 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
962 | * @orc: the specific action to perform, see the ESSA_SET_* macros. | |
963 | * @oldpte: the PTE will be saved there if the pointer is not NULL. | |
964 | * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL. | |
965 | * | |
966 | * Return: 1 if the page is to be added to the CBRL, otherwise 0, | |
967 | * or < 0 in case of error. -EINVAL is returned for invalid values | |
968 | * of orc, -EFAULT for invalid addresses. | |
969 | */ | |
970 | int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc, | |
971 | unsigned long *oldpte, unsigned long *oldpgste) | |
972 | { | |
973 | unsigned long pgstev; | |
974 | spinlock_t *ptl; | |
975 | pgste_t pgste; | |
976 | pte_t *ptep; | |
977 | int res = 0; | |
978 | ||
979 | WARN_ON_ONCE(orc > ESSA_MAX); | |
980 | if (unlikely(orc > ESSA_MAX)) | |
981 | return -EINVAL; | |
982 | ptep = get_locked_pte(mm, hva, &ptl); | |
983 | if (unlikely(!ptep)) | |
984 | return -EFAULT; | |
985 | pgste = pgste_get_lock(ptep); | |
986 | pgstev = pgste_val(pgste); | |
987 | if (oldpte) | |
988 | *oldpte = pte_val(*ptep); | |
989 | if (oldpgste) | |
990 | *oldpgste = pgstev; | |
991 | ||
992 | switch (orc) { | |
993 | case ESSA_GET_STATE: | |
994 | break; | |
995 | case ESSA_SET_STABLE: | |
1bab1c02 | 996 | pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT); |
2d42f947 CI |
997 | pgstev |= _PGSTE_GPS_USAGE_STABLE; |
998 | break; | |
999 | case ESSA_SET_UNUSED: | |
1000 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1001 | pgstev |= _PGSTE_GPS_USAGE_UNUSED; | |
1002 | if (pte_val(*ptep) & _PAGE_INVALID) | |
1003 | res = 1; | |
1004 | break; | |
1005 | case ESSA_SET_VOLATILE: | |
1006 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1007 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
1008 | if (pte_val(*ptep) & _PAGE_INVALID) | |
1009 | res = 1; | |
1010 | break; | |
1011 | case ESSA_SET_POT_VOLATILE: | |
1012 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1013 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
1014 | pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE; | |
1015 | break; | |
1016 | } | |
1017 | if (pgstev & _PGSTE_GPS_ZERO) { | |
1018 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
1019 | break; | |
1020 | } | |
1021 | if (!(pgstev & PGSTE_GC_BIT)) { | |
1022 | pgstev |= _PGSTE_GPS_USAGE_VOLATILE; | |
1023 | res = 1; | |
1024 | break; | |
1025 | } | |
1026 | break; | |
1027 | case ESSA_SET_STABLE_RESIDENT: | |
1028 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1029 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
1030 | /* | |
1031 | * Since the resident state can go away any time after this | |
1032 | * call, we will not make this page resident. We can revisit | |
1033 | * this decision if a guest will ever start using this. | |
1034 | */ | |
1035 | break; | |
1036 | case ESSA_SET_STABLE_IF_RESIDENT: | |
1037 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
1038 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1039 | pgstev |= _PGSTE_GPS_USAGE_STABLE; | |
1040 | } | |
1041 | break; | |
1bab1c02 CI |
1042 | case ESSA_SET_STABLE_NODAT: |
1043 | pgstev &= ~_PGSTE_GPS_USAGE_MASK; | |
1044 | pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT; | |
1045 | break; | |
2d42f947 CI |
1046 | default: |
1047 | /* we should never get here! */ | |
1048 | break; | |
1049 | } | |
1050 | /* If we are discarding a page, set it to logical zero */ | |
1051 | if (res) | |
1052 | pgstev |= _PGSTE_GPS_ZERO; | |
1053 | ||
1054 | pgste_val(pgste) = pgstev; | |
1055 | pgste_set_unlock(ptep, pgste); | |
1056 | pte_unmap_unlock(ptep, ptl); | |
1057 | return res; | |
1058 | } | |
1059 | EXPORT_SYMBOL(pgste_perform_essa); | |
1060 | ||
1061 | /** | |
1062 | * set_pgste_bits - set specific PGSTE bits. | |
1063 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
1064 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
1065 | * @bits: a bitmask representing the bits that will be touched | |
1066 | * @value: the values of the bits to be written. Only the bits in the mask | |
1067 | * will be written. | |
1068 | * | |
1069 | * Return: 0 on success, < 0 in case of error. | |
1070 | */ | |
1071 | int set_pgste_bits(struct mm_struct *mm, unsigned long hva, | |
1072 | unsigned long bits, unsigned long value) | |
1073 | { | |
1074 | spinlock_t *ptl; | |
1075 | pgste_t new; | |
1076 | pte_t *ptep; | |
1077 | ||
1078 | ptep = get_locked_pte(mm, hva, &ptl); | |
1079 | if (unlikely(!ptep)) | |
1080 | return -EFAULT; | |
1081 | new = pgste_get_lock(ptep); | |
1082 | ||
1083 | pgste_val(new) &= ~bits; | |
1084 | pgste_val(new) |= value & bits; | |
1085 | ||
1086 | pgste_set_unlock(ptep, new); | |
1087 | pte_unmap_unlock(ptep, ptl); | |
1088 | return 0; | |
1089 | } | |
1090 | EXPORT_SYMBOL(set_pgste_bits); | |
1091 | ||
1092 | /** | |
1093 | * get_pgste - get the current PGSTE for the given address. | |
1094 | * @mm: the memory context. It must have PGSTEs, no check is performed here! | |
1095 | * @hva: the host virtual address of the page whose PGSTE is to be processed | |
1096 | * @pgstep: will be written with the current PGSTE for the given address. | |
1097 | * | |
1098 | * Return: 0 on success, < 0 in case of error. | |
1099 | */ | |
1100 | int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep) | |
1101 | { | |
1102 | spinlock_t *ptl; | |
1103 | pte_t *ptep; | |
1104 | ||
1105 | ptep = get_locked_pte(mm, hva, &ptl); | |
1106 | if (unlikely(!ptep)) | |
1107 | return -EFAULT; | |
1108 | *pgstep = pgste_val(pgste_get(ptep)); | |
1109 | pte_unmap_unlock(ptep, ptl); | |
1110 | return 0; | |
1111 | } | |
1112 | EXPORT_SYMBOL(get_pgste); | |
1e133ab2 | 1113 | #endif |