Commit | Line | Data |
---|---|---|
3610cce8 | 1 | /* |
a53c8fab | 2 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
4 | */ | |
5 | ||
6 | #include <linux/sched.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/errno.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
3610cce8 MS |
10 | #include <linux/mm.h> |
11 | #include <linux/swap.h> | |
12 | #include <linux/smp.h> | |
3610cce8 | 13 | #include <linux/spinlock.h> |
80217147 | 14 | #include <linux/rcupdate.h> |
e5992f2e | 15 | #include <linux/slab.h> |
b31288fa | 16 | #include <linux/swapops.h> |
0b46e0a3 | 17 | #include <linux/sysctl.h> |
3ac8e380 DD |
18 | #include <linux/ksm.h> |
19 | #include <linux/mman.h> | |
3610cce8 | 20 | |
3610cce8 MS |
21 | #include <asm/pgtable.h> |
22 | #include <asm/pgalloc.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/tlbflush.h> | |
6252d702 | 25 | #include <asm/mmu_context.h> |
3610cce8 | 26 | |
ebde765c MS |
27 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
28 | unsigned long addr, pte_t *ptep) | |
29 | { | |
30 | int active, count; | |
31 | pte_t old; | |
32 | ||
33 | old = *ptep; | |
34 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
35 | return old; | |
36 | active = (mm == current->active_mm) ? 1 : 0; | |
37 | count = atomic_add_return(0x10000, &mm->context.attach_count); | |
38 | if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && | |
39 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
40 | __ptep_ipte_local(addr, ptep); | |
41 | else | |
42 | __ptep_ipte(addr, ptep); | |
43 | atomic_sub(0x10000, &mm->context.attach_count); | |
44 | return old; | |
45 | } | |
46 | ||
47 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
48 | unsigned long addr, pte_t *ptep) | |
49 | { | |
50 | int active, count; | |
51 | pte_t old; | |
52 | ||
53 | old = *ptep; | |
54 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
55 | return old; | |
56 | active = (mm == current->active_mm) ? 1 : 0; | |
57 | count = atomic_add_return(0x10000, &mm->context.attach_count); | |
58 | if ((count & 0xffff) <= active) { | |
59 | pte_val(*ptep) |= _PAGE_INVALID; | |
60 | mm->context.flush_mm = 1; | |
61 | } else | |
62 | __ptep_ipte(addr, ptep); | |
63 | atomic_sub(0x10000, &mm->context.attach_count); | |
64 | return old; | |
65 | } | |
66 | ||
1e133ab2 MS |
67 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
68 | { | |
69 | unsigned long new = 0; | |
70 | #ifdef CONFIG_PGSTE | |
71 | unsigned long old; | |
72 | ||
73 | preempt_disable(); | |
74 | asm( | |
75 | " lg %0,%2\n" | |
76 | "0: lgr %1,%0\n" | |
77 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
78 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
79 | " csg %0,%1,%2\n" | |
80 | " jl 0b\n" | |
81 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
82 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
83 | #endif | |
84 | return __pgste(new); | |
85 | } | |
86 | ||
87 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
88 | { | |
89 | #ifdef CONFIG_PGSTE | |
90 | asm( | |
91 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
92 | " stg %1,%0\n" | |
93 | : "=Q" (ptep[PTRS_PER_PTE]) | |
94 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
95 | : "cc", "memory"); | |
96 | preempt_enable(); | |
97 | #endif | |
98 | } | |
99 | ||
100 | static inline pgste_t pgste_get(pte_t *ptep) | |
101 | { | |
102 | unsigned long pgste = 0; | |
103 | #ifdef CONFIG_PGSTE | |
104 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
105 | #endif | |
106 | return __pgste(pgste); | |
107 | } | |
108 | ||
109 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
110 | { | |
111 | #ifdef CONFIG_PGSTE | |
112 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
113 | #endif | |
114 | } | |
115 | ||
ebde765c MS |
116 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
117 | struct mm_struct *mm) | |
118 | { | |
119 | #ifdef CONFIG_PGSTE | |
120 | unsigned long address, bits, skey; | |
121 | ||
122 | if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) | |
123 | return pgste; | |
124 | address = pte_val(pte) & PAGE_MASK; | |
125 | skey = (unsigned long) page_get_storage_key(address); | |
126 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
127 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
128 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
129 | /* Copy page access key and fetch protection bit to pgste */ | |
130 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
131 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
132 | #endif | |
133 | return pgste; | |
134 | ||
135 | } | |
136 | ||
137 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
138 | struct mm_struct *mm) | |
139 | { | |
140 | #ifdef CONFIG_PGSTE | |
141 | unsigned long address; | |
142 | unsigned long nkey; | |
143 | ||
144 | if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) | |
145 | return; | |
146 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
147 | address = pte_val(entry) & PAGE_MASK; | |
148 | /* | |
149 | * Set page access key and fetch protection bit from pgste. | |
150 | * The guest C/R information is still in the PGSTE, set real | |
151 | * key C/R to 0. | |
152 | */ | |
153 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
154 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
155 | page_set_storage_key(address, nkey, 0); | |
156 | #endif | |
157 | } | |
158 | ||
159 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
160 | { | |
161 | #ifdef CONFIG_PGSTE | |
162 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
163 | (pte_val(entry) & _PAGE_WRITE) && | |
164 | !(pte_val(entry) & _PAGE_INVALID)) { | |
165 | if (!MACHINE_HAS_ESOP) { | |
166 | /* | |
167 | * Without enhanced suppression-on-protection force | |
168 | * the dirty bit on for all writable ptes. | |
169 | */ | |
170 | pte_val(entry) |= _PAGE_DIRTY; | |
171 | pte_val(entry) &= ~_PAGE_PROTECT; | |
172 | } | |
173 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
174 | /* This pte allows write access, set user-dirty */ | |
175 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
176 | } | |
177 | #endif | |
178 | *ptep = entry; | |
179 | return pgste; | |
180 | } | |
181 | ||
182 | static inline pgste_t pgste_ipte_notify(struct mm_struct *mm, | |
183 | unsigned long addr, | |
184 | pte_t *ptep, pgste_t pgste) | |
185 | { | |
186 | #ifdef CONFIG_PGSTE | |
187 | if (pgste_val(pgste) & PGSTE_IN_BIT) { | |
188 | pgste_val(pgste) &= ~PGSTE_IN_BIT; | |
1e133ab2 | 189 | ptep_notify(mm, addr, ptep); |
ebde765c MS |
190 | } |
191 | #endif | |
192 | return pgste; | |
193 | } | |
194 | ||
ebde765c MS |
195 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
196 | unsigned long addr, pte_t *ptep) | |
197 | { | |
198 | pgste_t pgste = __pgste(0); | |
199 | ||
200 | if (mm_has_pgste(mm)) { | |
201 | pgste = pgste_get_lock(ptep); | |
202 | pgste = pgste_ipte_notify(mm, addr, ptep, pgste); | |
203 | } | |
204 | return pgste; | |
205 | } | |
206 | ||
207 | static inline void ptep_xchg_commit(struct mm_struct *mm, | |
208 | unsigned long addr, pte_t *ptep, | |
209 | pgste_t pgste, pte_t old, pte_t new) | |
210 | { | |
211 | if (mm_has_pgste(mm)) { | |
212 | if (pte_val(old) & _PAGE_INVALID) | |
213 | pgste_set_key(ptep, pgste, new, mm); | |
214 | if (pte_val(new) & _PAGE_INVALID) { | |
215 | pgste = pgste_update_all(old, pgste, mm); | |
216 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
217 | _PGSTE_GPS_USAGE_UNUSED) | |
218 | pte_val(old) |= _PAGE_UNUSED; | |
219 | } | |
220 | pgste = pgste_set_pte(ptep, pgste, new); | |
221 | pgste_set_unlock(ptep, pgste); | |
222 | } else { | |
223 | *ptep = new; | |
224 | } | |
225 | } | |
226 | ||
227 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
228 | pte_t *ptep, pte_t new) | |
229 | { | |
230 | pgste_t pgste; | |
231 | pte_t old; | |
232 | ||
233 | pgste = ptep_xchg_start(mm, addr, ptep); | |
234 | old = ptep_flush_direct(mm, addr, ptep); | |
235 | ptep_xchg_commit(mm, addr, ptep, pgste, old, new); | |
236 | return old; | |
237 | } | |
238 | EXPORT_SYMBOL(ptep_xchg_direct); | |
239 | ||
240 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
241 | pte_t *ptep, pte_t new) | |
242 | { | |
243 | pgste_t pgste; | |
244 | pte_t old; | |
245 | ||
246 | pgste = ptep_xchg_start(mm, addr, ptep); | |
247 | old = ptep_flush_lazy(mm, addr, ptep); | |
248 | ptep_xchg_commit(mm, addr, ptep, pgste, old, new); | |
249 | return old; | |
250 | } | |
251 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
252 | ||
253 | pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, | |
254 | pte_t *ptep) | |
255 | { | |
256 | pgste_t pgste; | |
257 | pte_t old; | |
258 | ||
259 | pgste = ptep_xchg_start(mm, addr, ptep); | |
260 | old = ptep_flush_lazy(mm, addr, ptep); | |
261 | if (mm_has_pgste(mm)) { | |
262 | pgste = pgste_update_all(old, pgste, mm); | |
263 | pgste_set(ptep, pgste); | |
264 | } | |
265 | return old; | |
266 | } | |
267 | EXPORT_SYMBOL(ptep_modify_prot_start); | |
268 | ||
269 | void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, | |
270 | pte_t *ptep, pte_t pte) | |
271 | { | |
272 | pgste_t pgste; | |
273 | ||
274 | if (mm_has_pgste(mm)) { | |
275 | pgste = pgste_get(ptep); | |
276 | pgste_set_key(ptep, pgste, pte, mm); | |
277 | pgste = pgste_set_pte(ptep, pgste, pte); | |
278 | pgste_set_unlock(ptep, pgste); | |
279 | } else { | |
280 | *ptep = pte; | |
281 | } | |
282 | } | |
283 | EXPORT_SYMBOL(ptep_modify_prot_commit); | |
284 | ||
227be799 MS |
285 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
286 | unsigned long addr, pmd_t *pmdp) | |
287 | { | |
288 | int active, count; | |
289 | pmd_t old; | |
290 | ||
291 | old = *pmdp; | |
292 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
293 | return old; | |
294 | if (!MACHINE_HAS_IDTE) { | |
295 | __pmdp_csp(pmdp); | |
296 | return old; | |
297 | } | |
298 | active = (mm == current->active_mm) ? 1 : 0; | |
299 | count = atomic_add_return(0x10000, &mm->context.attach_count); | |
300 | if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active && | |
301 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
302 | __pmdp_idte_local(addr, pmdp); | |
303 | else | |
304 | __pmdp_idte(addr, pmdp); | |
305 | atomic_sub(0x10000, &mm->context.attach_count); | |
306 | return old; | |
307 | } | |
308 | ||
309 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
310 | unsigned long addr, pmd_t *pmdp) | |
311 | { | |
312 | int active, count; | |
313 | pmd_t old; | |
314 | ||
315 | old = *pmdp; | |
316 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
317 | return old; | |
318 | active = (mm == current->active_mm) ? 1 : 0; | |
319 | count = atomic_add_return(0x10000, &mm->context.attach_count); | |
320 | if ((count & 0xffff) <= active) { | |
321 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; | |
322 | mm->context.flush_mm = 1; | |
323 | } else if (MACHINE_HAS_IDTE) | |
324 | __pmdp_idte(addr, pmdp); | |
325 | else | |
326 | __pmdp_csp(pmdp); | |
327 | atomic_sub(0x10000, &mm->context.attach_count); | |
328 | return old; | |
329 | } | |
330 | ||
331 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
332 | pmd_t *pmdp, pmd_t new) | |
333 | { | |
334 | pmd_t old; | |
335 | ||
336 | old = pmdp_flush_direct(mm, addr, pmdp); | |
337 | *pmdp = new; | |
338 | return old; | |
339 | } | |
340 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
341 | ||
342 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
343 | pmd_t *pmdp, pmd_t new) | |
344 | { | |
345 | pmd_t old; | |
346 | ||
347 | old = pmdp_flush_lazy(mm, addr, pmdp); | |
348 | *pmdp = new; | |
349 | return old; | |
350 | } | |
351 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
352 | ||
75077afb | 353 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
354 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
355 | pgtable_t pgtable) | |
9501d09f GS |
356 | { |
357 | struct list_head *lh = (struct list_head *) pgtable; | |
358 | ||
ec66ad66 | 359 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
360 | |
361 | /* FIFO */ | |
c389a250 | 362 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
363 | INIT_LIST_HEAD(lh); |
364 | else | |
c389a250 KS |
365 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
366 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
367 | } |
368 | ||
6b0b50b0 | 369 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
370 | { |
371 | struct list_head *lh; | |
372 | pgtable_t pgtable; | |
373 | pte_t *ptep; | |
374 | ||
ec66ad66 | 375 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
376 | |
377 | /* FIFO */ | |
c389a250 | 378 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
379 | lh = (struct list_head *) pgtable; |
380 | if (list_empty(lh)) | |
c389a250 | 381 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 382 | else { |
c389a250 | 383 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
384 | list_del(lh); |
385 | } | |
386 | ptep = (pte_t *) pgtable; | |
e5098611 | 387 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 388 | ptep++; |
e5098611 | 389 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
390 | return pgtable; |
391 | } | |
75077afb | 392 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
393 | |
394 | #ifdef CONFIG_PGSTE | |
395 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
396 | pte_t *ptep, pte_t entry) | |
397 | { | |
398 | pgste_t pgste; | |
399 | ||
400 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
401 | pgste = pgste_get_lock(ptep); | |
402 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
403 | pgste_set_key(ptep, pgste, entry, mm); | |
404 | pgste = pgste_set_pte(ptep, pgste, entry); | |
405 | pgste_set_unlock(ptep, pgste); | |
406 | } | |
407 | ||
408 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
409 | { | |
410 | pgste_t pgste; | |
411 | ||
412 | pgste = pgste_get_lock(ptep); | |
413 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
414 | pgste_set_unlock(ptep, pgste); | |
415 | } | |
416 | ||
417 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) | |
418 | { | |
419 | if (!non_swap_entry(entry)) | |
420 | dec_mm_counter(mm, MM_SWAPENTS); | |
421 | else if (is_migration_entry(entry)) { | |
422 | struct page *page = migration_entry_to_page(entry); | |
423 | ||
424 | dec_mm_counter(mm, mm_counter(page)); | |
425 | } | |
426 | free_swap_and_cache(entry); | |
427 | } | |
428 | ||
429 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
430 | pte_t *ptep, int reset) | |
431 | { | |
432 | unsigned long pgstev; | |
433 | pgste_t pgste; | |
434 | pte_t pte; | |
435 | ||
436 | /* Zap unused and logically-zero pages */ | |
437 | pgste = pgste_get_lock(ptep); | |
438 | pgstev = pgste_val(pgste); | |
439 | pte = *ptep; | |
1c343f7b | 440 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
441 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
442 | (pgstev & _PGSTE_GPS_ZERO))) { | |
443 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
444 | pte_clear(mm, addr, ptep); | |
445 | } | |
446 | if (reset) | |
447 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
448 | pgste_set_unlock(ptep, pgste); | |
449 | } | |
450 | ||
451 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
452 | { | |
453 | unsigned long ptev; | |
454 | pgste_t pgste; | |
455 | ||
456 | /* Clear storage key */ | |
457 | pgste = pgste_get_lock(ptep); | |
458 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | | |
459 | PGSTE_GR_BIT | PGSTE_GC_BIT); | |
460 | ptev = pte_val(*ptep); | |
461 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
462 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
463 | pgste_set_unlock(ptep, pgste); | |
464 | } | |
465 | ||
466 | /* | |
467 | * Test and reset if a guest page is dirty | |
468 | */ | |
469 | bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) | |
470 | { | |
471 | spinlock_t *ptl; | |
472 | pgste_t pgste; | |
473 | pte_t *ptep; | |
474 | pte_t pte; | |
475 | bool dirty; | |
476 | ||
477 | ptep = get_locked_pte(mm, addr, &ptl); | |
478 | if (unlikely(!ptep)) | |
479 | return false; | |
480 | ||
481 | pgste = pgste_get_lock(ptep); | |
482 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
483 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
484 | pte = *ptep; | |
485 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
486 | pgste = pgste_ipte_notify(mm, addr, ptep, pgste); | |
487 | __ptep_ipte(addr, ptep); | |
488 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) | |
489 | pte_val(pte) |= _PAGE_PROTECT; | |
490 | else | |
491 | pte_val(pte) |= _PAGE_INVALID; | |
492 | *ptep = pte; | |
493 | } | |
494 | pgste_set_unlock(ptep, pgste); | |
495 | ||
496 | spin_unlock(ptl); | |
497 | return dirty; | |
498 | } | |
499 | EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); | |
500 | ||
501 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
502 | unsigned char key, bool nq) | |
503 | { | |
504 | unsigned long keyul; | |
505 | spinlock_t *ptl; | |
506 | pgste_t old, new; | |
507 | pte_t *ptep; | |
508 | ||
509 | down_read(&mm->mmap_sem); | |
510 | ptep = get_locked_pte(mm, addr, &ptl); | |
511 | if (unlikely(!ptep)) { | |
512 | up_read(&mm->mmap_sem); | |
513 | return -EFAULT; | |
514 | } | |
515 | ||
516 | new = old = pgste_get_lock(ptep); | |
517 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
518 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
519 | keyul = (unsigned long) key; | |
520 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
521 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
522 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
523 | unsigned long address, bits, skey; | |
524 | ||
525 | address = pte_val(*ptep) & PAGE_MASK; | |
526 | skey = (unsigned long) page_get_storage_key(address); | |
527 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
528 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
529 | /* Set storage key ACC and FP */ | |
530 | page_set_storage_key(address, skey, !nq); | |
531 | /* Merge host changed & referenced into pgste */ | |
532 | pgste_val(new) |= bits << 52; | |
533 | } | |
534 | /* changing the guest storage key is considered a change of the page */ | |
535 | if ((pgste_val(new) ^ pgste_val(old)) & | |
536 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
537 | pgste_val(new) |= PGSTE_UC_BIT; | |
538 | ||
539 | pgste_set_unlock(ptep, new); | |
540 | pte_unmap_unlock(ptep, ptl); | |
541 | up_read(&mm->mmap_sem); | |
542 | return 0; | |
543 | } | |
544 | EXPORT_SYMBOL(set_guest_storage_key); | |
545 | ||
546 | unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr) | |
547 | { | |
548 | unsigned char key; | |
549 | spinlock_t *ptl; | |
550 | pgste_t pgste; | |
551 | pte_t *ptep; | |
552 | ||
553 | down_read(&mm->mmap_sem); | |
554 | ptep = get_locked_pte(mm, addr, &ptl); | |
555 | if (unlikely(!ptep)) { | |
556 | up_read(&mm->mmap_sem); | |
557 | return -EFAULT; | |
558 | } | |
559 | pgste = pgste_get_lock(ptep); | |
560 | ||
561 | if (pte_val(*ptep) & _PAGE_INVALID) { | |
562 | key = (pgste_val(pgste) & PGSTE_ACC_BITS) >> 56; | |
563 | key |= (pgste_val(pgste) & PGSTE_FP_BIT) >> 56; | |
564 | key |= (pgste_val(pgste) & PGSTE_GR_BIT) >> 48; | |
565 | key |= (pgste_val(pgste) & PGSTE_GC_BIT) >> 48; | |
566 | } else { | |
567 | key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); | |
568 | ||
569 | /* Reflect guest's logical view, not physical */ | |
570 | if (pgste_val(pgste) & PGSTE_GR_BIT) | |
571 | key |= _PAGE_REFERENCED; | |
572 | if (pgste_val(pgste) & PGSTE_GC_BIT) | |
573 | key |= _PAGE_CHANGED; | |
574 | } | |
575 | ||
576 | pgste_set_unlock(ptep, pgste); | |
577 | pte_unmap_unlock(ptep, ptl); | |
578 | up_read(&mm->mmap_sem); | |
579 | return key; | |
580 | } | |
581 | EXPORT_SYMBOL(get_guest_storage_key); | |
582 | #endif |