Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a8606e20 | 2 | /* |
a8606e20 PM |
3 | * |
4 | * Copyright 2010-2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
5 | */ | |
6 | ||
7 | #include <linux/types.h> | |
8 | #include <linux/string.h> | |
9 | #include <linux/kvm.h> | |
10 | #include <linux/kvm_host.h> | |
11 | #include <linux/hugetlb.h> | |
c77162de | 12 | #include <linux/module.h> |
08fe1e7b | 13 | #include <linux/log2.h> |
eadfb1c5 | 14 | #include <linux/sizes.h> |
a8606e20 | 15 | |
0428491c | 16 | #include <asm/trace.h> |
a8606e20 PM |
17 | #include <asm/kvm_ppc.h> |
18 | #include <asm/kvm_book3s.h> | |
f64e8084 | 19 | #include <asm/book3s/64/mmu-hash.h> |
a8606e20 PM |
20 | #include <asm/hvcall.h> |
21 | #include <asm/synch.h> | |
22 | #include <asm/ppc-opcode.h> | |
94171b19 | 23 | #include <asm/pte-walk.h> |
a8606e20 | 24 | |
8936dda4 | 25 | /* Translate address of a vmalloc'd thing to a linear map address */ |
5362a4b6 | 26 | static void *real_vmalloc_addr(void *addr) |
8936dda4 | 27 | { |
5362a4b6 | 28 | return __va(ppc_find_vmap_phys((unsigned long)addr)); |
8936dda4 | 29 | } |
a8606e20 | 30 | |
1b400ba0 | 31 | /* Return 1 if we need to do a global tlbie, 0 if we can use tlbiel */ |
76b03dc0 | 32 | static int global_invalidates(struct kvm *kvm) |
1b400ba0 PM |
33 | { |
34 | int global; | |
a29ebeaf | 35 | int cpu; |
1b400ba0 PM |
36 | |
37 | /* | |
38 | * If there is only one vcore, and it's currently running, | |
55765483 | 39 | * as indicated by local_paca->kvm_hstate.kvm_vcpu being set, |
1b400ba0 PM |
40 | * we can use tlbiel as long as we mark all other physical |
41 | * cores as potentially having stale TLB entries for this lpid. | |
1b400ba0 PM |
42 | * Otherwise, don't use tlbiel. |
43 | */ | |
55765483 | 44 | if (kvm->arch.online_vcores == 1 && local_paca->kvm_hstate.kvm_vcpu) |
1b400ba0 | 45 | global = 0; |
1b400ba0 | 46 | else |
c17b98cf | 47 | global = 1; |
1b400ba0 | 48 | |
079a09a5 NP |
49 | /* LPID has been switched to host if in virt mode so can't do local */ |
50 | if (!global && (mfmsr() & (MSR_IR|MSR_DR))) | |
51 | global = 1; | |
52 | ||
1b400ba0 PM |
53 | if (!global) { |
54 | /* any other core might now have stale TLB entries... */ | |
55 | smp_wmb(); | |
56 | cpumask_setall(&kvm->arch.need_tlb_flush); | |
a29ebeaf | 57 | cpu = local_paca->kvm_hstate.kvm_vcore->pcpu; |
a29ebeaf | 58 | cpumask_clear_cpu(cpu, &kvm->arch.need_tlb_flush); |
1b400ba0 PM |
59 | } |
60 | ||
61 | return global; | |
62 | } | |
63 | ||
06ce2c63 PM |
64 | /* |
65 | * Add this HPTE into the chain for the real page. | |
66 | * Must be called with the chain locked; it unlocks the chain. | |
67 | */ | |
342d3db7 | 68 | void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, |
06ce2c63 PM |
69 | unsigned long *rmap, long pte_index, int realmode) |
70 | { | |
71 | struct revmap_entry *head, *tail; | |
72 | unsigned long i; | |
73 | ||
74 | if (*rmap & KVMPPC_RMAP_PRESENT) { | |
75 | i = *rmap & KVMPPC_RMAP_INDEX; | |
3f9d4f5a | 76 | head = &kvm->arch.hpt.rev[i]; |
06ce2c63 PM |
77 | if (realmode) |
78 | head = real_vmalloc_addr(head); | |
3f9d4f5a | 79 | tail = &kvm->arch.hpt.rev[head->back]; |
06ce2c63 PM |
80 | if (realmode) |
81 | tail = real_vmalloc_addr(tail); | |
82 | rev->forw = i; | |
83 | rev->back = head->back; | |
84 | tail->forw = pte_index; | |
85 | head->back = pte_index; | |
86 | } else { | |
87 | rev->forw = rev->back = pte_index; | |
4879f241 | 88 | *rmap = (*rmap & ~KVMPPC_RMAP_INDEX) | |
d22deab6 | 89 | pte_index | KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_HPT; |
06ce2c63 | 90 | } |
4879f241 | 91 | unlock_rmap(rmap); |
06ce2c63 | 92 | } |
342d3db7 | 93 | EXPORT_SYMBOL_GPL(kvmppc_add_revmap_chain); |
06ce2c63 | 94 | |
e641a317 | 95 | /* Update the dirty bitmap of a memslot */ |
c43c3a86 | 96 | void kvmppc_update_dirty_map(const struct kvm_memory_slot *memslot, |
e641a317 | 97 | unsigned long gfn, unsigned long psize) |
08fe1e7b | 98 | { |
e641a317 | 99 | unsigned long npages; |
08fe1e7b | 100 | |
e641a317 | 101 | if (!psize || !memslot->dirty_bitmap) |
08fe1e7b | 102 | return; |
e641a317 PM |
103 | npages = (psize + PAGE_SIZE - 1) / PAGE_SIZE; |
104 | gfn -= memslot->base_gfn; | |
105 | set_dirty_bits_atomic(memslot->dirty_bitmap, gfn, npages); | |
106 | } | |
107 | EXPORT_SYMBOL_GPL(kvmppc_update_dirty_map); | |
108 | ||
109 | static void kvmppc_set_dirty_from_hpte(struct kvm *kvm, | |
110 | unsigned long hpte_v, unsigned long hpte_gr) | |
111 | { | |
112 | struct kvm_memory_slot *memslot; | |
113 | unsigned long gfn; | |
114 | unsigned long psize; | |
115 | ||
116 | psize = kvmppc_actual_pgsz(hpte_v, hpte_gr); | |
117 | gfn = hpte_rpn(hpte_gr, psize); | |
118 | memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); | |
119 | if (memslot && memslot->dirty_bitmap) | |
120 | kvmppc_update_dirty_map(memslot, gfn, psize); | |
08fe1e7b | 121 | } |
08fe1e7b | 122 | |
cdeee518 PM |
123 | /* Returns a pointer to the revmap entry for the page mapped by a HPTE */ |
124 | static unsigned long *revmap_for_hpte(struct kvm *kvm, unsigned long hpte_v, | |
e641a317 PM |
125 | unsigned long hpte_gr, |
126 | struct kvm_memory_slot **memslotp, | |
127 | unsigned long *gfnp) | |
cdeee518 PM |
128 | { |
129 | struct kvm_memory_slot *memslot; | |
130 | unsigned long *rmap; | |
131 | unsigned long gfn; | |
132 | ||
8dc6cca5 | 133 | gfn = hpte_rpn(hpte_gr, kvmppc_actual_pgsz(hpte_v, hpte_gr)); |
cdeee518 | 134 | memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); |
e641a317 PM |
135 | if (memslotp) |
136 | *memslotp = memslot; | |
137 | if (gfnp) | |
138 | *gfnp = gfn; | |
cdeee518 PM |
139 | if (!memslot) |
140 | return NULL; | |
141 | ||
142 | rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]); | |
143 | return rmap; | |
144 | } | |
145 | ||
06ce2c63 PM |
146 | /* Remove this HPTE from the chain for a real page */ |
147 | static void remove_revmap_chain(struct kvm *kvm, long pte_index, | |
bad3b507 PM |
148 | struct revmap_entry *rev, |
149 | unsigned long hpte_v, unsigned long hpte_r) | |
06ce2c63 | 150 | { |
bad3b507 | 151 | struct revmap_entry *next, *prev; |
cdeee518 | 152 | unsigned long ptel, head; |
06ce2c63 | 153 | unsigned long *rmap; |
bad3b507 | 154 | unsigned long rcbits; |
e641a317 PM |
155 | struct kvm_memory_slot *memslot; |
156 | unsigned long gfn; | |
06ce2c63 | 157 | |
bad3b507 PM |
158 | rcbits = hpte_r & (HPTE_R_R | HPTE_R_C); |
159 | ptel = rev->guest_rpte |= rcbits; | |
e641a317 | 160 | rmap = revmap_for_hpte(kvm, hpte_v, ptel, &memslot, &gfn); |
cdeee518 | 161 | if (!rmap) |
06ce2c63 | 162 | return; |
06ce2c63 PM |
163 | lock_rmap(rmap); |
164 | ||
165 | head = *rmap & KVMPPC_RMAP_INDEX; | |
3f9d4f5a DG |
166 | next = real_vmalloc_addr(&kvm->arch.hpt.rev[rev->forw]); |
167 | prev = real_vmalloc_addr(&kvm->arch.hpt.rev[rev->back]); | |
06ce2c63 PM |
168 | next->back = rev->back; |
169 | prev->forw = rev->forw; | |
170 | if (head == pte_index) { | |
171 | head = rev->forw; | |
172 | if (head == pte_index) | |
173 | *rmap &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX); | |
174 | else | |
175 | *rmap = (*rmap & ~KVMPPC_RMAP_INDEX) | head; | |
176 | } | |
bad3b507 | 177 | *rmap |= rcbits << KVMPPC_RMAP_RC_SHIFT; |
08fe1e7b | 178 | if (rcbits & HPTE_R_C) |
e641a317 PM |
179 | kvmppc_update_dirty_map(memslot, gfn, |
180 | kvmppc_actual_pgsz(hpte_v, hpte_r)); | |
06ce2c63 PM |
181 | unlock_rmap(rmap); |
182 | } | |
183 | ||
7ed661bf PM |
184 | long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags, |
185 | long pte_index, unsigned long pteh, unsigned long ptel, | |
186 | pgd_t *pgdir, bool realmode, unsigned long *pte_idx_ret) | |
a8606e20 | 187 | { |
c77162de | 188 | unsigned long i, pa, gpa, gfn, psize; |
342d3db7 | 189 | unsigned long slot_fn, hva; |
6f22bd32 | 190 | __be64 *hpte; |
8936dda4 | 191 | struct revmap_entry *rev; |
44e5f6be | 192 | unsigned long g_ptel; |
b2b2f165 | 193 | struct kvm_memory_slot *memslot; |
dac56570 | 194 | unsigned hpage_shift; |
30bda41a | 195 | bool is_ci; |
06ce2c63 | 196 | unsigned long *rmap; |
dac56570 | 197 | pte_t *ptep; |
4cf302bc | 198 | unsigned int writing; |
342d3db7 | 199 | unsigned long mmu_seq; |
e3d8ed55 | 200 | unsigned long rcbits; |
c77162de | 201 | |
65dae540 PM |
202 | if (kvm_is_radix(kvm)) |
203 | return H_FUNCTION; | |
322fda04 NP |
204 | /* |
205 | * The HPTE gets used by compute_tlbie_rb() to set TLBIE bits, so | |
206 | * these functions should work together -- must ensure a guest can not | |
207 | * cause problems with the TLBIE that KVM executes. | |
208 | */ | |
209 | if ((pteh >> HPTE_V_SSIZE_SHIFT) & 0x2) { | |
210 | /* B=0b1x is a reserved value, disallow it. */ | |
211 | return H_PARAMETER; | |
212 | } | |
8dc6cca5 | 213 | psize = kvmppc_actual_pgsz(pteh, ptel); |
c77162de | 214 | if (!psize) |
a8606e20 | 215 | return H_PARAMETER; |
4cf302bc | 216 | writing = hpte_is_writable(ptel); |
697d3899 | 217 | pteh &= ~(HPTE_V_HVLOCK | HPTE_V_ABSENT | HPTE_V_VALID); |
44e5f6be PM |
218 | ptel &= ~HPTE_GR_RESERVED; |
219 | g_ptel = ptel; | |
b2b2f165 | 220 | |
342d3db7 | 221 | /* used later to detect if we might have been invalidated */ |
20ec3ebd | 222 | mmu_seq = kvm->mmu_invalidate_seq; |
342d3db7 PM |
223 | smp_rmb(); |
224 | ||
c77162de PM |
225 | /* Find the memslot (if any) for this address */ |
226 | gpa = (ptel & HPTE_R_RPN) & ~(psize - 1); | |
227 | gfn = gpa >> PAGE_SHIFT; | |
797f9c07 | 228 | memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); |
697d3899 | 229 | pa = 0; |
30bda41a | 230 | is_ci = false; |
697d3899 PM |
231 | rmap = NULL; |
232 | if (!(memslot && !(memslot->flags & KVM_MEMSLOT_INVALID))) { | |
697d3899 PM |
233 | /* Emulated MMIO - mark this with key=31 */ |
234 | pteh |= HPTE_V_ABSENT; | |
235 | ptel |= HPTE_R_KEY_HI | HPTE_R_KEY_LO; | |
236 | goto do_insert; | |
237 | } | |
da9d1d7f PM |
238 | |
239 | /* Check if the requested page fits entirely in the memslot. */ | |
240 | if (!slot_is_aligned(memslot, psize)) | |
241 | return H_PARAMETER; | |
c77162de | 242 | slot_fn = gfn - memslot->base_gfn; |
d89cc617 | 243 | rmap = &memslot->arch.rmap[slot_fn]; |
c77162de | 244 | |
c17b98cf PM |
245 | /* Translate to host virtual address */ |
246 | hva = __gfn_to_hva_memslot(memslot, gfn); | |
e3d8ed55 AK |
247 | |
248 | arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock); | |
249 | ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &hpage_shift); | |
dac56570 AK |
250 | if (ptep) { |
251 | pte_t pte; | |
252 | unsigned int host_pte_size; | |
7ed661bf | 253 | |
dac56570 AK |
254 | if (hpage_shift) |
255 | host_pte_size = 1ul << hpage_shift; | |
256 | else | |
257 | host_pte_size = PAGE_SIZE; | |
258 | /* | |
259 | * We should always find the guest page size | |
260 | * to <= host page size, if host is using hugepage | |
261 | */ | |
691e95fd | 262 | if (host_pte_size < psize) { |
e3d8ed55 | 263 | arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock); |
dac56570 | 264 | return H_PARAMETER; |
691e95fd | 265 | } |
7d6e7f7f | 266 | pte = kvmppc_read_update_linux_pte(ptep, writing); |
dac56570 | 267 | if (pte_present(pte) && !pte_protnone(pte)) { |
d6379159 | 268 | if (writing && !pte_write(pte)) |
dac56570 AK |
269 | /* make the actual HPTE be read-only */ |
270 | ptel = hpte_make_readonly(ptel); | |
30bda41a | 271 | is_ci = pte_ci(pte); |
dac56570 AK |
272 | pa = pte_pfn(pte) << PAGE_SHIFT; |
273 | pa |= hva & (host_pte_size - 1); | |
274 | pa |= gpa & ~PAGE_MASK; | |
275 | } | |
276 | } | |
e3d8ed55 | 277 | arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock); |
c77162de | 278 | |
d182b8fd | 279 | ptel &= HPTE_R_KEY | HPTE_R_PP0 | (psize-1); |
c77162de | 280 | ptel |= pa; |
342d3db7 PM |
281 | |
282 | if (pa) | |
283 | pteh |= HPTE_V_VALID; | |
f0585982 | 284 | else { |
342d3db7 | 285 | pteh |= HPTE_V_ABSENT; |
f0585982 YX |
286 | ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO); |
287 | } | |
c77162de | 288 | |
30bda41a AK |
289 | /*If we had host pte mapping then Check WIMG */ |
290 | if (ptep && !hpte_cache_flags_ok(ptel, is_ci)) { | |
291 | if (is_ci) | |
9d0ef5ea PM |
292 | return H_PARAMETER; |
293 | /* | |
294 | * Allow guest to map emulated device memory as | |
295 | * uncacheable, but actually make it cacheable. | |
296 | */ | |
297 | ptel &= ~(HPTE_R_W|HPTE_R_I|HPTE_R_G); | |
298 | ptel |= HPTE_R_M; | |
299 | } | |
075295dd | 300 | |
342d3db7 | 301 | /* Find and lock the HPTEG slot to use */ |
697d3899 | 302 | do_insert: |
3d089f84 | 303 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
a8606e20 PM |
304 | return H_PARAMETER; |
305 | if (likely((flags & H_EXACT) == 0)) { | |
306 | pte_index &= ~7UL; | |
3f9d4f5a | 307 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); |
075295dd | 308 | for (i = 0; i < 8; ++i) { |
6f22bd32 | 309 | if ((be64_to_cpu(*hpte) & HPTE_V_VALID) == 0 && |
697d3899 PM |
310 | try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID | |
311 | HPTE_V_ABSENT)) | |
a8606e20 PM |
312 | break; |
313 | hpte += 2; | |
314 | } | |
075295dd PM |
315 | if (i == 8) { |
316 | /* | |
317 | * Since try_lock_hpte doesn't retry (not even stdcx. | |
318 | * failures), it could be that there is a free slot | |
319 | * but we transiently failed to lock it. Try again, | |
320 | * actually locking each slot and checking it. | |
321 | */ | |
322 | hpte -= 16; | |
323 | for (i = 0; i < 8; ++i) { | |
6f22bd32 | 324 | u64 pte; |
075295dd PM |
325 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
326 | cpu_relax(); | |
a4bd6eb0 | 327 | pte = be64_to_cpu(hpte[0]); |
6f22bd32 | 328 | if (!(pte & (HPTE_V_VALID | HPTE_V_ABSENT))) |
075295dd | 329 | break; |
a4bd6eb0 | 330 | __unlock_hpte(hpte, pte); |
075295dd PM |
331 | hpte += 2; |
332 | } | |
333 | if (i == 8) | |
334 | return H_PTEG_FULL; | |
335 | } | |
8936dda4 | 336 | pte_index += i; |
a8606e20 | 337 | } else { |
3f9d4f5a | 338 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); |
697d3899 PM |
339 | if (!try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID | |
340 | HPTE_V_ABSENT)) { | |
075295dd | 341 | /* Lock the slot and check again */ |
6f22bd32 AG |
342 | u64 pte; |
343 | ||
075295dd PM |
344 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
345 | cpu_relax(); | |
a4bd6eb0 | 346 | pte = be64_to_cpu(hpte[0]); |
6f22bd32 | 347 | if (pte & (HPTE_V_VALID | HPTE_V_ABSENT)) { |
a4bd6eb0 | 348 | __unlock_hpte(hpte, pte); |
075295dd PM |
349 | return H_PTEG_FULL; |
350 | } | |
351 | } | |
a8606e20 | 352 | } |
8936dda4 PM |
353 | |
354 | /* Save away the guest's idea of the second HPTE dword */ | |
3f9d4f5a | 355 | rev = &kvm->arch.hpt.rev[pte_index]; |
06ce2c63 PM |
356 | if (realmode) |
357 | rev = real_vmalloc_addr(rev); | |
44e5f6be | 358 | if (rev) { |
8936dda4 | 359 | rev->guest_rpte = g_ptel; |
44e5f6be PM |
360 | note_hpte_modification(kvm, rev); |
361 | } | |
06ce2c63 PM |
362 | |
363 | /* Link HPTE into reverse-map chain */ | |
697d3899 PM |
364 | if (pteh & HPTE_V_VALID) { |
365 | if (realmode) | |
366 | rmap = real_vmalloc_addr(rmap); | |
367 | lock_rmap(rmap); | |
342d3db7 | 368 | /* Check for pending invalidations under the rmap chain lock */ |
20ec3ebd | 369 | if (mmu_invalidate_retry(kvm, mmu_seq)) { |
342d3db7 PM |
370 | /* inval in progress, write a non-present HPTE */ |
371 | pteh |= HPTE_V_ABSENT; | |
372 | pteh &= ~HPTE_V_VALID; | |
f0585982 | 373 | ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO); |
342d3db7 PM |
374 | unlock_rmap(rmap); |
375 | } else { | |
376 | kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index, | |
377 | realmode); | |
bad3b507 PM |
378 | /* Only set R/C in real HPTE if already set in *rmap */ |
379 | rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT; | |
380 | ptel &= rcbits | ~(HPTE_R_R | HPTE_R_C); | |
342d3db7 | 381 | } |
697d3899 | 382 | } |
06ce2c63 | 383 | |
abb7c7dd PM |
384 | /* Convert to new format on P9 */ |
385 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
386 | ptel = hpte_old_to_new_r(pteh, ptel); | |
387 | pteh = hpte_old_to_new_v(pteh); | |
388 | } | |
6f22bd32 | 389 | hpte[1] = cpu_to_be64(ptel); |
06ce2c63 PM |
390 | |
391 | /* Write the first HPTE dword, unlocking the HPTE and making it valid */ | |
a8606e20 | 392 | eieio(); |
a4bd6eb0 | 393 | __unlock_hpte(hpte, pteh); |
a8606e20 | 394 | asm volatile("ptesync" : : : "memory"); |
06ce2c63 | 395 | |
7ed661bf | 396 | *pte_idx_ret = pte_index; |
a8606e20 PM |
397 | return H_SUCCESS; |
398 | } | |
7ed661bf PM |
399 | EXPORT_SYMBOL_GPL(kvmppc_do_h_enter); |
400 | ||
401 | long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, | |
402 | long pte_index, unsigned long pteh, unsigned long ptel) | |
403 | { | |
404 | return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel, | |
1143a706 SG |
405 | vcpu->arch.pgdir, true, |
406 | &vcpu->arch.regs.gpr[4]); | |
7ed661bf | 407 | } |
6165d5dd | 408 | EXPORT_SYMBOL_GPL(kvmppc_h_enter); |
a8606e20 | 409 | |
54bb7f4b | 410 | #ifdef __BIG_ENDIAN__ |
a8606e20 | 411 | #define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) |
54bb7f4b AB |
412 | #else |
413 | #define LOCK_TOKEN (*(u32 *)(&get_paca()->paca_index)) | |
414 | #endif | |
a8606e20 | 415 | |
a56ee9f8 YX |
416 | static inline int is_mmio_hpte(unsigned long v, unsigned long r) |
417 | { | |
418 | return ((v & HPTE_V_ABSENT) && | |
419 | (r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) == | |
420 | (HPTE_R_KEY_HI | HPTE_R_KEY_LO)); | |
421 | } | |
422 | ||
047e6575 AK |
423 | static inline void fixup_tlbie_lpid(unsigned long rb_value, unsigned long lpid) |
424 | { | |
425 | ||
426 | if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { | |
427 | /* Radix flush for a hash guest */ | |
428 | ||
429 | unsigned long rb,rs,prs,r,ric; | |
430 | ||
431 | rb = PPC_BIT(52); /* IS = 2 */ | |
432 | rs = 0; /* lpid = 0 */ | |
433 | prs = 0; /* partition scoped */ | |
434 | r = 1; /* radix format */ | |
435 | ric = 0; /* RIC_FLSUH_TLB */ | |
436 | ||
437 | /* | |
438 | * Need the extra ptesync to make sure we don't | |
439 | * re-order the tlbie | |
440 | */ | |
441 | asm volatile("ptesync": : :"memory"); | |
442 | asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) | |
443 | : : "r"(rb), "i"(r), "i"(prs), | |
444 | "i"(ric), "r"(rs) : "memory"); | |
445 | } | |
446 | ||
447 | if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { | |
448 | asm volatile("ptesync": : :"memory"); | |
449 | asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : : | |
450 | "r" (rb_value), "r" (lpid)); | |
451 | } | |
452 | } | |
453 | ||
54480501 PM |
454 | static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, |
455 | long npages, int global, bool need_sync) | |
456 | { | |
457 | long i; | |
458 | ||
7c5b06ca PM |
459 | /* |
460 | * We use the POWER9 5-operand versions of tlbie and tlbiel here. | |
461 | * Since we are using RIC=0 PRS=0 R=0, and P7/P8 tlbiel ignores | |
462 | * the RS field, this is backwards-compatible with P7 and P8. | |
463 | */ | |
54480501 | 464 | if (global) { |
54480501 PM |
465 | if (need_sync) |
466 | asm volatile("ptesync" : : : "memory"); | |
0428491c | 467 | for (i = 0; i < npages; ++i) { |
7c5b06ca | 468 | asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : : |
54480501 | 469 | "r" (rbvalues[i]), "r" (kvm->arch.lpid)); |
0428491c | 470 | } |
a5d4b589 | 471 | |
047e6575 | 472 | fixup_tlbie_lpid(rbvalues[i - 1], kvm->arch.lpid); |
54480501 | 473 | asm volatile("eieio; tlbsync; ptesync" : : : "memory"); |
54480501 PM |
474 | } else { |
475 | if (need_sync) | |
476 | asm volatile("ptesync" : : : "memory"); | |
0428491c | 477 | for (i = 0; i < npages; ++i) { |
7c5b06ca PM |
478 | asm volatile(PPC_TLBIEL(%0,%1,0,0,0) : : |
479 | "r" (rbvalues[i]), "r" (0)); | |
0428491c | 480 | } |
54480501 PM |
481 | asm volatile("ptesync" : : : "memory"); |
482 | } | |
483 | } | |
484 | ||
6b445ad4 PM |
485 | long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, |
486 | unsigned long pte_index, unsigned long avpn, | |
487 | unsigned long *hpret) | |
a8606e20 | 488 | { |
6f22bd32 | 489 | __be64 *hpte; |
a8606e20 | 490 | unsigned long v, r, rb; |
a92bce95 | 491 | struct revmap_entry *rev; |
abb7c7dd | 492 | u64 pte, orig_pte, pte_r; |
a8606e20 | 493 | |
65dae540 PM |
494 | if (kvm_is_radix(kvm)) |
495 | return H_FUNCTION; | |
3d089f84 | 496 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
a8606e20 | 497 | return H_PARAMETER; |
3f9d4f5a | 498 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); |
075295dd | 499 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
a8606e20 | 500 | cpu_relax(); |
abb7c7dd PM |
501 | pte = orig_pte = be64_to_cpu(hpte[0]); |
502 | pte_r = be64_to_cpu(hpte[1]); | |
503 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
504 | pte = hpte_new_to_old_v(pte, pte_r); | |
505 | pte_r = hpte_new_to_old_r(pte_r); | |
506 | } | |
6f22bd32 AG |
507 | if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || |
508 | ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) || | |
509 | ((flags & H_ANDCOND) && (pte & avpn) != 0)) { | |
abb7c7dd | 510 | __unlock_hpte(hpte, orig_pte); |
a8606e20 PM |
511 | return H_NOT_FOUND; |
512 | } | |
a92bce95 | 513 | |
3f9d4f5a | 514 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
6f22bd32 | 515 | v = pte & ~HPTE_V_HVLOCK; |
a92bce95 | 516 | if (v & HPTE_V_VALID) { |
6f22bd32 | 517 | hpte[0] &= ~cpu_to_be64(HPTE_V_VALID); |
abb7c7dd | 518 | rb = compute_tlbie_rb(v, pte_r, pte_index); |
76b03dc0 | 519 | do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true); |
1e5bf454 PM |
520 | /* |
521 | * The reference (R) and change (C) bits in a HPT | |
522 | * entry can be set by hardware at any time up until | |
523 | * the HPTE is invalidated and the TLB invalidation | |
524 | * sequence has completed. This means that when | |
525 | * removing a HPTE, we need to re-read the HPTE after | |
526 | * the invalidation sequence has completed in order to | |
527 | * obtain reliable values of R and C. | |
528 | */ | |
529 | remove_revmap_chain(kvm, pte_index, rev, v, | |
530 | be64_to_cpu(hpte[1])); | |
a8606e20 | 531 | } |
44e5f6be PM |
532 | r = rev->guest_rpte & ~HPTE_GR_RESERVED; |
533 | note_hpte_modification(kvm, rev); | |
a92bce95 PM |
534 | unlock_hpte(hpte, 0); |
535 | ||
abb7c7dd | 536 | if (is_mmio_hpte(v, pte_r)) |
a56ee9f8 YX |
537 | atomic64_inc(&kvm->arch.mmio_update); |
538 | ||
c64dfe2a PM |
539 | if (v & HPTE_V_ABSENT) |
540 | v = (v & ~HPTE_V_ABSENT) | HPTE_V_VALID; | |
6b445ad4 PM |
541 | hpret[0] = v; |
542 | hpret[1] = r; | |
a8606e20 PM |
543 | return H_SUCCESS; |
544 | } | |
6b445ad4 PM |
545 | EXPORT_SYMBOL_GPL(kvmppc_do_h_remove); |
546 | ||
547 | long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags, | |
548 | unsigned long pte_index, unsigned long avpn) | |
549 | { | |
550 | return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn, | |
1143a706 | 551 | &vcpu->arch.regs.gpr[4]); |
6b445ad4 | 552 | } |
6165d5dd | 553 | EXPORT_SYMBOL_GPL(kvmppc_h_remove); |
a8606e20 PM |
554 | |
555 | long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) | |
556 | { | |
557 | struct kvm *kvm = vcpu->kvm; | |
1143a706 | 558 | unsigned long *args = &vcpu->arch.regs.gpr[4]; |
6f22bd32 AG |
559 | __be64 *hp, *hptes[4]; |
560 | unsigned long tlbrb[4]; | |
a92bce95 PM |
561 | long int i, j, k, n, found, indexes[4]; |
562 | unsigned long flags, req, pte_index, rcbits; | |
54480501 | 563 | int global; |
a8606e20 | 564 | long int ret = H_SUCCESS; |
a92bce95 | 565 | struct revmap_entry *rev, *revs[4]; |
a56ee9f8 | 566 | u64 hp0, hp1; |
a8606e20 | 567 | |
65dae540 PM |
568 | if (kvm_is_radix(kvm)) |
569 | return H_FUNCTION; | |
76b03dc0 | 570 | global = global_invalidates(kvm); |
a92bce95 PM |
571 | for (i = 0; i < 4 && ret == H_SUCCESS; ) { |
572 | n = 0; | |
573 | for (; i < 4; ++i) { | |
574 | j = i * 2; | |
575 | pte_index = args[j]; | |
576 | flags = pte_index >> 56; | |
577 | pte_index &= ((1ul << 56) - 1); | |
578 | req = flags >> 6; | |
579 | flags &= 3; | |
580 | if (req == 3) { /* no more requests */ | |
581 | i = 4; | |
a8606e20 | 582 | break; |
a92bce95 | 583 | } |
32fad281 | 584 | if (req != 1 || flags == 3 || |
3d089f84 | 585 | pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) { |
a92bce95 PM |
586 | /* parameter error */ |
587 | args[j] = ((0xa0 | flags) << 56) + pte_index; | |
588 | ret = H_PARAMETER; | |
a8606e20 | 589 | break; |
a92bce95 | 590 | } |
3f9d4f5a | 591 | hp = (__be64 *) (kvm->arch.hpt.virt + (pte_index << 4)); |
a92bce95 PM |
592 | /* to avoid deadlock, don't spin except for first */ |
593 | if (!try_lock_hpte(hp, HPTE_V_HVLOCK)) { | |
594 | if (n) | |
595 | break; | |
596 | while (!try_lock_hpte(hp, HPTE_V_HVLOCK)) | |
597 | cpu_relax(); | |
598 | } | |
599 | found = 0; | |
6f22bd32 | 600 | hp0 = be64_to_cpu(hp[0]); |
a56ee9f8 | 601 | hp1 = be64_to_cpu(hp[1]); |
abb7c7dd PM |
602 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
603 | hp0 = hpte_new_to_old_v(hp0, hp1); | |
604 | hp1 = hpte_new_to_old_r(hp1); | |
605 | } | |
6f22bd32 | 606 | if (hp0 & (HPTE_V_ABSENT | HPTE_V_VALID)) { |
a92bce95 PM |
607 | switch (flags & 3) { |
608 | case 0: /* absolute */ | |
a8606e20 | 609 | found = 1; |
a92bce95 PM |
610 | break; |
611 | case 1: /* andcond */ | |
6f22bd32 | 612 | if (!(hp0 & args[j + 1])) |
a92bce95 PM |
613 | found = 1; |
614 | break; | |
615 | case 2: /* AVPN */ | |
6f22bd32 | 616 | if ((hp0 & ~0x7fUL) == args[j + 1]) |
a92bce95 PM |
617 | found = 1; |
618 | break; | |
619 | } | |
620 | } | |
621 | if (!found) { | |
6f22bd32 | 622 | hp[0] &= ~cpu_to_be64(HPTE_V_HVLOCK); |
a92bce95 PM |
623 | args[j] = ((0x90 | flags) << 56) + pte_index; |
624 | continue; | |
a8606e20 | 625 | } |
a92bce95 PM |
626 | |
627 | args[j] = ((0x80 | flags) << 56) + pte_index; | |
3f9d4f5a | 628 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
44e5f6be | 629 | note_hpte_modification(kvm, rev); |
a92bce95 | 630 | |
6f22bd32 | 631 | if (!(hp0 & HPTE_V_VALID)) { |
bad3b507 PM |
632 | /* insert R and C bits from PTE */ |
633 | rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); | |
634 | args[j] |= rcbits << (56 - 5); | |
51bfd299 | 635 | hp[0] = 0; |
a56ee9f8 YX |
636 | if (is_mmio_hpte(hp0, hp1)) |
637 | atomic64_inc(&kvm->arch.mmio_update); | |
a92bce95 | 638 | continue; |
bad3b507 | 639 | } |
a92bce95 | 640 | |
6f22bd32 AG |
641 | /* leave it locked */ |
642 | hp[0] &= ~cpu_to_be64(HPTE_V_VALID); | |
abb7c7dd | 643 | tlbrb[n] = compute_tlbie_rb(hp0, hp1, pte_index); |
a92bce95 PM |
644 | indexes[n] = j; |
645 | hptes[n] = hp; | |
646 | revs[n] = rev; | |
647 | ++n; | |
a8606e20 | 648 | } |
a92bce95 PM |
649 | |
650 | if (!n) | |
651 | break; | |
652 | ||
653 | /* Now that we've collected a batch, do the tlbies */ | |
54480501 | 654 | do_tlbies(kvm, tlbrb, n, global, true); |
a92bce95 | 655 | |
bad3b507 | 656 | /* Read PTE low words after tlbie to get final R/C values */ |
a92bce95 PM |
657 | for (k = 0; k < n; ++k) { |
658 | j = indexes[k]; | |
659 | pte_index = args[j] & ((1ul << 56) - 1); | |
660 | hp = hptes[k]; | |
661 | rev = revs[k]; | |
6f22bd32 AG |
662 | remove_revmap_chain(kvm, pte_index, rev, |
663 | be64_to_cpu(hp[0]), be64_to_cpu(hp[1])); | |
bad3b507 PM |
664 | rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C); |
665 | args[j] |= rcbits << (56 - 5); | |
a4bd6eb0 | 666 | __unlock_hpte(hp, 0); |
697d3899 | 667 | } |
a8606e20 | 668 | } |
a92bce95 | 669 | |
a8606e20 PM |
670 | return ret; |
671 | } | |
6165d5dd | 672 | EXPORT_SYMBOL_GPL(kvmppc_h_bulk_remove); |
a8606e20 PM |
673 | |
674 | long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, | |
6c12c437 | 675 | unsigned long pte_index, unsigned long avpn) |
a8606e20 PM |
676 | { |
677 | struct kvm *kvm = vcpu->kvm; | |
6f22bd32 | 678 | __be64 *hpte; |
8936dda4 PM |
679 | struct revmap_entry *rev; |
680 | unsigned long v, r, rb, mask, bits; | |
abb7c7dd | 681 | u64 pte_v, pte_r; |
a8606e20 | 682 | |
65dae540 PM |
683 | if (kvm_is_radix(kvm)) |
684 | return H_FUNCTION; | |
3d089f84 | 685 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
a8606e20 | 686 | return H_PARAMETER; |
697d3899 | 687 | |
3f9d4f5a | 688 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); |
075295dd | 689 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
a8606e20 | 690 | cpu_relax(); |
abb7c7dd PM |
691 | v = pte_v = be64_to_cpu(hpte[0]); |
692 | if (cpu_has_feature(CPU_FTR_ARCH_300)) | |
693 | v = hpte_new_to_old_v(v, be64_to_cpu(hpte[1])); | |
694 | if ((v & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 || | |
695 | ((flags & H_AVPN) && (v & ~0x7fUL) != avpn)) { | |
696 | __unlock_hpte(hpte, pte_v); | |
a8606e20 PM |
697 | return H_NOT_FOUND; |
698 | } | |
697d3899 | 699 | |
abb7c7dd | 700 | pte_r = be64_to_cpu(hpte[1]); |
8936dda4 PM |
701 | bits = (flags << 55) & HPTE_R_PP0; |
702 | bits |= (flags << 48) & HPTE_R_KEY_HI; | |
703 | bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO); | |
704 | ||
705 | /* Update guest view of 2nd HPTE dword */ | |
706 | mask = HPTE_R_PP0 | HPTE_R_PP | HPTE_R_N | | |
707 | HPTE_R_KEY_HI | HPTE_R_KEY_LO; | |
3f9d4f5a | 708 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
8936dda4 PM |
709 | if (rev) { |
710 | r = (rev->guest_rpte & ~mask) | bits; | |
711 | rev->guest_rpte = r; | |
44e5f6be | 712 | note_hpte_modification(kvm, rev); |
8936dda4 | 713 | } |
8936dda4 PM |
714 | |
715 | /* Update HPTE */ | |
697d3899 | 716 | if (v & HPTE_V_VALID) { |
1cc8ed0b | 717 | /* |
b4a83900 PM |
718 | * If the page is valid, don't let it transition from |
719 | * readonly to writable. If it should be writable, we'll | |
720 | * take a trap and let the page fault code sort it out. | |
1cc8ed0b | 721 | */ |
abb7c7dd PM |
722 | r = (pte_r & ~mask) | bits; |
723 | if (hpte_is_writable(r) && !hpte_is_writable(pte_r)) | |
b4a83900 PM |
724 | r = hpte_make_readonly(r); |
725 | /* If the PTE is changing, invalidate it first */ | |
abb7c7dd | 726 | if (r != pte_r) { |
b4a83900 | 727 | rb = compute_tlbie_rb(v, r, pte_index); |
abb7c7dd | 728 | hpte[0] = cpu_to_be64((pte_v & ~HPTE_V_VALID) | |
b4a83900 | 729 | HPTE_V_ABSENT); |
76b03dc0 | 730 | do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true); |
f064a0de PM |
731 | /* Don't lose R/C bit updates done by hardware */ |
732 | r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C); | |
b4a83900 | 733 | hpte[1] = cpu_to_be64(r); |
1cc8ed0b | 734 | } |
a8606e20 | 735 | } |
abb7c7dd | 736 | unlock_hpte(hpte, pte_v & ~HPTE_V_HVLOCK); |
a8606e20 | 737 | asm volatile("ptesync" : : : "memory"); |
abb7c7dd | 738 | if (is_mmio_hpte(v, pte_r)) |
a56ee9f8 YX |
739 | atomic64_inc(&kvm->arch.mmio_update); |
740 | ||
a8606e20 PM |
741 | return H_SUCCESS; |
742 | } | |
6165d5dd | 743 | EXPORT_SYMBOL_GPL(kvmppc_h_protect); |
a8606e20 | 744 | |
a8606e20 PM |
745 | long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, |
746 | unsigned long pte_index) | |
747 | { | |
748 | struct kvm *kvm = vcpu->kvm; | |
6f22bd32 AG |
749 | __be64 *hpte; |
750 | unsigned long v, r; | |
a8606e20 | 751 | int i, n = 1; |
8936dda4 | 752 | struct revmap_entry *rev = NULL; |
a8606e20 | 753 | |
65dae540 PM |
754 | if (kvm_is_radix(kvm)) |
755 | return H_FUNCTION; | |
3d089f84 | 756 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
a8606e20 PM |
757 | return H_PARAMETER; |
758 | if (flags & H_READ_4) { | |
759 | pte_index &= ~3; | |
760 | n = 4; | |
761 | } | |
3f9d4f5a | 762 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
a8606e20 | 763 | for (i = 0; i < n; ++i, ++pte_index) { |
3f9d4f5a | 764 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); |
6f22bd32 AG |
765 | v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK; |
766 | r = be64_to_cpu(hpte[1]); | |
abb7c7dd PM |
767 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
768 | v = hpte_new_to_old_v(v, r); | |
769 | r = hpte_new_to_old_r(r); | |
770 | } | |
697d3899 PM |
771 | if (v & HPTE_V_ABSENT) { |
772 | v &= ~HPTE_V_ABSENT; | |
773 | v |= HPTE_V_VALID; | |
774 | } | |
44e5f6be | 775 | if (v & HPTE_V_VALID) { |
bad3b507 | 776 | r = rev[i].guest_rpte | (r & (HPTE_R_R | HPTE_R_C)); |
44e5f6be PM |
777 | r &= ~HPTE_GR_RESERVED; |
778 | } | |
0e85b7df JN |
779 | kvmppc_set_gpr(vcpu, 4 + i * 2, v); |
780 | kvmppc_set_gpr(vcpu, 5 + i * 2, r); | |
a8606e20 PM |
781 | } |
782 | return H_SUCCESS; | |
783 | } | |
6165d5dd | 784 | EXPORT_SYMBOL_GPL(kvmppc_h_read); |
697d3899 | 785 | |
cdeee518 PM |
786 | long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, |
787 | unsigned long pte_index) | |
788 | { | |
789 | struct kvm *kvm = vcpu->kvm; | |
790 | __be64 *hpte; | |
791 | unsigned long v, r, gr; | |
792 | struct revmap_entry *rev; | |
793 | unsigned long *rmap; | |
794 | long ret = H_NOT_FOUND; | |
795 | ||
65dae540 PM |
796 | if (kvm_is_radix(kvm)) |
797 | return H_FUNCTION; | |
3d089f84 | 798 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
cdeee518 PM |
799 | return H_PARAMETER; |
800 | ||
3f9d4f5a DG |
801 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
802 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); | |
cdeee518 PM |
803 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
804 | cpu_relax(); | |
805 | v = be64_to_cpu(hpte[0]); | |
806 | r = be64_to_cpu(hpte[1]); | |
807 | if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT))) | |
808 | goto out; | |
809 | ||
810 | gr = rev->guest_rpte; | |
811 | if (rev->guest_rpte & HPTE_R_R) { | |
812 | rev->guest_rpte &= ~HPTE_R_R; | |
813 | note_hpte_modification(kvm, rev); | |
814 | } | |
815 | if (v & HPTE_V_VALID) { | |
816 | gr |= r & (HPTE_R_R | HPTE_R_C); | |
817 | if (r & HPTE_R_R) { | |
818 | kvmppc_clear_ref_hpte(kvm, hpte, pte_index); | |
e641a317 | 819 | rmap = revmap_for_hpte(kvm, v, gr, NULL, NULL); |
cdeee518 PM |
820 | if (rmap) { |
821 | lock_rmap(rmap); | |
822 | *rmap |= KVMPPC_RMAP_REFERENCED; | |
823 | unlock_rmap(rmap); | |
824 | } | |
825 | } | |
826 | } | |
0e85b7df | 827 | kvmppc_set_gpr(vcpu, 4, gr); |
cdeee518 PM |
828 | ret = H_SUCCESS; |
829 | out: | |
830 | unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); | |
831 | return ret; | |
832 | } | |
6165d5dd | 833 | EXPORT_SYMBOL_GPL(kvmppc_h_clear_ref); |
cdeee518 PM |
834 | |
835 | long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, | |
836 | unsigned long pte_index) | |
837 | { | |
838 | struct kvm *kvm = vcpu->kvm; | |
839 | __be64 *hpte; | |
840 | unsigned long v, r, gr; | |
841 | struct revmap_entry *rev; | |
cdeee518 PM |
842 | long ret = H_NOT_FOUND; |
843 | ||
65dae540 PM |
844 | if (kvm_is_radix(kvm)) |
845 | return H_FUNCTION; | |
3d089f84 | 846 | if (pte_index >= kvmppc_hpt_npte(&kvm->arch.hpt)) |
cdeee518 PM |
847 | return H_PARAMETER; |
848 | ||
3f9d4f5a DG |
849 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[pte_index]); |
850 | hpte = (__be64 *)(kvm->arch.hpt.virt + (pte_index << 4)); | |
cdeee518 PM |
851 | while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) |
852 | cpu_relax(); | |
853 | v = be64_to_cpu(hpte[0]); | |
854 | r = be64_to_cpu(hpte[1]); | |
855 | if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT))) | |
856 | goto out; | |
857 | ||
858 | gr = rev->guest_rpte; | |
859 | if (gr & HPTE_R_C) { | |
860 | rev->guest_rpte &= ~HPTE_R_C; | |
861 | note_hpte_modification(kvm, rev); | |
862 | } | |
863 | if (v & HPTE_V_VALID) { | |
864 | /* need to make it temporarily absent so C is stable */ | |
865 | hpte[0] |= cpu_to_be64(HPTE_V_ABSENT); | |
866 | kvmppc_invalidate_hpte(kvm, hpte, pte_index); | |
867 | r = be64_to_cpu(hpte[1]); | |
868 | gr |= r & (HPTE_R_R | HPTE_R_C); | |
869 | if (r & HPTE_R_C) { | |
cdeee518 PM |
870 | hpte[1] = cpu_to_be64(r & ~HPTE_R_C); |
871 | eieio(); | |
e641a317 | 872 | kvmppc_set_dirty_from_hpte(kvm, v, gr); |
cdeee518 PM |
873 | } |
874 | } | |
0e85b7df | 875 | kvmppc_set_gpr(vcpu, 4, gr); |
cdeee518 PM |
876 | ret = H_SUCCESS; |
877 | out: | |
878 | unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); | |
879 | return ret; | |
880 | } | |
6165d5dd | 881 | EXPORT_SYMBOL_GPL(kvmppc_h_clear_mod); |
cdeee518 | 882 | |
9fd4236f AK |
883 | static int kvmppc_get_hpa(struct kvm_vcpu *vcpu, unsigned long mmu_seq, |
884 | unsigned long gpa, int writing, unsigned long *hpa, | |
eadfb1c5 SJS |
885 | struct kvm_memory_slot **memslot_p) |
886 | { | |
887 | struct kvm *kvm = vcpu->kvm; | |
888 | struct kvm_memory_slot *memslot; | |
889 | unsigned long gfn, hva, pa, psize = PAGE_SHIFT; | |
890 | unsigned int shift; | |
891 | pte_t *ptep, pte; | |
892 | ||
893 | /* Find the memslot for this address */ | |
894 | gfn = gpa >> PAGE_SHIFT; | |
895 | memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); | |
896 | if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) | |
897 | return H_PARAMETER; | |
898 | ||
899 | /* Translate to host virtual address */ | |
900 | hva = __gfn_to_hva_memslot(memslot, gfn); | |
901 | ||
902 | /* Try to find the host pte for that virtual address */ | |
9fd4236f | 903 | ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift); |
eadfb1c5 SJS |
904 | if (!ptep) |
905 | return H_TOO_HARD; | |
906 | pte = kvmppc_read_update_linux_pte(ptep, writing); | |
907 | if (!pte_present(pte)) | |
908 | return H_TOO_HARD; | |
909 | ||
910 | /* Convert to a physical address */ | |
911 | if (shift) | |
912 | psize = 1UL << shift; | |
913 | pa = pte_pfn(pte) << PAGE_SHIFT; | |
914 | pa |= hva & (psize - 1); | |
915 | pa |= gpa & ~PAGE_MASK; | |
916 | ||
917 | if (hpa) | |
918 | *hpa = pa; | |
919 | if (memslot_p) | |
920 | *memslot_p = memslot; | |
921 | ||
922 | return H_SUCCESS; | |
923 | } | |
924 | ||
925 | static long kvmppc_do_h_page_init_zero(struct kvm_vcpu *vcpu, | |
926 | unsigned long dest) | |
927 | { | |
928 | struct kvm_memory_slot *memslot; | |
929 | struct kvm *kvm = vcpu->kvm; | |
930 | unsigned long pa, mmu_seq; | |
931 | long ret = H_SUCCESS; | |
932 | int i; | |
933 | ||
934 | /* Used later to detect if we might have been invalidated */ | |
20ec3ebd | 935 | mmu_seq = kvm->mmu_invalidate_seq; |
eadfb1c5 SJS |
936 | smp_rmb(); |
937 | ||
87013f9c | 938 | arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock); |
9fd4236f AK |
939 | |
940 | ret = kvmppc_get_hpa(vcpu, mmu_seq, dest, 1, &pa, &memslot); | |
941 | if (ret != H_SUCCESS) | |
eadfb1c5 | 942 | goto out_unlock; |
eadfb1c5 SJS |
943 | |
944 | /* Zero the page */ | |
945 | for (i = 0; i < SZ_4K; i += L1_CACHE_BYTES, pa += L1_CACHE_BYTES) | |
946 | dcbz((void *)pa); | |
947 | kvmppc_update_dirty_map(memslot, dest >> PAGE_SHIFT, PAGE_SIZE); | |
948 | ||
949 | out_unlock: | |
87013f9c | 950 | arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock); |
eadfb1c5 SJS |
951 | return ret; |
952 | } | |
953 | ||
954 | static long kvmppc_do_h_page_init_copy(struct kvm_vcpu *vcpu, | |
955 | unsigned long dest, unsigned long src) | |
956 | { | |
957 | unsigned long dest_pa, src_pa, mmu_seq; | |
958 | struct kvm_memory_slot *dest_memslot; | |
959 | struct kvm *kvm = vcpu->kvm; | |
960 | long ret = H_SUCCESS; | |
961 | ||
962 | /* Used later to detect if we might have been invalidated */ | |
20ec3ebd | 963 | mmu_seq = kvm->mmu_invalidate_seq; |
eadfb1c5 SJS |
964 | smp_rmb(); |
965 | ||
9fd4236f AK |
966 | arch_spin_lock(&kvm->mmu_lock.rlock.raw_lock); |
967 | ret = kvmppc_get_hpa(vcpu, mmu_seq, dest, 1, &dest_pa, &dest_memslot); | |
eadfb1c5 | 968 | if (ret != H_SUCCESS) |
9fd4236f | 969 | goto out_unlock; |
eadfb1c5 | 970 | |
9fd4236f AK |
971 | ret = kvmppc_get_hpa(vcpu, mmu_seq, src, 0, &src_pa, NULL); |
972 | if (ret != H_SUCCESS) | |
eadfb1c5 | 973 | goto out_unlock; |
eadfb1c5 SJS |
974 | |
975 | /* Copy the page */ | |
976 | memcpy((void *)dest_pa, (void *)src_pa, SZ_4K); | |
977 | ||
978 | kvmppc_update_dirty_map(dest_memslot, dest >> PAGE_SHIFT, PAGE_SIZE); | |
979 | ||
980 | out_unlock: | |
87013f9c | 981 | arch_spin_unlock(&kvm->mmu_lock.rlock.raw_lock); |
eadfb1c5 SJS |
982 | return ret; |
983 | } | |
984 | ||
985 | long kvmppc_rm_h_page_init(struct kvm_vcpu *vcpu, unsigned long flags, | |
986 | unsigned long dest, unsigned long src) | |
987 | { | |
988 | struct kvm *kvm = vcpu->kvm; | |
989 | u64 pg_mask = SZ_4K - 1; /* 4K page size */ | |
990 | long ret = H_SUCCESS; | |
991 | ||
992 | /* Don't handle radix mode here, go up to the virtual mode handler */ | |
993 | if (kvm_is_radix(kvm)) | |
994 | return H_TOO_HARD; | |
995 | ||
996 | /* Check for invalid flags (H_PAGE_SET_LOANED covers all CMO flags) */ | |
997 | if (flags & ~(H_ICACHE_INVALIDATE | H_ICACHE_SYNCHRONIZE | | |
998 | H_ZERO_PAGE | H_COPY_PAGE | H_PAGE_SET_LOANED)) | |
999 | return H_PARAMETER; | |
1000 | ||
1001 | /* dest (and src if copy_page flag set) must be page aligned */ | |
1002 | if ((dest & pg_mask) || ((flags & H_COPY_PAGE) && (src & pg_mask))) | |
1003 | return H_PARAMETER; | |
1004 | ||
1005 | /* zero and/or copy the page as determined by the flags */ | |
1006 | if (flags & H_COPY_PAGE) | |
1007 | ret = kvmppc_do_h_page_init_copy(vcpu, dest, src); | |
1008 | else if (flags & H_ZERO_PAGE) | |
1009 | ret = kvmppc_do_h_page_init_zero(vcpu, dest); | |
1010 | ||
1011 | /* We can ignore the other flags */ | |
1012 | ||
1013 | return ret; | |
1014 | } | |
1015 | ||
6f22bd32 | 1016 | void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep, |
342d3db7 PM |
1017 | unsigned long pte_index) |
1018 | { | |
1019 | unsigned long rb; | |
abb7c7dd | 1020 | u64 hp0, hp1; |
342d3db7 | 1021 | |
6f22bd32 | 1022 | hptep[0] &= ~cpu_to_be64(HPTE_V_VALID); |
abb7c7dd PM |
1023 | hp0 = be64_to_cpu(hptep[0]); |
1024 | hp1 = be64_to_cpu(hptep[1]); | |
1025 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
1026 | hp0 = hpte_new_to_old_v(hp0, hp1); | |
1027 | hp1 = hpte_new_to_old_r(hp1); | |
1028 | } | |
1029 | rb = compute_tlbie_rb(hp0, hp1, pte_index); | |
54480501 | 1030 | do_tlbies(kvm, &rb, 1, 1, true); |
342d3db7 PM |
1031 | } |
1032 | EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte); | |
1033 | ||
6f22bd32 | 1034 | void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep, |
55514893 PM |
1035 | unsigned long pte_index) |
1036 | { | |
1037 | unsigned long rb; | |
1038 | unsigned char rbyte; | |
abb7c7dd | 1039 | u64 hp0, hp1; |
55514893 | 1040 | |
abb7c7dd PM |
1041 | hp0 = be64_to_cpu(hptep[0]); |
1042 | hp1 = be64_to_cpu(hptep[1]); | |
1043 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
1044 | hp0 = hpte_new_to_old_v(hp0, hp1); | |
1045 | hp1 = hpte_new_to_old_r(hp1); | |
1046 | } | |
1047 | rb = compute_tlbie_rb(hp0, hp1, pte_index); | |
6f22bd32 | 1048 | rbyte = (be64_to_cpu(hptep[1]) & ~HPTE_R_R) >> 8; |
55514893 PM |
1049 | /* modify only the second-last byte, which contains the ref bit */ |
1050 | *((char *)hptep + 14) = rbyte; | |
54480501 | 1051 | do_tlbies(kvm, &rb, 1, 1, false); |
55514893 PM |
1052 | } |
1053 | EXPORT_SYMBOL_GPL(kvmppc_clear_ref_hpte); | |
1054 | ||
697d3899 PM |
1055 | static int slb_base_page_shift[4] = { |
1056 | 24, /* 16M */ | |
1057 | 16, /* 64k */ | |
1058 | 34, /* 16G */ | |
1059 | 20, /* 1M, unsupported */ | |
1060 | }; | |
1061 | ||
a56ee9f8 YX |
1062 | static struct mmio_hpte_cache_entry *mmio_cache_search(struct kvm_vcpu *vcpu, |
1063 | unsigned long eaddr, unsigned long slb_v, long mmio_update) | |
1064 | { | |
1065 | struct mmio_hpte_cache_entry *entry = NULL; | |
1066 | unsigned int pshift; | |
1067 | unsigned int i; | |
1068 | ||
1069 | for (i = 0; i < MMIO_HPTE_CACHE_SIZE; i++) { | |
1070 | entry = &vcpu->arch.mmio_cache.entry[i]; | |
1071 | if (entry->mmio_update == mmio_update) { | |
1072 | pshift = entry->slb_base_pshift; | |
1073 | if ((entry->eaddr >> pshift) == (eaddr >> pshift) && | |
1074 | entry->slb_v == slb_v) | |
1075 | return entry; | |
1076 | } | |
1077 | } | |
1078 | return NULL; | |
1079 | } | |
1080 | ||
1081 | static struct mmio_hpte_cache_entry * | |
1082 | next_mmio_cache_entry(struct kvm_vcpu *vcpu) | |
1083 | { | |
1084 | unsigned int index = vcpu->arch.mmio_cache.index; | |
1085 | ||
1086 | vcpu->arch.mmio_cache.index++; | |
1087 | if (vcpu->arch.mmio_cache.index == MMIO_HPTE_CACHE_SIZE) | |
1088 | vcpu->arch.mmio_cache.index = 0; | |
1089 | ||
1090 | return &vcpu->arch.mmio_cache.entry[index]; | |
1091 | } | |
1092 | ||
91648ec0 | 1093 | /* When called from virtmode, this func should be protected by |
1094 | * preempt_disable(), otherwise, the holding of HPTE_V_HVLOCK | |
1095 | * can trigger deadlock issue. | |
1096 | */ | |
697d3899 PM |
1097 | long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, |
1098 | unsigned long valid) | |
1099 | { | |
1100 | unsigned int i; | |
1101 | unsigned int pshift; | |
1102 | unsigned long somask; | |
1103 | unsigned long vsid, hash; | |
1104 | unsigned long avpn; | |
6f22bd32 | 1105 | __be64 *hpte; |
697d3899 | 1106 | unsigned long mask, val; |
abb7c7dd | 1107 | unsigned long v, r, orig_v; |
697d3899 PM |
1108 | |
1109 | /* Get page shift, work out hash and AVPN etc. */ | |
1110 | mask = SLB_VSID_B | HPTE_V_AVPN | HPTE_V_SECONDARY; | |
1111 | val = 0; | |
1112 | pshift = 12; | |
1113 | if (slb_v & SLB_VSID_L) { | |
1114 | mask |= HPTE_V_LARGE; | |
1115 | val |= HPTE_V_LARGE; | |
1116 | pshift = slb_base_page_shift[(slb_v & SLB_VSID_LP) >> 4]; | |
1117 | } | |
1118 | if (slb_v & SLB_VSID_B_1T) { | |
1119 | somask = (1UL << 40) - 1; | |
1120 | vsid = (slb_v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T; | |
1121 | vsid ^= vsid << 25; | |
1122 | } else { | |
1123 | somask = (1UL << 28) - 1; | |
1124 | vsid = (slb_v & ~SLB_VSID_B) >> SLB_VSID_SHIFT; | |
1125 | } | |
3d089f84 | 1126 | hash = (vsid ^ ((eaddr & somask) >> pshift)) & kvmppc_hpt_mask(&kvm->arch.hpt); |
697d3899 PM |
1127 | avpn = slb_v & ~(somask >> 16); /* also includes B */ |
1128 | avpn |= (eaddr & somask) >> 16; | |
1129 | ||
1130 | if (pshift >= 24) | |
1131 | avpn &= ~((1UL << (pshift - 16)) - 1); | |
1132 | else | |
1133 | avpn &= ~0x7fUL; | |
1134 | val |= avpn; | |
1135 | ||
1136 | for (;;) { | |
3f9d4f5a | 1137 | hpte = (__be64 *)(kvm->arch.hpt.virt + (hash << 7)); |
697d3899 PM |
1138 | |
1139 | for (i = 0; i < 16; i += 2) { | |
1140 | /* Read the PTE racily */ | |
6f22bd32 | 1141 | v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK; |
abb7c7dd PM |
1142 | if (cpu_has_feature(CPU_FTR_ARCH_300)) |
1143 | v = hpte_new_to_old_v(v, be64_to_cpu(hpte[i+1])); | |
697d3899 PM |
1144 | |
1145 | /* Check valid/absent, hash, segment size and AVPN */ | |
1146 | if (!(v & valid) || (v & mask) != val) | |
1147 | continue; | |
1148 | ||
1149 | /* Lock the PTE and read it under the lock */ | |
1150 | while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK)) | |
1151 | cpu_relax(); | |
abb7c7dd | 1152 | v = orig_v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK; |
6f22bd32 | 1153 | r = be64_to_cpu(hpte[i+1]); |
abb7c7dd PM |
1154 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
1155 | v = hpte_new_to_old_v(v, r); | |
1156 | r = hpte_new_to_old_r(r); | |
1157 | } | |
697d3899 PM |
1158 | |
1159 | /* | |
341acbb3 | 1160 | * Check the HPTE again, including base page size |
697d3899 PM |
1161 | */ |
1162 | if ((v & valid) && (v & mask) == val && | |
8dc6cca5 | 1163 | kvmppc_hpte_base_page_shift(v, r) == pshift) |
697d3899 PM |
1164 | /* Return with the HPTE still locked */ |
1165 | return (hash << 3) + (i >> 1); | |
1166 | ||
abb7c7dd | 1167 | __unlock_hpte(&hpte[i], orig_v); |
697d3899 PM |
1168 | } |
1169 | ||
1170 | if (val & HPTE_V_SECONDARY) | |
1171 | break; | |
1172 | val |= HPTE_V_SECONDARY; | |
3d089f84 | 1173 | hash = hash ^ kvmppc_hpt_mask(&kvm->arch.hpt); |
697d3899 PM |
1174 | } |
1175 | return -1; | |
1176 | } | |
1177 | EXPORT_SYMBOL(kvmppc_hv_find_lock_hpte); | |
1178 | ||
1179 | /* | |
1180 | * Called in real mode to check whether an HPTE not found fault | |
4cf302bc PM |
1181 | * is due to accessing a paged-out page or an emulated MMIO page, |
1182 | * or if a protection fault is due to accessing a page that the | |
1183 | * guest wanted read/write access to but which we made read-only. | |
697d3899 PM |
1184 | * Returns a possibly modified status (DSISR) value if not |
1185 | * (i.e. pass the interrupt to the guest), | |
1186 | * -1 to pass the fault up to host kernel mode code, -2 to do that | |
342d3db7 | 1187 | * and also load the instruction word (for MMIO emulation), |
697d3899 PM |
1188 | * or 0 if we should make the guest retry the access. |
1189 | */ | |
1190 | long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, | |
342d3db7 | 1191 | unsigned long slb_v, unsigned int status, bool data) |
697d3899 PM |
1192 | { |
1193 | struct kvm *kvm = vcpu->kvm; | |
1194 | long int index; | |
abb7c7dd | 1195 | unsigned long v, r, gr, orig_v; |
6f22bd32 | 1196 | __be64 *hpte; |
697d3899 PM |
1197 | unsigned long valid; |
1198 | struct revmap_entry *rev; | |
1199 | unsigned long pp, key; | |
a56ee9f8 YX |
1200 | struct mmio_hpte_cache_entry *cache_entry = NULL; |
1201 | long mmio_update = 0; | |
697d3899 | 1202 | |
4cf302bc PM |
1203 | /* For protection fault, expect to find a valid HPTE */ |
1204 | valid = HPTE_V_VALID; | |
a56ee9f8 | 1205 | if (status & DSISR_NOHPTE) { |
4cf302bc | 1206 | valid |= HPTE_V_ABSENT; |
a56ee9f8 YX |
1207 | mmio_update = atomic64_read(&kvm->arch.mmio_update); |
1208 | cache_entry = mmio_cache_search(vcpu, addr, slb_v, mmio_update); | |
4cf302bc | 1209 | } |
a56ee9f8 YX |
1210 | if (cache_entry) { |
1211 | index = cache_entry->pte_index; | |
1212 | v = cache_entry->hpte_v; | |
1213 | r = cache_entry->hpte_r; | |
1214 | gr = cache_entry->rpte; | |
1215 | } else { | |
1216 | index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid); | |
1217 | if (index < 0) { | |
1218 | if (status & DSISR_NOHPTE) | |
1219 | return status; /* there really was no HPTE */ | |
1220 | return 0; /* for prot fault, HPTE disappeared */ | |
1221 | } | |
3f9d4f5a | 1222 | hpte = (__be64 *)(kvm->arch.hpt.virt + (index << 4)); |
abb7c7dd | 1223 | v = orig_v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK; |
a56ee9f8 | 1224 | r = be64_to_cpu(hpte[1]); |
abb7c7dd PM |
1225 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { |
1226 | v = hpte_new_to_old_v(v, r); | |
1227 | r = hpte_new_to_old_r(r); | |
1228 | } | |
3f9d4f5a | 1229 | rev = real_vmalloc_addr(&kvm->arch.hpt.rev[index]); |
a56ee9f8 | 1230 | gr = rev->guest_rpte; |
697d3899 | 1231 | |
abb7c7dd | 1232 | unlock_hpte(hpte, orig_v); |
a56ee9f8 | 1233 | } |
697d3899 | 1234 | |
4cf302bc PM |
1235 | /* For not found, if the HPTE is valid by now, retry the instruction */ |
1236 | if ((status & DSISR_NOHPTE) && (v & HPTE_V_VALID)) | |
697d3899 PM |
1237 | return 0; |
1238 | ||
1239 | /* Check access permissions to the page */ | |
1240 | pp = gr & (HPTE_R_PP0 | HPTE_R_PP); | |
1241 | key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS; | |
342d3db7 PM |
1242 | status &= ~DSISR_NOHPTE; /* DSISR_NOHPTE == SRR1_ISI_NOPT */ |
1243 | if (!data) { | |
1244 | if (gr & (HPTE_R_N | HPTE_R_G)) | |
b691505e | 1245 | return status | SRR1_ISI_N_G_OR_CIP; |
342d3db7 PM |
1246 | if (!hpte_read_permission(pp, slb_v & key)) |
1247 | return status | SRR1_ISI_PROT; | |
1248 | } else if (status & DSISR_ISSTORE) { | |
697d3899 PM |
1249 | /* check write permission */ |
1250 | if (!hpte_write_permission(pp, slb_v & key)) | |
342d3db7 | 1251 | return status | DSISR_PROTFAULT; |
697d3899 PM |
1252 | } else { |
1253 | if (!hpte_read_permission(pp, slb_v & key)) | |
342d3db7 | 1254 | return status | DSISR_PROTFAULT; |
697d3899 PM |
1255 | } |
1256 | ||
1257 | /* Check storage key, if applicable */ | |
342d3db7 | 1258 | if (data && (vcpu->arch.shregs.msr & MSR_DR)) { |
697d3899 PM |
1259 | unsigned int perm = hpte_get_skey_perm(gr, vcpu->arch.amr); |
1260 | if (status & DSISR_ISSTORE) | |
1261 | perm >>= 1; | |
1262 | if (perm & 1) | |
342d3db7 | 1263 | return status | DSISR_KEYFAULT; |
697d3899 PM |
1264 | } |
1265 | ||
1266 | /* Save HPTE info for virtual-mode handler */ | |
1267 | vcpu->arch.pgfault_addr = addr; | |
1268 | vcpu->arch.pgfault_index = index; | |
1269 | vcpu->arch.pgfault_hpte[0] = v; | |
1270 | vcpu->arch.pgfault_hpte[1] = r; | |
a56ee9f8 | 1271 | vcpu->arch.pgfault_cache = cache_entry; |
697d3899 | 1272 | |
342d3db7 | 1273 | /* Check the storage key to see if it is possibly emulated MMIO */ |
a56ee9f8 YX |
1274 | if ((r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) == |
1275 | (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) { | |
1276 | if (!cache_entry) { | |
1277 | unsigned int pshift = 12; | |
1278 | unsigned int pshift_index; | |
1279 | ||
1280 | if (slb_v & SLB_VSID_L) { | |
1281 | pshift_index = ((slb_v & SLB_VSID_LP) >> 4); | |
1282 | pshift = slb_base_page_shift[pshift_index]; | |
1283 | } | |
1284 | cache_entry = next_mmio_cache_entry(vcpu); | |
1285 | cache_entry->eaddr = addr; | |
1286 | cache_entry->slb_base_pshift = pshift; | |
1287 | cache_entry->pte_index = index; | |
1288 | cache_entry->hpte_v = v; | |
1289 | cache_entry->hpte_r = r; | |
1290 | cache_entry->rpte = gr; | |
1291 | cache_entry->slb_v = slb_v; | |
1292 | cache_entry->mmio_update = mmio_update; | |
1293 | } | |
1294 | if (data && (vcpu->arch.shregs.msr & MSR_IR)) | |
1295 | return -2; /* MMIO emulation - load instr word */ | |
1296 | } | |
697d3899 PM |
1297 | |
1298 | return -1; /* send fault up to host kernel mode */ | |
697d3899 | 1299 | } |
6165d5dd | 1300 | EXPORT_SYMBOL_GPL(kvmppc_hpte_hv_fault); |