Commit | Line | Data |
---|---|---|
d32154f1 AG |
1 | /* |
2 | * Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved. | |
3 | * | |
4 | * Authors: | |
5 | * Alexander Graf <agraf@suse.de> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License, version 2, as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program; if not, write to the Free Software | |
18 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |
19 | */ | |
20 | ||
21 | #include <linux/kvm_host.h> | |
22 | ||
23 | #include <asm/kvm_ppc.h> | |
24 | #include <asm/kvm_book3s.h> | |
25 | #include <asm/mmu-hash32.h> | |
26 | #include <asm/machdep.h> | |
27 | #include <asm/mmu_context.h> | |
28 | #include <asm/hw_irq.h> | |
29 | ||
30 | /* #define DEBUG_MMU */ | |
31 | /* #define DEBUG_SR */ | |
32 | ||
33 | #ifdef DEBUG_MMU | |
34 | #define dprintk_mmu(a, ...) printk(KERN_INFO a, __VA_ARGS__) | |
35 | #else | |
36 | #define dprintk_mmu(a, ...) do { } while(0) | |
37 | #endif | |
38 | ||
39 | #ifdef DEBUG_SR | |
40 | #define dprintk_sr(a, ...) printk(KERN_INFO a, __VA_ARGS__) | |
41 | #else | |
42 | #define dprintk_sr(a, ...) do { } while(0) | |
43 | #endif | |
44 | ||
45 | #if PAGE_SHIFT != 12 | |
46 | #error Unknown page size | |
47 | #endif | |
48 | ||
49 | #ifdef CONFIG_SMP | |
50 | #error XXX need to grab mmu_hash_lock | |
51 | #endif | |
52 | ||
53 | #ifdef CONFIG_PTE_64BIT | |
54 | #error Only 32 bit pages are supported for now | |
55 | #endif | |
56 | ||
251585b5 AG |
57 | static ulong htab; |
58 | static u32 htabmask; | |
59 | ||
fef093be | 60 | void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte) |
d32154f1 AG |
61 | { |
62 | volatile u32 *pteg; | |
63 | ||
fef093be | 64 | /* Remove from host HTAB */ |
d32154f1 | 65 | pteg = (u32*)pte->slot; |
d32154f1 | 66 | pteg[0] = 0; |
fef093be AG |
67 | |
68 | /* And make sure it's gone from the TLB too */ | |
d32154f1 AG |
69 | asm volatile ("sync"); |
70 | asm volatile ("tlbie %0" : : "r" (pte->pte.eaddr) : "memory"); | |
71 | asm volatile ("sync"); | |
72 | asm volatile ("tlbsync"); | |
d32154f1 AG |
73 | } |
74 | ||
75 | /* We keep 512 gvsid->hvsid entries, mapping the guest ones to the array using | |
76 | * a hash, so we don't waste cycles on looping */ | |
77 | static u16 kvmppc_sid_hash(struct kvm_vcpu *vcpu, u64 gvsid) | |
78 | { | |
b9877ce2 AG |
79 | return (u16)(((gvsid >> (SID_MAP_BITS * 7)) & SID_MAP_MASK) ^ |
80 | ((gvsid >> (SID_MAP_BITS * 6)) & SID_MAP_MASK) ^ | |
81 | ((gvsid >> (SID_MAP_BITS * 5)) & SID_MAP_MASK) ^ | |
82 | ((gvsid >> (SID_MAP_BITS * 4)) & SID_MAP_MASK) ^ | |
83 | ((gvsid >> (SID_MAP_BITS * 3)) & SID_MAP_MASK) ^ | |
84 | ((gvsid >> (SID_MAP_BITS * 2)) & SID_MAP_MASK) ^ | |
85 | ((gvsid >> (SID_MAP_BITS * 1)) & SID_MAP_MASK) ^ | |
86 | ((gvsid >> (SID_MAP_BITS * 0)) & SID_MAP_MASK)); | |
d32154f1 AG |
87 | } |
88 | ||
89 | ||
90 | static struct kvmppc_sid_map *find_sid_vsid(struct kvm_vcpu *vcpu, u64 gvsid) | |
91 | { | |
92 | struct kvmppc_sid_map *map; | |
93 | u16 sid_map_mask; | |
94 | ||
666e7252 | 95 | if (vcpu->arch.shared->msr & MSR_PR) |
d32154f1 AG |
96 | gvsid |= VSID_PR; |
97 | ||
98 | sid_map_mask = kvmppc_sid_hash(vcpu, gvsid); | |
99 | map = &to_book3s(vcpu)->sid_map[sid_map_mask]; | |
100 | if (map->guest_vsid == gvsid) { | |
101 | dprintk_sr("SR: Searching 0x%llx -> 0x%llx\n", | |
102 | gvsid, map->host_vsid); | |
103 | return map; | |
104 | } | |
105 | ||
106 | map = &to_book3s(vcpu)->sid_map[SID_MAP_MASK - sid_map_mask]; | |
107 | if (map->guest_vsid == gvsid) { | |
108 | dprintk_sr("SR: Searching 0x%llx -> 0x%llx\n", | |
109 | gvsid, map->host_vsid); | |
110 | return map; | |
111 | } | |
112 | ||
113 | dprintk_sr("SR: Searching 0x%llx -> not found\n", gvsid); | |
114 | return NULL; | |
115 | } | |
116 | ||
d32154f1 AG |
117 | static u32 *kvmppc_mmu_get_pteg(struct kvm_vcpu *vcpu, u32 vsid, u32 eaddr, |
118 | bool primary) | |
119 | { | |
251585b5 AG |
120 | u32 page, hash; |
121 | ulong pteg = htab; | |
d32154f1 AG |
122 | |
123 | page = (eaddr & ~ESID_MASK) >> 12; | |
124 | ||
125 | hash = ((vsid ^ page) << 6); | |
126 | if (!primary) | |
127 | hash = ~hash; | |
128 | ||
d32154f1 AG |
129 | hash &= htabmask; |
130 | ||
131 | pteg |= hash; | |
132 | ||
251585b5 AG |
133 | dprintk_mmu("htab: %lx | hash: %x | htabmask: %x | pteg: %lx\n", |
134 | htab, hash, htabmask, pteg); | |
d32154f1 AG |
135 | |
136 | return (u32*)pteg; | |
137 | } | |
138 | ||
139 | extern char etext[]; | |
140 | ||
93b159b4 PM |
141 | int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *orig_pte, |
142 | bool iswrite) | |
d32154f1 AG |
143 | { |
144 | pfn_t hpaddr; | |
5524a27d | 145 | u64 vpn; |
d32154f1 AG |
146 | u64 vsid; |
147 | struct kvmppc_sid_map *map; | |
148 | volatile u32 *pteg; | |
149 | u32 eaddr = orig_pte->eaddr; | |
150 | u32 pteg0, pteg1; | |
151 | register int rr = 0; | |
152 | bool primary = false; | |
153 | bool evict = false; | |
d32154f1 | 154 | struct hpte_cache *pte; |
468a12c2 | 155 | int r = 0; |
93b159b4 | 156 | bool writable; |
d32154f1 AG |
157 | |
158 | /* Get host physical address for gpa */ | |
93b159b4 PM |
159 | hpaddr = kvmppc_gfn_to_pfn(vcpu, orig_pte->raddr >> PAGE_SHIFT, |
160 | iswrite, &writable); | |
81c52c56 | 161 | if (is_error_noslot_pfn(hpaddr)) { |
af7b4d10 | 162 | printk(KERN_INFO "Couldn't get guest page for gfn %lx!\n", |
d32154f1 | 163 | orig_pte->eaddr); |
468a12c2 AG |
164 | r = -EINVAL; |
165 | goto out; | |
d32154f1 AG |
166 | } |
167 | hpaddr <<= PAGE_SHIFT; | |
168 | ||
169 | /* and write the mapping ea -> hpa into the pt */ | |
170 | vcpu->arch.mmu.esid_to_vsid(vcpu, orig_pte->eaddr >> SID_SHIFT, &vsid); | |
171 | map = find_sid_vsid(vcpu, vsid); | |
172 | if (!map) { | |
173 | kvmppc_mmu_map_segment(vcpu, eaddr); | |
174 | map = find_sid_vsid(vcpu, vsid); | |
175 | } | |
176 | BUG_ON(!map); | |
177 | ||
178 | vsid = map->host_vsid; | |
ce236ab5 AK |
179 | vpn = (vsid << (SID_SHIFT - VPN_SHIFT)) | |
180 | ((eaddr & ~ESID_MASK) >> VPN_SHIFT); | |
d32154f1 AG |
181 | next_pteg: |
182 | if (rr == 16) { | |
183 | primary = !primary; | |
184 | evict = true; | |
185 | rr = 0; | |
186 | } | |
187 | ||
188 | pteg = kvmppc_mmu_get_pteg(vcpu, vsid, eaddr, primary); | |
189 | ||
190 | /* not evicting yet */ | |
191 | if (!evict && (pteg[rr] & PTE_V)) { | |
192 | rr += 2; | |
193 | goto next_pteg; | |
194 | } | |
195 | ||
196 | dprintk_mmu("KVM: old PTEG: %p (%d)\n", pteg, rr); | |
197 | dprintk_mmu("KVM: %08x - %08x\n", pteg[0], pteg[1]); | |
198 | dprintk_mmu("KVM: %08x - %08x\n", pteg[2], pteg[3]); | |
199 | dprintk_mmu("KVM: %08x - %08x\n", pteg[4], pteg[5]); | |
200 | dprintk_mmu("KVM: %08x - %08x\n", pteg[6], pteg[7]); | |
201 | dprintk_mmu("KVM: %08x - %08x\n", pteg[8], pteg[9]); | |
202 | dprintk_mmu("KVM: %08x - %08x\n", pteg[10], pteg[11]); | |
203 | dprintk_mmu("KVM: %08x - %08x\n", pteg[12], pteg[13]); | |
204 | dprintk_mmu("KVM: %08x - %08x\n", pteg[14], pteg[15]); | |
205 | ||
206 | pteg0 = ((eaddr & 0x0fffffff) >> 22) | (vsid << 7) | PTE_V | | |
207 | (primary ? 0 : PTE_SEC); | |
208 | pteg1 = hpaddr | PTE_M | PTE_R | PTE_C; | |
209 | ||
93b159b4 | 210 | if (orig_pte->may_write && writable) { |
d32154f1 AG |
211 | pteg1 |= PP_RWRW; |
212 | mark_page_dirty(vcpu->kvm, orig_pte->raddr >> PAGE_SHIFT); | |
213 | } else { | |
214 | pteg1 |= PP_RWRX; | |
215 | } | |
216 | ||
249ba1ee AG |
217 | if (orig_pte->may_execute) |
218 | kvmppc_mmu_flush_icache(hpaddr >> PAGE_SHIFT); | |
219 | ||
d32154f1 AG |
220 | local_irq_disable(); |
221 | ||
222 | if (pteg[rr]) { | |
223 | pteg[rr] = 0; | |
224 | asm volatile ("sync"); | |
225 | } | |
226 | pteg[rr + 1] = pteg1; | |
227 | pteg[rr] = pteg0; | |
228 | asm volatile ("sync"); | |
229 | ||
230 | local_irq_enable(); | |
231 | ||
232 | dprintk_mmu("KVM: new PTEG: %p\n", pteg); | |
233 | dprintk_mmu("KVM: %08x - %08x\n", pteg[0], pteg[1]); | |
234 | dprintk_mmu("KVM: %08x - %08x\n", pteg[2], pteg[3]); | |
235 | dprintk_mmu("KVM: %08x - %08x\n", pteg[4], pteg[5]); | |
236 | dprintk_mmu("KVM: %08x - %08x\n", pteg[6], pteg[7]); | |
237 | dprintk_mmu("KVM: %08x - %08x\n", pteg[8], pteg[9]); | |
238 | dprintk_mmu("KVM: %08x - %08x\n", pteg[10], pteg[11]); | |
239 | dprintk_mmu("KVM: %08x - %08x\n", pteg[12], pteg[13]); | |
240 | dprintk_mmu("KVM: %08x - %08x\n", pteg[14], pteg[15]); | |
241 | ||
242 | ||
243 | /* Now tell our Shadow PTE code about the new page */ | |
244 | ||
fef093be | 245 | pte = kvmppc_mmu_hpte_cache_next(vcpu); |
d32154f1 AG |
246 | |
247 | dprintk_mmu("KVM: %c%c Map 0x%llx: [%lx] 0x%llx (0x%llx) -> %lx\n", | |
248 | orig_pte->may_write ? 'w' : '-', | |
249 | orig_pte->may_execute ? 'x' : '-', | |
5524a27d | 250 | orig_pte->eaddr, (ulong)pteg, vpn, |
d32154f1 AG |
251 | orig_pte->vpage, hpaddr); |
252 | ||
253 | pte->slot = (ulong)&pteg[rr]; | |
5524a27d | 254 | pte->host_vpn = vpn; |
d32154f1 AG |
255 | pte->pte = *orig_pte; |
256 | pte->pfn = hpaddr >> PAGE_SHIFT; | |
257 | ||
fef093be AG |
258 | kvmppc_mmu_hpte_cache_map(vcpu, pte); |
259 | ||
9b0cb3c8 | 260 | kvm_release_pfn_clean(hpaddr >> PAGE_SHIFT); |
468a12c2 AG |
261 | out: |
262 | return r; | |
d32154f1 AG |
263 | } |
264 | ||
93b159b4 PM |
265 | void kvmppc_mmu_unmap_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) |
266 | { | |
267 | kvmppc_mmu_pte_vflush(vcpu, pte->vpage, 0xfffffffffULL); | |
268 | } | |
269 | ||
d32154f1 AG |
270 | static struct kvmppc_sid_map *create_sid_map(struct kvm_vcpu *vcpu, u64 gvsid) |
271 | { | |
272 | struct kvmppc_sid_map *map; | |
273 | struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); | |
274 | u16 sid_map_mask; | |
275 | static int backwards_map = 0; | |
276 | ||
666e7252 | 277 | if (vcpu->arch.shared->msr & MSR_PR) |
d32154f1 AG |
278 | gvsid |= VSID_PR; |
279 | ||
280 | /* We might get collisions that trap in preceding order, so let's | |
281 | map them differently */ | |
282 | ||
283 | sid_map_mask = kvmppc_sid_hash(vcpu, gvsid); | |
284 | if (backwards_map) | |
285 | sid_map_mask = SID_MAP_MASK - sid_map_mask; | |
286 | ||
287 | map = &to_book3s(vcpu)->sid_map[sid_map_mask]; | |
288 | ||
289 | /* Make sure we're taking the other map next time */ | |
290 | backwards_map = !backwards_map; | |
291 | ||
292 | /* Uh-oh ... out of mappings. Let's flush! */ | |
8b6db3bc AG |
293 | if (vcpu_book3s->vsid_next >= VSID_POOL_SIZE) { |
294 | vcpu_book3s->vsid_next = 0; | |
d32154f1 AG |
295 | memset(vcpu_book3s->sid_map, 0, |
296 | sizeof(struct kvmppc_sid_map) * SID_MAP_NUM); | |
297 | kvmppc_mmu_pte_flush(vcpu, 0, 0); | |
298 | kvmppc_mmu_flush_segments(vcpu); | |
299 | } | |
8b6db3bc AG |
300 | map->host_vsid = vcpu_book3s->vsid_pool[vcpu_book3s->vsid_next]; |
301 | vcpu_book3s->vsid_next++; | |
d32154f1 AG |
302 | |
303 | map->guest_vsid = gvsid; | |
304 | map->valid = true; | |
305 | ||
306 | return map; | |
307 | } | |
308 | ||
309 | int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr) | |
310 | { | |
311 | u32 esid = eaddr >> SID_SHIFT; | |
312 | u64 gvsid; | |
313 | u32 sr; | |
314 | struct kvmppc_sid_map *map; | |
468a12c2 AG |
315 | struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); |
316 | int r = 0; | |
d32154f1 AG |
317 | |
318 | if (vcpu->arch.mmu.esid_to_vsid(vcpu, esid, &gvsid)) { | |
319 | /* Invalidate an entry */ | |
320 | svcpu->sr[esid] = SR_INVALID; | |
468a12c2 AG |
321 | r = -ENOENT; |
322 | goto out; | |
d32154f1 AG |
323 | } |
324 | ||
325 | map = find_sid_vsid(vcpu, gvsid); | |
326 | if (!map) | |
327 | map = create_sid_map(vcpu, gvsid); | |
328 | ||
329 | map->guest_esid = esid; | |
330 | sr = map->host_vsid | SR_KP; | |
331 | svcpu->sr[esid] = sr; | |
332 | ||
333 | dprintk_sr("MMU: mtsr %d, 0x%x\n", esid, sr); | |
334 | ||
468a12c2 AG |
335 | out: |
336 | svcpu_put(svcpu); | |
337 | return r; | |
d32154f1 AG |
338 | } |
339 | ||
340 | void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu) | |
341 | { | |
342 | int i; | |
468a12c2 | 343 | struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); |
d32154f1 AG |
344 | |
345 | dprintk_sr("MMU: flushing all segments (%d)\n", ARRAY_SIZE(svcpu->sr)); | |
346 | for (i = 0; i < ARRAY_SIZE(svcpu->sr); i++) | |
347 | svcpu->sr[i] = SR_INVALID; | |
468a12c2 AG |
348 | |
349 | svcpu_put(svcpu); | |
d32154f1 AG |
350 | } |
351 | ||
352 | void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu) | |
353 | { | |
8b6db3bc AG |
354 | int i; |
355 | ||
fef093be | 356 | kvmppc_mmu_hpte_destroy(vcpu); |
d32154f1 | 357 | preempt_disable(); |
8b6db3bc AG |
358 | for (i = 0; i < SID_CONTEXTS; i++) |
359 | __destroy_context(to_book3s(vcpu)->context_id[i]); | |
d32154f1 AG |
360 | preempt_enable(); |
361 | } | |
362 | ||
363 | /* From mm/mmu_context_hash32.c */ | |
8b6db3bc | 364 | #define CTX_TO_VSID(c, id) ((((c) * (897 * 16)) + (id * 0x111)) & 0xffffff) |
d32154f1 AG |
365 | |
366 | int kvmppc_mmu_init(struct kvm_vcpu *vcpu) | |
367 | { | |
368 | struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); | |
369 | int err; | |
251585b5 | 370 | ulong sdr1; |
8b6db3bc AG |
371 | int i; |
372 | int j; | |
d32154f1 | 373 | |
8b6db3bc AG |
374 | for (i = 0; i < SID_CONTEXTS; i++) { |
375 | err = __init_new_context(); | |
376 | if (err < 0) | |
377 | goto init_fail; | |
378 | vcpu3s->context_id[i] = err; | |
d32154f1 | 379 | |
8b6db3bc AG |
380 | /* Remember context id for this combination */ |
381 | for (j = 0; j < 16; j++) | |
382 | vcpu3s->vsid_pool[(i * 16) + j] = CTX_TO_VSID(err, j); | |
383 | } | |
d32154f1 | 384 | |
8b6db3bc | 385 | vcpu3s->vsid_next = 0; |
d32154f1 | 386 | |
251585b5 AG |
387 | /* Remember where the HTAB is */ |
388 | asm ( "mfsdr1 %0" : "=r"(sdr1) ); | |
389 | htabmask = ((sdr1 & 0x1FF) << 16) | 0xFFC0; | |
390 | htab = (ulong)__va(sdr1 & 0xffff0000); | |
391 | ||
fef093be AG |
392 | kvmppc_mmu_hpte_init(vcpu); |
393 | ||
d32154f1 | 394 | return 0; |
8b6db3bc AG |
395 | |
396 | init_fail: | |
397 | for (j = 0; j < i; j++) { | |
398 | if (!vcpu3s->context_id[j]) | |
399 | continue; | |
400 | ||
401 | __destroy_context(to_book3s(vcpu)->context_id[j]); | |
402 | } | |
403 | ||
404 | return -1; | |
d32154f1 | 405 | } |