Commit | Line | Data |
---|---|---|
403015b3 JH |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * KVM/MIPS MMU handling in the KVM module. | |
7 | * | |
8 | * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. | |
9 | * Authors: Sanjay Lal <sanjayl@kymasys.com> | |
10 | */ | |
11 | ||
28cc5bd5 | 12 | #include <linux/highmem.h> |
403015b3 | 13 | #include <linux/kvm_host.h> |
dacc3ed1 | 14 | #include <linux/uaccess.h> |
403015b3 | 15 | #include <asm/mmu_context.h> |
a31b50d7 | 16 | #include <asm/pgalloc.h> |
403015b3 | 17 | |
fb995893 JH |
18 | /* |
19 | * KVM_MMU_CACHE_MIN_PAGES is the number of GPA page table translation levels | |
20 | * for which pages need to be cached. | |
21 | */ | |
22 | #if defined(__PAGETABLE_PMD_FOLDED) | |
23 | #define KVM_MMU_CACHE_MIN_PAGES 1 | |
24 | #else | |
25 | #define KVM_MMU_CACHE_MIN_PAGES 2 | |
26 | #endif | |
27 | ||
28 | static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, | |
29 | int min, int max) | |
30 | { | |
31 | void *page; | |
32 | ||
33 | BUG_ON(max > KVM_NR_MEM_OBJS); | |
34 | if (cache->nobjs >= min) | |
35 | return 0; | |
36 | while (cache->nobjs < max) { | |
37 | page = (void *)__get_free_page(GFP_KERNEL); | |
38 | if (!page) | |
39 | return -ENOMEM; | |
40 | cache->objects[cache->nobjs++] = page; | |
41 | } | |
42 | return 0; | |
43 | } | |
44 | ||
aba85929 JH |
45 | static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) |
46 | { | |
47 | while (mc->nobjs) | |
48 | free_page((unsigned long)mc->objects[--mc->nobjs]); | |
49 | } | |
50 | ||
51 | static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) | |
52 | { | |
53 | void *p; | |
54 | ||
55 | BUG_ON(!mc || !mc->nobjs); | |
56 | p = mc->objects[--mc->nobjs]; | |
57 | return p; | |
58 | } | |
59 | ||
60 | void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) | |
61 | { | |
62 | mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); | |
63 | } | |
64 | ||
06c158c9 JH |
65 | /** |
66 | * kvm_pgd_init() - Initialise KVM GPA page directory. | |
67 | * @page: Pointer to page directory (PGD) for KVM GPA. | |
68 | * | |
69 | * Initialise a KVM GPA page directory with pointers to the invalid table, i.e. | |
70 | * representing no mappings. This is similar to pgd_init(), however it | |
71 | * initialises all the page directory pointers, not just the ones corresponding | |
72 | * to the userland address space (since it is for the guest physical address | |
73 | * space rather than a virtual address space). | |
74 | */ | |
75 | static void kvm_pgd_init(void *page) | |
76 | { | |
77 | unsigned long *p, *end; | |
78 | unsigned long entry; | |
79 | ||
80 | #ifdef __PAGETABLE_PMD_FOLDED | |
81 | entry = (unsigned long)invalid_pte_table; | |
82 | #else | |
83 | entry = (unsigned long)invalid_pmd_table; | |
84 | #endif | |
85 | ||
86 | p = (unsigned long *)page; | |
87 | end = p + PTRS_PER_PGD; | |
88 | ||
89 | do { | |
90 | p[0] = entry; | |
91 | p[1] = entry; | |
92 | p[2] = entry; | |
93 | p[3] = entry; | |
94 | p[4] = entry; | |
95 | p += 8; | |
96 | p[-3] = entry; | |
97 | p[-2] = entry; | |
98 | p[-1] = entry; | |
99 | } while (p != end); | |
100 | } | |
101 | ||
102 | /** | |
103 | * kvm_pgd_alloc() - Allocate and initialise a KVM GPA page directory. | |
104 | * | |
105 | * Allocate a blank KVM GPA page directory (PGD) for representing guest physical | |
106 | * to host physical page mappings. | |
107 | * | |
108 | * Returns: Pointer to new KVM GPA page directory. | |
109 | * NULL on allocation failure. | |
110 | */ | |
111 | pgd_t *kvm_pgd_alloc(void) | |
112 | { | |
113 | pgd_t *ret; | |
114 | ||
115 | ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGD_ORDER); | |
116 | if (ret) | |
117 | kvm_pgd_init(ret); | |
118 | ||
119 | return ret; | |
120 | } | |
121 | ||
aba85929 JH |
122 | /** |
123 | * kvm_mips_walk_pgd() - Walk page table with optional allocation. | |
124 | * @pgd: Page directory pointer. | |
125 | * @addr: Address to index page table using. | |
126 | * @cache: MMU page cache to allocate new page tables from, or NULL. | |
127 | * | |
128 | * Walk the page tables pointed to by @pgd to find the PTE corresponding to the | |
129 | * address @addr. If page tables don't exist for @addr, they will be created | |
130 | * from the MMU cache if @cache is not NULL. | |
131 | * | |
132 | * Returns: Pointer to pte_t corresponding to @addr. | |
133 | * NULL if a page table doesn't exist for @addr and !@cache. | |
134 | * NULL if a page table allocation failed. | |
135 | */ | |
136 | static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, | |
137 | unsigned long addr) | |
138 | { | |
139 | pud_t *pud; | |
140 | pmd_t *pmd; | |
141 | ||
142 | pgd += pgd_index(addr); | |
143 | if (pgd_none(*pgd)) { | |
144 | /* Not used on MIPS yet */ | |
145 | BUG(); | |
146 | return NULL; | |
147 | } | |
148 | pud = pud_offset(pgd, addr); | |
149 | if (pud_none(*pud)) { | |
150 | pmd_t *new_pmd; | |
151 | ||
152 | if (!cache) | |
153 | return NULL; | |
154 | new_pmd = mmu_memory_cache_alloc(cache); | |
155 | pmd_init((unsigned long)new_pmd, | |
156 | (unsigned long)invalid_pte_table); | |
157 | pud_populate(NULL, pud, new_pmd); | |
158 | } | |
159 | pmd = pmd_offset(pud, addr); | |
160 | if (pmd_none(*pmd)) { | |
161 | pte_t *new_pte; | |
162 | ||
163 | if (!cache) | |
164 | return NULL; | |
165 | new_pte = mmu_memory_cache_alloc(cache); | |
166 | clear_page(new_pte); | |
167 | pmd_populate_kernel(NULL, pmd, new_pte); | |
168 | } | |
169 | return pte_offset(pmd, addr); | |
170 | } | |
171 | ||
06c158c9 JH |
172 | /* Caller must hold kvm->mm_lock */ |
173 | static pte_t *kvm_mips_pte_for_gpa(struct kvm *kvm, | |
174 | struct kvm_mmu_memory_cache *cache, | |
175 | unsigned long addr) | |
403015b3 | 176 | { |
06c158c9 JH |
177 | return kvm_mips_walk_pgd(kvm->arch.gpa_mm.pgd, cache, addr); |
178 | } | |
403015b3 | 179 | |
06c158c9 JH |
180 | /* |
181 | * kvm_mips_flush_gpa_{pte,pmd,pud,pgd,pt}. | |
182 | * Flush a range of guest physical address space from the VM's GPA page tables. | |
183 | */ | |
184 | ||
185 | static bool kvm_mips_flush_gpa_pte(pte_t *pte, unsigned long start_gpa, | |
186 | unsigned long end_gpa) | |
187 | { | |
188 | int i_min = __pte_offset(start_gpa); | |
189 | int i_max = __pte_offset(end_gpa); | |
190 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); | |
191 | int i; | |
192 | ||
193 | for (i = i_min; i <= i_max; ++i) { | |
194 | if (!pte_present(pte[i])) | |
195 | continue; | |
196 | ||
06c158c9 JH |
197 | set_pte(pte + i, __pte(0)); |
198 | } | |
199 | return safe_to_remove; | |
200 | } | |
201 | ||
202 | static bool kvm_mips_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, | |
203 | unsigned long end_gpa) | |
204 | { | |
205 | pte_t *pte; | |
206 | unsigned long end = ~0ul; | |
207 | int i_min = __pmd_offset(start_gpa); | |
208 | int i_max = __pmd_offset(end_gpa); | |
209 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1); | |
210 | int i; | |
211 | ||
212 | for (i = i_min; i <= i_max; ++i, start_gpa = 0) { | |
213 | if (!pmd_present(pmd[i])) | |
214 | continue; | |
215 | ||
216 | pte = pte_offset(pmd + i, 0); | |
217 | if (i == i_max) | |
218 | end = end_gpa; | |
219 | ||
220 | if (kvm_mips_flush_gpa_pte(pte, start_gpa, end)) { | |
221 | pmd_clear(pmd + i); | |
222 | pte_free_kernel(NULL, pte); | |
223 | } else { | |
224 | safe_to_remove = false; | |
225 | } | |
226 | } | |
227 | return safe_to_remove; | |
228 | } | |
229 | ||
230 | static bool kvm_mips_flush_gpa_pud(pud_t *pud, unsigned long start_gpa, | |
231 | unsigned long end_gpa) | |
232 | { | |
233 | pmd_t *pmd; | |
234 | unsigned long end = ~0ul; | |
235 | int i_min = __pud_offset(start_gpa); | |
236 | int i_max = __pud_offset(end_gpa); | |
237 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1); | |
238 | int i; | |
239 | ||
240 | for (i = i_min; i <= i_max; ++i, start_gpa = 0) { | |
241 | if (!pud_present(pud[i])) | |
242 | continue; | |
243 | ||
244 | pmd = pmd_offset(pud + i, 0); | |
245 | if (i == i_max) | |
246 | end = end_gpa; | |
247 | ||
248 | if (kvm_mips_flush_gpa_pmd(pmd, start_gpa, end)) { | |
249 | pud_clear(pud + i); | |
250 | pmd_free(NULL, pmd); | |
251 | } else { | |
252 | safe_to_remove = false; | |
253 | } | |
254 | } | |
255 | return safe_to_remove; | |
256 | } | |
257 | ||
258 | static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa, | |
259 | unsigned long end_gpa) | |
260 | { | |
261 | pud_t *pud; | |
262 | unsigned long end = ~0ul; | |
263 | int i_min = pgd_index(start_gpa); | |
264 | int i_max = pgd_index(end_gpa); | |
265 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1); | |
266 | int i; | |
267 | ||
268 | for (i = i_min; i <= i_max; ++i, start_gpa = 0) { | |
269 | if (!pgd_present(pgd[i])) | |
270 | continue; | |
271 | ||
272 | pud = pud_offset(pgd + i, 0); | |
273 | if (i == i_max) | |
274 | end = end_gpa; | |
275 | ||
276 | if (kvm_mips_flush_gpa_pud(pud, start_gpa, end)) { | |
277 | pgd_clear(pgd + i); | |
278 | pud_free(NULL, pud); | |
279 | } else { | |
280 | safe_to_remove = false; | |
281 | } | |
282 | } | |
283 | return safe_to_remove; | |
284 | } | |
285 | ||
286 | /** | |
287 | * kvm_mips_flush_gpa_pt() - Flush a range of guest physical addresses. | |
288 | * @kvm: KVM pointer. | |
289 | * @start_gfn: Guest frame number of first page in GPA range to flush. | |
290 | * @end_gfn: Guest frame number of last page in GPA range to flush. | |
291 | * | |
292 | * Flushes a range of GPA mappings from the GPA page tables. | |
293 | * | |
294 | * The caller must hold the @kvm->mmu_lock spinlock. | |
295 | * | |
296 | * Returns: Whether its safe to remove the top level page directory because | |
297 | * all lower levels have been removed. | |
298 | */ | |
299 | bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn) | |
300 | { | |
301 | return kvm_mips_flush_gpa_pgd(kvm->arch.gpa_mm.pgd, | |
302 | start_gfn << PAGE_SHIFT, | |
303 | end_gfn << PAGE_SHIFT); | |
304 | } | |
305 | ||
f0c0c330 JH |
306 | #define BUILD_PTE_RANGE_OP(name, op) \ |
307 | static int kvm_mips_##name##_pte(pte_t *pte, unsigned long start, \ | |
308 | unsigned long end) \ | |
309 | { \ | |
310 | int ret = 0; \ | |
311 | int i_min = __pte_offset(start); \ | |
312 | int i_max = __pte_offset(end); \ | |
313 | int i; \ | |
314 | pte_t old, new; \ | |
315 | \ | |
316 | for (i = i_min; i <= i_max; ++i) { \ | |
317 | if (!pte_present(pte[i])) \ | |
318 | continue; \ | |
319 | \ | |
320 | old = pte[i]; \ | |
321 | new = op(old); \ | |
322 | if (pte_val(new) == pte_val(old)) \ | |
323 | continue; \ | |
324 | set_pte(pte + i, new); \ | |
325 | ret = 1; \ | |
326 | } \ | |
327 | return ret; \ | |
328 | } \ | |
329 | \ | |
330 | /* returns true if anything was done */ \ | |
331 | static int kvm_mips_##name##_pmd(pmd_t *pmd, unsigned long start, \ | |
332 | unsigned long end) \ | |
333 | { \ | |
334 | int ret = 0; \ | |
335 | pte_t *pte; \ | |
336 | unsigned long cur_end = ~0ul; \ | |
337 | int i_min = __pmd_offset(start); \ | |
338 | int i_max = __pmd_offset(end); \ | |
339 | int i; \ | |
340 | \ | |
341 | for (i = i_min; i <= i_max; ++i, start = 0) { \ | |
342 | if (!pmd_present(pmd[i])) \ | |
343 | continue; \ | |
344 | \ | |
345 | pte = pte_offset(pmd + i, 0); \ | |
346 | if (i == i_max) \ | |
347 | cur_end = end; \ | |
348 | \ | |
349 | ret |= kvm_mips_##name##_pte(pte, start, cur_end); \ | |
350 | } \ | |
351 | return ret; \ | |
352 | } \ | |
353 | \ | |
354 | static int kvm_mips_##name##_pud(pud_t *pud, unsigned long start, \ | |
355 | unsigned long end) \ | |
356 | { \ | |
357 | int ret = 0; \ | |
358 | pmd_t *pmd; \ | |
359 | unsigned long cur_end = ~0ul; \ | |
360 | int i_min = __pud_offset(start); \ | |
361 | int i_max = __pud_offset(end); \ | |
362 | int i; \ | |
363 | \ | |
364 | for (i = i_min; i <= i_max; ++i, start = 0) { \ | |
365 | if (!pud_present(pud[i])) \ | |
366 | continue; \ | |
367 | \ | |
368 | pmd = pmd_offset(pud + i, 0); \ | |
369 | if (i == i_max) \ | |
370 | cur_end = end; \ | |
371 | \ | |
372 | ret |= kvm_mips_##name##_pmd(pmd, start, cur_end); \ | |
373 | } \ | |
374 | return ret; \ | |
375 | } \ | |
376 | \ | |
377 | static int kvm_mips_##name##_pgd(pgd_t *pgd, unsigned long start, \ | |
378 | unsigned long end) \ | |
379 | { \ | |
380 | int ret = 0; \ | |
381 | pud_t *pud; \ | |
382 | unsigned long cur_end = ~0ul; \ | |
383 | int i_min = pgd_index(start); \ | |
384 | int i_max = pgd_index(end); \ | |
385 | int i; \ | |
386 | \ | |
387 | for (i = i_min; i <= i_max; ++i, start = 0) { \ | |
388 | if (!pgd_present(pgd[i])) \ | |
389 | continue; \ | |
390 | \ | |
391 | pud = pud_offset(pgd + i, 0); \ | |
392 | if (i == i_max) \ | |
393 | cur_end = end; \ | |
394 | \ | |
395 | ret |= kvm_mips_##name##_pud(pud, start, cur_end); \ | |
396 | } \ | |
397 | return ret; \ | |
398 | } | |
399 | ||
400 | /* | |
401 | * kvm_mips_mkclean_gpa_pt. | |
402 | * Mark a range of guest physical address space clean (writes fault) in the VM's | |
403 | * GPA page table to allow dirty page tracking. | |
404 | */ | |
405 | ||
406 | BUILD_PTE_RANGE_OP(mkclean, pte_mkclean) | |
407 | ||
408 | /** | |
409 | * kvm_mips_mkclean_gpa_pt() - Make a range of guest physical addresses clean. | |
410 | * @kvm: KVM pointer. | |
411 | * @start_gfn: Guest frame number of first page in GPA range to flush. | |
412 | * @end_gfn: Guest frame number of last page in GPA range to flush. | |
413 | * | |
414 | * Make a range of GPA mappings clean so that guest writes will fault and | |
415 | * trigger dirty page logging. | |
416 | * | |
417 | * The caller must hold the @kvm->mmu_lock spinlock. | |
418 | * | |
419 | * Returns: Whether any GPA mappings were modified, which would require | |
420 | * derived mappings (GVA page tables & TLB enties) to be | |
421 | * invalidated. | |
422 | */ | |
423 | int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn) | |
424 | { | |
425 | return kvm_mips_mkclean_pgd(kvm->arch.gpa_mm.pgd, | |
426 | start_gfn << PAGE_SHIFT, | |
427 | end_gfn << PAGE_SHIFT); | |
428 | } | |
429 | ||
e88643ba JH |
430 | /** |
431 | * kvm_arch_mmu_enable_log_dirty_pt_masked() - write protect dirty pages | |
432 | * @kvm: The KVM pointer | |
433 | * @slot: The memory slot associated with mask | |
434 | * @gfn_offset: The gfn offset in memory slot | |
435 | * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory | |
436 | * slot to be write protected | |
437 | * | |
438 | * Walks bits set in mask write protects the associated pte's. Caller must | |
439 | * acquire @kvm->mmu_lock. | |
440 | */ | |
441 | void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, | |
442 | struct kvm_memory_slot *slot, | |
443 | gfn_t gfn_offset, unsigned long mask) | |
444 | { | |
445 | gfn_t base_gfn = slot->base_gfn + gfn_offset; | |
446 | gfn_t start = base_gfn + __ffs(mask); | |
447 | gfn_t end = base_gfn + __fls(mask); | |
448 | ||
449 | kvm_mips_mkclean_gpa_pt(kvm, start, end); | |
450 | } | |
451 | ||
411740f5 JH |
452 | /* |
453 | * kvm_mips_mkold_gpa_pt. | |
454 | * Mark a range of guest physical address space old (all accesses fault) in the | |
455 | * VM's GPA page table to allow detection of commonly used pages. | |
456 | */ | |
457 | ||
458 | BUILD_PTE_RANGE_OP(mkold, pte_mkold) | |
459 | ||
460 | static int kvm_mips_mkold_gpa_pt(struct kvm *kvm, gfn_t start_gfn, | |
461 | gfn_t end_gfn) | |
462 | { | |
463 | return kvm_mips_mkold_pgd(kvm->arch.gpa_mm.pgd, | |
464 | start_gfn << PAGE_SHIFT, | |
465 | end_gfn << PAGE_SHIFT); | |
466 | } | |
467 | ||
468 | static int handle_hva_to_gpa(struct kvm *kvm, | |
469 | unsigned long start, | |
470 | unsigned long end, | |
471 | int (*handler)(struct kvm *kvm, gfn_t gfn, | |
472 | gpa_t gfn_end, | |
473 | struct kvm_memory_slot *memslot, | |
474 | void *data), | |
475 | void *data) | |
476 | { | |
477 | struct kvm_memslots *slots; | |
478 | struct kvm_memory_slot *memslot; | |
479 | int ret = 0; | |
480 | ||
481 | slots = kvm_memslots(kvm); | |
482 | ||
483 | /* we only care about the pages that the guest sees */ | |
484 | kvm_for_each_memslot(memslot, slots) { | |
485 | unsigned long hva_start, hva_end; | |
486 | gfn_t gfn, gfn_end; | |
487 | ||
488 | hva_start = max(start, memslot->userspace_addr); | |
489 | hva_end = min(end, memslot->userspace_addr + | |
490 | (memslot->npages << PAGE_SHIFT)); | |
491 | if (hva_start >= hva_end) | |
492 | continue; | |
493 | ||
494 | /* | |
495 | * {gfn(page) | page intersects with [hva_start, hva_end)} = | |
496 | * {gfn_start, gfn_start+1, ..., gfn_end-1}. | |
497 | */ | |
498 | gfn = hva_to_gfn_memslot(hva_start, memslot); | |
499 | gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); | |
500 | ||
501 | ret |= handler(kvm, gfn, gfn_end, memslot, data); | |
502 | } | |
503 | ||
504 | return ret; | |
505 | } | |
506 | ||
507 | ||
508 | static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, | |
509 | struct kvm_memory_slot *memslot, void *data) | |
510 | { | |
511 | kvm_mips_flush_gpa_pt(kvm, gfn, gfn_end); | |
512 | return 1; | |
513 | } | |
514 | ||
515 | int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) | |
516 | { | |
517 | unsigned long end = hva + PAGE_SIZE; | |
518 | ||
519 | handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL); | |
520 | ||
521 | kvm_mips_callbacks->flush_shadow_all(kvm); | |
522 | return 0; | |
523 | } | |
524 | ||
525 | int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end) | |
526 | { | |
527 | handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); | |
528 | ||
529 | kvm_mips_callbacks->flush_shadow_all(kvm); | |
530 | return 0; | |
531 | } | |
532 | ||
533 | static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, | |
534 | struct kvm_memory_slot *memslot, void *data) | |
535 | { | |
536 | gpa_t gpa = gfn << PAGE_SHIFT; | |
537 | pte_t hva_pte = *(pte_t *)data; | |
538 | pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa); | |
539 | pte_t old_pte; | |
540 | ||
541 | if (!gpa_pte) | |
542 | return 0; | |
543 | ||
544 | /* Mapping may need adjusting depending on memslot flags */ | |
545 | old_pte = *gpa_pte; | |
546 | if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte)) | |
547 | hva_pte = pte_mkclean(hva_pte); | |
548 | else if (memslot->flags & KVM_MEM_READONLY) | |
549 | hva_pte = pte_wrprotect(hva_pte); | |
550 | ||
551 | set_pte(gpa_pte, hva_pte); | |
552 | ||
553 | /* Replacing an absent or old page doesn't need flushes */ | |
554 | if (!pte_present(old_pte) || !pte_young(old_pte)) | |
555 | return 0; | |
556 | ||
557 | /* Pages swapped, aged, moved, or cleaned require flushes */ | |
558 | return !pte_present(hva_pte) || | |
559 | !pte_young(hva_pte) || | |
560 | pte_pfn(old_pte) != pte_pfn(hva_pte) || | |
561 | (pte_dirty(old_pte) && !pte_dirty(hva_pte)); | |
562 | } | |
563 | ||
564 | void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) | |
565 | { | |
566 | unsigned long end = hva + PAGE_SIZE; | |
567 | int ret; | |
568 | ||
569 | ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte); | |
570 | if (ret) | |
571 | kvm_mips_callbacks->flush_shadow_all(kvm); | |
572 | } | |
573 | ||
574 | static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, | |
575 | struct kvm_memory_slot *memslot, void *data) | |
576 | { | |
577 | return kvm_mips_mkold_gpa_pt(kvm, gfn, gfn_end); | |
578 | } | |
579 | ||
580 | static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end, | |
581 | struct kvm_memory_slot *memslot, void *data) | |
582 | { | |
583 | gpa_t gpa = gfn << PAGE_SHIFT; | |
584 | pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa); | |
585 | ||
586 | if (!gpa_pte) | |
587 | return 0; | |
588 | return pte_young(*gpa_pte); | |
589 | } | |
590 | ||
591 | int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) | |
592 | { | |
593 | return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); | |
594 | } | |
595 | ||
596 | int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) | |
597 | { | |
598 | return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); | |
599 | } | |
600 | ||
b5f1dd1b JH |
601 | /** |
602 | * _kvm_mips_map_page_fast() - Fast path GPA fault handler. | |
603 | * @vcpu: VCPU pointer. | |
604 | * @gpa: Guest physical address of fault. | |
605 | * @write_fault: Whether the fault was due to a write. | |
606 | * @out_entry: New PTE for @gpa (written on success unless NULL). | |
607 | * @out_buddy: New PTE for @gpa's buddy (written on success unless | |
608 | * NULL). | |
609 | * | |
610 | * Perform fast path GPA fault handling, doing all that can be done without | |
411740f5 JH |
611 | * calling into KVM. This handles marking old pages young (for idle page |
612 | * tracking), and dirtying of clean pages (for dirty page logging). | |
b5f1dd1b JH |
613 | * |
614 | * Returns: 0 on success, in which case we can update derived mappings and | |
615 | * resume guest execution. | |
616 | * -EFAULT on failure due to absent GPA mapping or write to | |
617 | * read-only page, in which case KVM must be consulted. | |
618 | */ | |
619 | static int _kvm_mips_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa, | |
620 | bool write_fault, | |
621 | pte_t *out_entry, pte_t *out_buddy) | |
622 | { | |
623 | struct kvm *kvm = vcpu->kvm; | |
624 | gfn_t gfn = gpa >> PAGE_SHIFT; | |
625 | pte_t *ptep; | |
411740f5 JH |
626 | kvm_pfn_t pfn = 0; /* silence bogus GCC warning */ |
627 | bool pfn_valid = false; | |
b5f1dd1b JH |
628 | int ret = 0; |
629 | ||
630 | spin_lock(&kvm->mmu_lock); | |
631 | ||
632 | /* Fast path - just check GPA page table for an existing entry */ | |
633 | ptep = kvm_mips_pte_for_gpa(kvm, NULL, gpa); | |
634 | if (!ptep || !pte_present(*ptep)) { | |
635 | ret = -EFAULT; | |
636 | goto out; | |
637 | } | |
638 | ||
411740f5 JH |
639 | /* Track access to pages marked old */ |
640 | if (!pte_young(*ptep)) { | |
641 | set_pte(ptep, pte_mkyoung(*ptep)); | |
642 | pfn = pte_pfn(*ptep); | |
643 | pfn_valid = true; | |
644 | /* call kvm_set_pfn_accessed() after unlock */ | |
645 | } | |
b5f1dd1b | 646 | if (write_fault && !pte_dirty(*ptep)) { |
411740f5 JH |
647 | if (!pte_write(*ptep)) { |
648 | ret = -EFAULT; | |
649 | goto out; | |
650 | } | |
651 | ||
652 | /* Track dirtying of writeable pages */ | |
b5f1dd1b | 653 | set_pte(ptep, pte_mkdirty(*ptep)); |
411740f5 | 654 | pfn = pte_pfn(*ptep); |
b5f1dd1b | 655 | mark_page_dirty(kvm, gfn); |
411740f5 | 656 | kvm_set_pfn_dirty(pfn); |
b5f1dd1b JH |
657 | } |
658 | ||
659 | if (out_entry) | |
660 | *out_entry = *ptep; | |
661 | if (out_buddy) | |
662 | *out_buddy = *ptep_buddy(ptep); | |
663 | ||
664 | out: | |
665 | spin_unlock(&kvm->mmu_lock); | |
411740f5 JH |
666 | if (pfn_valid) |
667 | kvm_set_pfn_accessed(pfn); | |
b5f1dd1b JH |
668 | return ret; |
669 | } | |
670 | ||
06c158c9 JH |
671 | /** |
672 | * kvm_mips_map_page() - Map a guest physical page. | |
673 | * @vcpu: VCPU pointer. | |
674 | * @gpa: Guest physical address of fault. | |
577ed7f7 | 675 | * @write_fault: Whether the fault was due to a write. |
06c158c9 JH |
676 | * @out_entry: New PTE for @gpa (written on success unless NULL). |
677 | * @out_buddy: New PTE for @gpa's buddy (written on success unless | |
678 | * NULL). | |
679 | * | |
680 | * Handle GPA faults by creating a new GPA mapping (or updating an existing | |
681 | * one). | |
682 | * | |
411740f5 JH |
683 | * This takes care of marking pages young or dirty (idle/dirty page tracking), |
684 | * asking KVM for the corresponding PFN, and creating a mapping in the GPA page | |
685 | * tables. Derived mappings (GVA page tables and TLBs) must be handled by the | |
686 | * caller. | |
06c158c9 JH |
687 | * |
688 | * Returns: 0 on success, in which case the caller may use the @out_entry | |
689 | * and @out_buddy PTEs to update derived mappings and resume guest | |
690 | * execution. | |
691 | * -EFAULT if there is no memory region at @gpa or a write was | |
692 | * attempted to a read-only memory region. This is usually handled | |
693 | * as an MMIO access. | |
694 | */ | |
695 | static int kvm_mips_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, | |
577ed7f7 | 696 | bool write_fault, |
06c158c9 JH |
697 | pte_t *out_entry, pte_t *out_buddy) |
698 | { | |
699 | struct kvm *kvm = vcpu->kvm; | |
700 | struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; | |
701 | gfn_t gfn = gpa >> PAGE_SHIFT; | |
702 | int srcu_idx, err; | |
703 | kvm_pfn_t pfn; | |
704 | pte_t *ptep, entry, old_pte; | |
411740f5 | 705 | bool writeable; |
06c158c9 | 706 | unsigned long prot_bits; |
411740f5 | 707 | unsigned long mmu_seq; |
403015b3 | 708 | |
411740f5 | 709 | /* Try the fast path to handle old / clean pages */ |
403015b3 | 710 | srcu_idx = srcu_read_lock(&kvm->srcu); |
b5f1dd1b JH |
711 | err = _kvm_mips_map_page_fast(vcpu, gpa, write_fault, out_entry, |
712 | out_buddy); | |
713 | if (!err) | |
714 | goto out; | |
06c158c9 JH |
715 | |
716 | /* We need a minimum of cached pages ready for page table creation */ | |
717 | err = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, | |
718 | KVM_NR_MEM_OBJS); | |
719 | if (err) | |
720 | goto out; | |
721 | ||
411740f5 JH |
722 | retry: |
723 | /* | |
724 | * Used to check for invalidations in progress, of the pfn that is | |
725 | * returned by pfn_to_pfn_prot below. | |
726 | */ | |
727 | mmu_seq = kvm->mmu_notifier_seq; | |
728 | /* | |
729 | * Ensure the read of mmu_notifier_seq isn't reordered with PTE reads in | |
730 | * gfn_to_pfn_prot() (which calls get_user_pages()), so that we don't | |
731 | * risk the page we get a reference to getting unmapped before we have a | |
732 | * chance to grab the mmu_lock without mmu_notifier_retry() noticing. | |
733 | * | |
734 | * This smp_rmb() pairs with the effective smp_wmb() of the combination | |
735 | * of the pte_unmap_unlock() after the PTE is zapped, and the | |
736 | * spin_lock() in kvm_mmu_notifier_invalidate_<page|range_end>() before | |
737 | * mmu_notifier_seq is incremented. | |
738 | */ | |
739 | smp_rmb(); | |
403015b3 | 740 | |
411740f5 JH |
741 | /* Slow path - ask KVM core whether we can access this GPA */ |
742 | pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writeable); | |
ba913e4f | 743 | if (is_error_noslot_pfn(pfn)) { |
403015b3 JH |
744 | err = -EFAULT; |
745 | goto out; | |
746 | } | |
747 | ||
06c158c9 | 748 | spin_lock(&kvm->mmu_lock); |
411740f5 JH |
749 | /* Check if an invalidation has taken place since we got pfn */ |
750 | if (mmu_notifier_retry(kvm, mmu_seq)) { | |
751 | /* | |
752 | * This can happen when mappings are changed asynchronously, but | |
753 | * also synchronously if a COW is triggered by | |
754 | * gfn_to_pfn_prot(). | |
755 | */ | |
756 | spin_unlock(&kvm->mmu_lock); | |
757 | kvm_release_pfn_clean(pfn); | |
758 | goto retry; | |
759 | } | |
06c158c9 | 760 | |
b5f1dd1b | 761 | /* Ensure page tables are allocated */ |
06c158c9 JH |
762 | ptep = kvm_mips_pte_for_gpa(kvm, memcache, gpa); |
763 | ||
b5f1dd1b | 764 | /* Set up the PTE */ |
411740f5 JH |
765 | prot_bits = _PAGE_PRESENT | __READABLE | _page_cachable_default; |
766 | if (writeable) { | |
767 | prot_bits |= _PAGE_WRITE; | |
768 | if (write_fault) { | |
769 | prot_bits |= __WRITEABLE; | |
770 | mark_page_dirty(kvm, gfn); | |
771 | kvm_set_pfn_dirty(pfn); | |
772 | } | |
b5f1dd1b | 773 | } |
06c158c9 JH |
774 | entry = pfn_pte(pfn, __pgprot(prot_bits)); |
775 | ||
b5f1dd1b | 776 | /* Write the PTE */ |
06c158c9 JH |
777 | old_pte = *ptep; |
778 | set_pte(ptep, entry); | |
06c158c9 JH |
779 | |
780 | err = 0; | |
781 | if (out_entry) | |
782 | *out_entry = *ptep; | |
783 | if (out_buddy) | |
784 | *out_buddy = *ptep_buddy(ptep); | |
785 | ||
786 | spin_unlock(&kvm->mmu_lock); | |
411740f5 JH |
787 | kvm_release_pfn_clean(pfn); |
788 | kvm_set_pfn_accessed(pfn); | |
403015b3 JH |
789 | out: |
790 | srcu_read_unlock(&kvm->srcu, srcu_idx); | |
791 | return err; | |
792 | } | |
793 | ||
fb995893 JH |
794 | static pte_t *kvm_trap_emul_pte_for_gva(struct kvm_vcpu *vcpu, |
795 | unsigned long addr) | |
796 | { | |
797 | struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; | |
798 | pgd_t *pgdp; | |
799 | int ret; | |
800 | ||
801 | /* We need a minimum of cached pages ready for page table creation */ | |
802 | ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, | |
803 | KVM_NR_MEM_OBJS); | |
804 | if (ret) | |
805 | return NULL; | |
806 | ||
807 | if (KVM_GUEST_KERNEL_MODE(vcpu)) | |
808 | pgdp = vcpu->arch.guest_kernel_mm.pgd; | |
809 | else | |
810 | pgdp = vcpu->arch.guest_user_mm.pgd; | |
811 | ||
812 | return kvm_mips_walk_pgd(pgdp, memcache, addr); | |
813 | } | |
814 | ||
aba85929 JH |
815 | void kvm_trap_emul_invalidate_gva(struct kvm_vcpu *vcpu, unsigned long addr, |
816 | bool user) | |
817 | { | |
818 | pgd_t *pgdp; | |
819 | pte_t *ptep; | |
820 | ||
821 | addr &= PAGE_MASK << 1; | |
822 | ||
823 | pgdp = vcpu->arch.guest_kernel_mm.pgd; | |
824 | ptep = kvm_mips_walk_pgd(pgdp, NULL, addr); | |
825 | if (ptep) { | |
826 | ptep[0] = pfn_pte(0, __pgprot(0)); | |
827 | ptep[1] = pfn_pte(0, __pgprot(0)); | |
828 | } | |
829 | ||
830 | if (user) { | |
831 | pgdp = vcpu->arch.guest_user_mm.pgd; | |
832 | ptep = kvm_mips_walk_pgd(pgdp, NULL, addr); | |
833 | if (ptep) { | |
834 | ptep[0] = pfn_pte(0, __pgprot(0)); | |
835 | ptep[1] = pfn_pte(0, __pgprot(0)); | |
836 | } | |
837 | } | |
838 | } | |
839 | ||
a31b50d7 JH |
840 | /* |
841 | * kvm_mips_flush_gva_{pte,pmd,pud,pgd,pt}. | |
842 | * Flush a range of guest physical address space from the VM's GPA page tables. | |
843 | */ | |
844 | ||
845 | static bool kvm_mips_flush_gva_pte(pte_t *pte, unsigned long start_gva, | |
846 | unsigned long end_gva) | |
847 | { | |
848 | int i_min = __pte_offset(start_gva); | |
849 | int i_max = __pte_offset(end_gva); | |
850 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); | |
851 | int i; | |
852 | ||
853 | /* | |
854 | * There's no freeing to do, so there's no point clearing individual | |
855 | * entries unless only part of the last level page table needs flushing. | |
856 | */ | |
857 | if (safe_to_remove) | |
858 | return true; | |
859 | ||
860 | for (i = i_min; i <= i_max; ++i) { | |
861 | if (!pte_present(pte[i])) | |
862 | continue; | |
863 | ||
864 | set_pte(pte + i, __pte(0)); | |
865 | } | |
866 | return false; | |
867 | } | |
868 | ||
869 | static bool kvm_mips_flush_gva_pmd(pmd_t *pmd, unsigned long start_gva, | |
870 | unsigned long end_gva) | |
871 | { | |
872 | pte_t *pte; | |
873 | unsigned long end = ~0ul; | |
874 | int i_min = __pmd_offset(start_gva); | |
875 | int i_max = __pmd_offset(end_gva); | |
876 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1); | |
877 | int i; | |
878 | ||
879 | for (i = i_min; i <= i_max; ++i, start_gva = 0) { | |
880 | if (!pmd_present(pmd[i])) | |
881 | continue; | |
882 | ||
883 | pte = pte_offset(pmd + i, 0); | |
884 | if (i == i_max) | |
885 | end = end_gva; | |
886 | ||
887 | if (kvm_mips_flush_gva_pte(pte, start_gva, end)) { | |
888 | pmd_clear(pmd + i); | |
889 | pte_free_kernel(NULL, pte); | |
890 | } else { | |
891 | safe_to_remove = false; | |
892 | } | |
893 | } | |
894 | return safe_to_remove; | |
895 | } | |
896 | ||
897 | static bool kvm_mips_flush_gva_pud(pud_t *pud, unsigned long start_gva, | |
898 | unsigned long end_gva) | |
899 | { | |
900 | pmd_t *pmd; | |
901 | unsigned long end = ~0ul; | |
902 | int i_min = __pud_offset(start_gva); | |
903 | int i_max = __pud_offset(end_gva); | |
904 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1); | |
905 | int i; | |
906 | ||
907 | for (i = i_min; i <= i_max; ++i, start_gva = 0) { | |
908 | if (!pud_present(pud[i])) | |
909 | continue; | |
910 | ||
911 | pmd = pmd_offset(pud + i, 0); | |
912 | if (i == i_max) | |
913 | end = end_gva; | |
914 | ||
915 | if (kvm_mips_flush_gva_pmd(pmd, start_gva, end)) { | |
916 | pud_clear(pud + i); | |
917 | pmd_free(NULL, pmd); | |
918 | } else { | |
919 | safe_to_remove = false; | |
920 | } | |
921 | } | |
922 | return safe_to_remove; | |
923 | } | |
924 | ||
925 | static bool kvm_mips_flush_gva_pgd(pgd_t *pgd, unsigned long start_gva, | |
926 | unsigned long end_gva) | |
927 | { | |
928 | pud_t *pud; | |
929 | unsigned long end = ~0ul; | |
930 | int i_min = pgd_index(start_gva); | |
931 | int i_max = pgd_index(end_gva); | |
932 | bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1); | |
933 | int i; | |
934 | ||
935 | for (i = i_min; i <= i_max; ++i, start_gva = 0) { | |
936 | if (!pgd_present(pgd[i])) | |
937 | continue; | |
938 | ||
939 | pud = pud_offset(pgd + i, 0); | |
940 | if (i == i_max) | |
941 | end = end_gva; | |
942 | ||
943 | if (kvm_mips_flush_gva_pud(pud, start_gva, end)) { | |
944 | pgd_clear(pgd + i); | |
945 | pud_free(NULL, pud); | |
946 | } else { | |
947 | safe_to_remove = false; | |
948 | } | |
949 | } | |
950 | return safe_to_remove; | |
951 | } | |
952 | ||
953 | void kvm_mips_flush_gva_pt(pgd_t *pgd, enum kvm_mips_flush flags) | |
954 | { | |
955 | if (flags & KMF_GPA) { | |
956 | /* all of guest virtual address space could be affected */ | |
957 | if (flags & KMF_KERN) | |
958 | /* useg, kseg0, seg2/3 */ | |
959 | kvm_mips_flush_gva_pgd(pgd, 0, 0x7fffffff); | |
960 | else | |
961 | /* useg */ | |
962 | kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff); | |
963 | } else { | |
964 | /* useg */ | |
965 | kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff); | |
966 | ||
967 | /* kseg2/3 */ | |
968 | if (flags & KMF_KERN) | |
969 | kvm_mips_flush_gva_pgd(pgd, 0x60000000, 0x7fffffff); | |
970 | } | |
971 | } | |
972 | ||
b584f460 JH |
973 | static pte_t kvm_mips_gpa_pte_to_gva_unmapped(pte_t pte) |
974 | { | |
975 | /* | |
976 | * Don't leak writeable but clean entries from GPA page tables. We don't | |
977 | * want the normal Linux tlbmod handler to handle dirtying when KVM | |
978 | * accesses guest memory. | |
979 | */ | |
980 | if (!pte_dirty(pte)) | |
981 | pte = pte_wrprotect(pte); | |
982 | ||
983 | return pte; | |
984 | } | |
985 | ||
f9b11e51 JH |
986 | static pte_t kvm_mips_gpa_pte_to_gva_mapped(pte_t pte, long entrylo) |
987 | { | |
988 | /* Guest EntryLo overrides host EntryLo */ | |
989 | if (!(entrylo & ENTRYLO_D)) | |
990 | pte = pte_mkclean(pte); | |
991 | ||
992 | return kvm_mips_gpa_pte_to_gva_unmapped(pte); | |
993 | } | |
994 | ||
403015b3 JH |
995 | /* XXXKYMA: Must be called with interrupts disabled */ |
996 | int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr, | |
577ed7f7 JH |
997 | struct kvm_vcpu *vcpu, |
998 | bool write_fault) | |
403015b3 | 999 | { |
06c158c9 | 1000 | unsigned long gpa; |
06c158c9 | 1001 | pte_t pte_gpa[2], *ptep_gva; |
b584f460 | 1002 | int idx; |
403015b3 JH |
1003 | |
1004 | if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) { | |
1005 | kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr); | |
1006 | kvm_mips_dump_host_tlbs(); | |
1007 | return -1; | |
1008 | } | |
1009 | ||
b584f460 JH |
1010 | /* Get the GPA page table entry */ |
1011 | gpa = KVM_GUEST_CPHYSADDR(badvaddr); | |
1012 | idx = (badvaddr >> PAGE_SHIFT) & 1; | |
1013 | if (kvm_mips_map_page(vcpu, gpa, write_fault, &pte_gpa[idx], | |
1014 | &pte_gpa[!idx]) < 0) | |
403015b3 JH |
1015 | return -1; |
1016 | ||
b584f460 JH |
1017 | /* Get the GVA page table entry */ |
1018 | ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, badvaddr & ~PAGE_SIZE); | |
fb995893 | 1019 | if (!ptep_gva) { |
b584f460 | 1020 | kvm_err("No ptep for gva %lx\n", badvaddr); |
fb995893 JH |
1021 | return -1; |
1022 | } | |
403015b3 | 1023 | |
b584f460 JH |
1024 | /* Copy a pair of entries from GPA page table to GVA page table */ |
1025 | ptep_gva[0] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[0]); | |
1026 | ptep_gva[1] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[1]); | |
403015b3 | 1027 | |
fb995893 | 1028 | /* Invalidate this entry in the TLB, guest kernel ASID only */ |
b584f460 | 1029 | kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true); |
fb995893 | 1030 | return 0; |
403015b3 JH |
1031 | } |
1032 | ||
1033 | int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu, | |
7e3d2a75 | 1034 | struct kvm_mips_tlb *tlb, |
577ed7f7 JH |
1035 | unsigned long gva, |
1036 | bool write_fault) | |
403015b3 | 1037 | { |
f9b11e51 JH |
1038 | struct kvm *kvm = vcpu->kvm; |
1039 | long tlb_lo[2]; | |
1040 | pte_t pte_gpa[2], *ptep_buddy, *ptep_gva; | |
1041 | unsigned int idx = TLB_LO_IDX(*tlb, gva); | |
7e3d2a75 | 1042 | bool kernel = KVM_GUEST_KERNEL_MODE(vcpu); |
c604cffa | 1043 | |
f9b11e51 JH |
1044 | tlb_lo[0] = tlb->tlb_lo[0]; |
1045 | tlb_lo[1] = tlb->tlb_lo[1]; | |
1046 | ||
c604cffa JH |
1047 | /* |
1048 | * The commpage address must not be mapped to anything else if the guest | |
1049 | * TLB contains entries nearby, or commpage accesses will break. | |
1050 | */ | |
f9b11e51 JH |
1051 | if (!((gva ^ KVM_GUEST_COMMPAGE_ADDR) & VPN2_MASK & (PAGE_MASK << 1))) |
1052 | tlb_lo[TLB_LO_IDX(*tlb, KVM_GUEST_COMMPAGE_ADDR)] = 0; | |
7e3d2a75 | 1053 | |
f9b11e51 JH |
1054 | /* Get the GPA page table entry */ |
1055 | if (kvm_mips_map_page(vcpu, mips3_tlbpfn_to_paddr(tlb_lo[idx]), | |
1056 | write_fault, &pte_gpa[idx], NULL) < 0) | |
c604cffa JH |
1057 | return -1; |
1058 | ||
f9b11e51 JH |
1059 | /* And its GVA buddy's GPA page table entry if it also exists */ |
1060 | pte_gpa[!idx] = pfn_pte(0, __pgprot(0)); | |
1061 | if (tlb_lo[!idx] & ENTRYLO_V) { | |
1062 | spin_lock(&kvm->mmu_lock); | |
1063 | ptep_buddy = kvm_mips_pte_for_gpa(kvm, NULL, | |
1064 | mips3_tlbpfn_to_paddr(tlb_lo[!idx])); | |
1065 | if (ptep_buddy) | |
1066 | pte_gpa[!idx] = *ptep_buddy; | |
1067 | spin_unlock(&kvm->mmu_lock); | |
1068 | } | |
1069 | ||
1070 | /* Get the GVA page table entry pair */ | |
1071 | ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, gva & ~PAGE_SIZE); | |
7e3d2a75 JH |
1072 | if (!ptep_gva) { |
1073 | kvm_err("No ptep for gva %lx\n", gva); | |
c604cffa | 1074 | return -1; |
7e3d2a75 | 1075 | } |
c604cffa | 1076 | |
f9b11e51 JH |
1077 | /* Copy a pair of entries from GPA page table to GVA page table */ |
1078 | ptep_gva[0] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[0], tlb_lo[0]); | |
1079 | ptep_gva[1] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[1], tlb_lo[1]); | |
403015b3 | 1080 | |
7e3d2a75 JH |
1081 | /* Invalidate this entry in the TLB, current guest mode ASID only */ |
1082 | kvm_mips_host_tlb_inv(vcpu, gva, !kernel, kernel); | |
403015b3 JH |
1083 | |
1084 | kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc, | |
9fbfb06a | 1085 | tlb->tlb_lo[0], tlb->tlb_lo[1]); |
403015b3 | 1086 | |
7e3d2a75 | 1087 | return 0; |
403015b3 JH |
1088 | } |
1089 | ||
4c86460c JH |
1090 | int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr, |
1091 | struct kvm_vcpu *vcpu) | |
1092 | { | |
1093 | kvm_pfn_t pfn; | |
1094 | pte_t *ptep; | |
1095 | ||
1096 | ptep = kvm_trap_emul_pte_for_gva(vcpu, badvaddr); | |
1097 | if (!ptep) { | |
1098 | kvm_err("No ptep for commpage %lx\n", badvaddr); | |
1099 | return -1; | |
1100 | } | |
1101 | ||
1102 | pfn = PFN_DOWN(virt_to_phys(vcpu->arch.kseg0_commpage)); | |
1103 | /* Also set valid and dirty, so refill handler doesn't have to */ | |
1104 | *ptep = pte_mkyoung(pte_mkdirty(pfn_pte(pfn, PAGE_SHARED))); | |
1105 | ||
1106 | /* Invalidate this entry in the TLB, guest kernel ASID only */ | |
1107 | kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true); | |
1108 | return 0; | |
1109 | } | |
1110 | ||
403015b3 JH |
1111 | /** |
1112 | * kvm_mips_migrate_count() - Migrate timer. | |
1113 | * @vcpu: Virtual CPU. | |
1114 | * | |
1115 | * Migrate CP0_Count hrtimer to the current CPU by cancelling and restarting it | |
1116 | * if it was running prior to being cancelled. | |
1117 | * | |
1118 | * Must be called when the VCPU is migrated to a different CPU to ensure that | |
1119 | * timer expiry during guest execution interrupts the guest and causes the | |
1120 | * interrupt to be delivered in a timely manner. | |
1121 | */ | |
1122 | static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu) | |
1123 | { | |
1124 | if (hrtimer_cancel(&vcpu->arch.comparecount_timer)) | |
1125 | hrtimer_restart(&vcpu->arch.comparecount_timer); | |
1126 | } | |
1127 | ||
1128 | /* Restore ASID once we are scheduled back after preemption */ | |
1129 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) | |
1130 | { | |
403015b3 | 1131 | unsigned long flags; |
403015b3 JH |
1132 | |
1133 | kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu); | |
1134 | ||
403015b3 JH |
1135 | local_irq_save(flags); |
1136 | ||
4841e0dd | 1137 | vcpu->cpu = cpu; |
403015b3 JH |
1138 | if (vcpu->arch.last_sched_cpu != cpu) { |
1139 | kvm_debug("[%d->%d]KVM VCPU[%d] switch\n", | |
1140 | vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); | |
1141 | /* | |
1142 | * Migrate the timer interrupt to the current CPU so that it | |
1143 | * always interrupts the guest and synchronously triggers a | |
1144 | * guest timer interrupt. | |
1145 | */ | |
1146 | kvm_mips_migrate_count(vcpu); | |
1147 | } | |
1148 | ||
403015b3 | 1149 | /* restore guest state to registers */ |
a60b8438 | 1150 | kvm_mips_callbacks->vcpu_load(vcpu, cpu); |
403015b3 JH |
1151 | |
1152 | local_irq_restore(flags); | |
403015b3 JH |
1153 | } |
1154 | ||
1155 | /* ASID can change if another task is scheduled during preemption */ | |
1156 | void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) | |
1157 | { | |
1158 | unsigned long flags; | |
1159 | int cpu; | |
1160 | ||
1161 | local_irq_save(flags); | |
1162 | ||
1163 | cpu = smp_processor_id(); | |
403015b3 | 1164 | vcpu->arch.last_sched_cpu = cpu; |
4841e0dd | 1165 | vcpu->cpu = -1; |
403015b3 JH |
1166 | |
1167 | /* save guest state in registers */ | |
a60b8438 | 1168 | kvm_mips_callbacks->vcpu_put(vcpu, cpu); |
403015b3 | 1169 | |
403015b3 JH |
1170 | local_irq_restore(flags); |
1171 | } | |
1172 | ||
1880afd6 JH |
1173 | /** |
1174 | * kvm_trap_emul_gva_fault() - Safely attempt to handle a GVA access fault. | |
1175 | * @vcpu: Virtual CPU. | |
1176 | * @gva: Guest virtual address to be accessed. | |
1177 | * @write: True if write attempted (must be dirtied and made writable). | |
1178 | * | |
1179 | * Safely attempt to handle a GVA fault, mapping GVA pages if necessary, and | |
1180 | * dirtying the page if @write so that guest instructions can be modified. | |
1181 | * | |
1182 | * Returns: KVM_MIPS_MAPPED on success. | |
1183 | * KVM_MIPS_GVA if bad guest virtual address. | |
1184 | * KVM_MIPS_GPA if bad guest physical address. | |
1185 | * KVM_MIPS_TLB if guest TLB not present. | |
1186 | * KVM_MIPS_TLBINV if guest TLB present but not valid. | |
1187 | * KVM_MIPS_TLBMOD if guest TLB read only. | |
1188 | */ | |
1189 | enum kvm_mips_fault_result kvm_trap_emul_gva_fault(struct kvm_vcpu *vcpu, | |
1190 | unsigned long gva, | |
1191 | bool write) | |
1192 | { | |
1193 | struct mips_coproc *cop0 = vcpu->arch.cop0; | |
1194 | struct kvm_mips_tlb *tlb; | |
1195 | int index; | |
1196 | ||
1197 | if (KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG0) { | |
577ed7f7 | 1198 | if (kvm_mips_handle_kseg0_tlb_fault(gva, vcpu, write) < 0) |
1880afd6 JH |
1199 | return KVM_MIPS_GPA; |
1200 | } else if ((KVM_GUEST_KSEGX(gva) < KVM_GUEST_KSEG0) || | |
1201 | KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG23) { | |
1202 | /* Address should be in the guest TLB */ | |
1203 | index = kvm_mips_guest_tlb_lookup(vcpu, (gva & VPN2_MASK) | | |
1204 | (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID)); | |
1205 | if (index < 0) | |
1206 | return KVM_MIPS_TLB; | |
1207 | tlb = &vcpu->arch.guest_tlb[index]; | |
1208 | ||
1209 | /* Entry should be valid, and dirty for writes */ | |
1210 | if (!TLB_IS_VALID(*tlb, gva)) | |
1211 | return KVM_MIPS_TLBINV; | |
1212 | if (write && !TLB_IS_DIRTY(*tlb, gva)) | |
1213 | return KVM_MIPS_TLBMOD; | |
1214 | ||
577ed7f7 | 1215 | if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, gva, write)) |
1880afd6 JH |
1216 | return KVM_MIPS_GPA; |
1217 | } else { | |
1218 | return KVM_MIPS_GVA; | |
1219 | } | |
1220 | ||
1221 | return KVM_MIPS_MAPPED; | |
1222 | } | |
1223 | ||
122e51d4 | 1224 | int kvm_get_inst(u32 *opc, struct kvm_vcpu *vcpu, u32 *out) |
403015b3 | 1225 | { |
dacc3ed1 JH |
1226 | int err; |
1227 | ||
5207ce14 JH |
1228 | retry: |
1229 | kvm_trap_emul_gva_lockless_begin(vcpu); | |
122e51d4 | 1230 | err = get_user(*out, opc); |
5207ce14 JH |
1231 | kvm_trap_emul_gva_lockless_end(vcpu); |
1232 | ||
dacc3ed1 | 1233 | if (unlikely(err)) { |
5207ce14 JH |
1234 | /* |
1235 | * Try to handle the fault, maybe we just raced with a GVA | |
1236 | * invalidation. | |
1237 | */ | |
1238 | err = kvm_trap_emul_gva_fault(vcpu, (unsigned long)opc, | |
1239 | false); | |
1240 | if (unlikely(err)) { | |
1241 | kvm_err("%s: illegal address: %p\n", | |
1242 | __func__, opc); | |
1243 | return -EFAULT; | |
1244 | } | |
403015b3 | 1245 | |
5207ce14 JH |
1246 | /* Hopefully it'll work now */ |
1247 | goto retry; | |
1248 | } | |
122e51d4 | 1249 | return 0; |
403015b3 | 1250 | } |