Merge branches 'pm-cpuidle', 'pm-sleep' and 'pm-powercap'
[linux-block.git] / arch / x86 / kvm / mmu / mmu.c
CommitLineData
20c8ccb1 1// SPDX-License-Identifier: GPL-2.0-only
6aa8b732
AK
2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
7 *
8 * MMU support
9 *
10 * Copyright (C) 2006 Qumranet, Inc.
9611c187 11 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
6aa8b732
AK
12 *
13 * Authors:
14 * Yaniv Kamay <yaniv@qumranet.com>
15 * Avi Kivity <avi@qumranet.com>
6aa8b732 16 */
8d20bd63 17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
e495606d 18
af585b92 19#include "irq.h"
88197e6a 20#include "ioapic.h"
1d737c8a 21#include "mmu.h"
6ca9a6f3 22#include "mmu_internal.h"
fe5db27d 23#include "tdp_mmu.h"
836a1b3c 24#include "x86.h"
6de4f3ad 25#include "kvm_cache_regs.h"
b0b42197 26#include "smm.h"
2f728d66 27#include "kvm_emulate.h"
58ea7cf7 28#include "page_track.h"
5f7dde7b 29#include "cpuid.h"
5a9624af 30#include "spte.h"
e495606d 31
edf88417 32#include <linux/kvm_host.h>
6aa8b732
AK
33#include <linux/types.h>
34#include <linux/string.h>
6aa8b732
AK
35#include <linux/mm.h>
36#include <linux/highmem.h>
1767e931
PG
37#include <linux/moduleparam.h>
38#include <linux/export.h>
448353ca 39#include <linux/swap.h>
05da4558 40#include <linux/hugetlb.h>
2f333bcb 41#include <linux/compiler.h>
bc6678a3 42#include <linux/srcu.h>
5a0e3ad6 43#include <linux/slab.h>
3f07c014 44#include <linux/sched/signal.h>
bf998156 45#include <linux/uaccess.h>
114df303 46#include <linux/hash.h>
f160c7b7 47#include <linux/kern_levels.h>
11b36fe7 48#include <linux/kstrtox.h>
1aa9b957 49#include <linux/kthread.h>
66a5c40f 50#include <linux/wordpart.h>
6aa8b732 51
e495606d 52#include <asm/page.h>
eb243d1d 53#include <asm/memtype.h>
e495606d 54#include <asm/cmpxchg.h>
4e542370 55#include <asm/io.h>
4a98623d 56#include <asm/set_memory.h>
65efc4dc 57#include <asm/spec-ctrl.h>
13673a90 58#include <asm/vmx.h>
58ea7cf7 59
1261bfa3 60#include "trace.h"
6aa8b732 61
0b210faf
SC
62static bool nx_hugepage_mitigation_hard_disabled;
63
a9d6496d 64int __read_mostly nx_huge_pages = -1;
4dfe4f40 65static uint __read_mostly nx_huge_pages_recovery_period_ms;
13fb5927
PB
66#ifdef CONFIG_PREEMPT_RT
67/* Recovery can cause latency spikes, disable it for PREEMPT_RT. */
68static uint __read_mostly nx_huge_pages_recovery_ratio = 0;
69#else
1aa9b957 70static uint __read_mostly nx_huge_pages_recovery_ratio = 60;
13fb5927 71#endif
b8e8c830 72
0b210faf 73static int get_nx_huge_pages(char *buffer, const struct kernel_param *kp);
b8e8c830 74static int set_nx_huge_pages(const char *val, const struct kernel_param *kp);
4dfe4f40 75static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp);
b8e8c830 76
d5d6c18d 77static const struct kernel_param_ops nx_huge_pages_ops = {
b8e8c830 78 .set = set_nx_huge_pages,
0b210faf 79 .get = get_nx_huge_pages,
b8e8c830
PB
80};
81
4dfe4f40
JS
82static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = {
83 .set = set_nx_huge_pages_recovery_param,
1aa9b957
JS
84 .get = param_get_uint,
85};
86
b8e8c830
PB
87module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644);
88__MODULE_PARM_TYPE(nx_huge_pages, "bool");
4dfe4f40 89module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops,
1aa9b957
JS
90 &nx_huge_pages_recovery_ratio, 0644);
91__MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint");
4dfe4f40
JS
92module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops,
93 &nx_huge_pages_recovery_period_ms, 0644);
94__MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint");
b8e8c830 95
71fe7013
SC
96static bool __read_mostly force_flush_and_sync_on_reuse;
97module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);
98
18552672
JR
99/*
100 * When setting this variable to true it enables Two-Dimensional-Paging
101 * where the hardware walks 2 page tables:
102 * 1. the guest-virtual to guest-physical
103 * 2. while doing 1. it walks guest-physical to host-physical
104 * If the hardware supports that we don't need to do shadow paging.
105 */
2f333bcb 106bool tdp_enabled = false;
18552672 107
7f604e92 108static bool __ro_after_init tdp_mmu_allowed;
1f98f2bd
DM
109
110#ifdef CONFIG_X86_64
111bool __read_mostly tdp_mmu_enabled = true;
112module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0444);
113#endif
114
1d92d2e8 115static int max_huge_page_level __read_mostly;
746700d2 116static int tdp_root_level __read_mostly;
83013059 117static int max_tdp_level __read_mostly;
703c335d 118
957ed9ef
XG
119#define PTE_PREFETCH_NUM 8
120
90bb6fc5
AK
121#include <trace/events/kvm.h>
122
dc1cff96 123/* make pte_list_desc fit well in cache lines */
13236e25 124#define PTE_LIST_EXT 14
220f773a 125
13236e25 126/*
141705b7
LJ
127 * struct pte_list_desc is the core data structure used to implement a custom
128 * list for tracking a set of related SPTEs, e.g. all the SPTEs that map a
129 * given GFN when used in the context of rmaps. Using a custom list allows KVM
130 * to optimize for the common case where many GFNs will have at most a handful
131 * of SPTEs pointing at them, i.e. allows packing multiple SPTEs into a small
132 * memory footprint, which in turn improves runtime performance by exploiting
133 * cache locality.
134 *
135 * A list is comprised of one or more pte_list_desc objects (descriptors).
136 * Each individual descriptor stores up to PTE_LIST_EXT SPTEs. If a descriptor
137 * is full and a new SPTEs needs to be added, a new descriptor is allocated and
138 * becomes the head of the list. This means that by definitions, all tail
139 * descriptors are full.
140 *
141 * Note, the meta data fields are deliberately placed at the start of the
142 * structure to optimize the cacheline layout; accessing the descriptor will
143 * touch only a single cacheline so long as @spte_count<=6 (or if only the
144 * descriptors metadata is accessed).
13236e25 145 */
53c07b18 146struct pte_list_desc {
53c07b18 147 struct pte_list_desc *more;
141705b7
LJ
148 /* The number of PTEs stored in _this_ descriptor. */
149 u32 spte_count;
150 /* The number of PTEs stored in all tails of this descriptor. */
151 u32 tail_count;
13236e25 152 u64 *sptes[PTE_LIST_EXT];
cd4a4e53
AK
153};
154
2d11123a
AK
155struct kvm_shadow_walk_iterator {
156 u64 addr;
157 hpa_t shadow_addr;
2d11123a 158 u64 *sptep;
dd3bfd59 159 int level;
2d11123a
AK
160 unsigned index;
161};
162
7eb77e9f
JS
163#define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker) \
164 for (shadow_walk_init_using_root(&(_walker), (_vcpu), \
165 (_root), (_addr)); \
166 shadow_walk_okay(&(_walker)); \
167 shadow_walk_next(&(_walker)))
168
169#define for_each_shadow_entry(_vcpu, _addr, _walker) \
2d11123a
AK
170 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
171 shadow_walk_okay(&(_walker)); \
172 shadow_walk_next(&(_walker)))
173
c2a2ac2b
XG
174#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte) \
175 for (shadow_walk_init(&(_walker), _vcpu, _addr); \
176 shadow_walk_okay(&(_walker)) && \
177 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
178 __shadow_walk_next(&(_walker), spte))
179
53c07b18 180static struct kmem_cache *pte_list_desc_cache;
02c00b3a 181struct kmem_cache *mmu_page_header_cache;
45221ab6 182static struct percpu_counter kvm_total_used_mmu_pages;
b5a33a75 183
ce88decf
XG
184static void mmu_spte_set(u64 *sptep, u64 spte);
185
594e91a1
SC
186struct kvm_mmu_role_regs {
187 const unsigned long cr0;
188 const unsigned long cr4;
189 const u64 efer;
190};
191
335e192a
PB
192#define CREATE_TRACE_POINTS
193#include "mmutrace.h"
194
594e91a1
SC
195/*
196 * Yes, lot's of underscores. They're a hint that you probably shouldn't be
7a458f0e 197 * reading from the role_regs. Once the root_role is constructed, it becomes
594e91a1
SC
198 * the single source of truth for the MMU's state.
199 */
200#define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag) \
82ffa13f
PB
201static inline bool __maybe_unused \
202____is_##reg##_##name(const struct kvm_mmu_role_regs *regs) \
594e91a1
SC
203{ \
204 return !!(regs->reg & flag); \
205}
206BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG);
207BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP);
208BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE);
209BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE);
210BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP);
211BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP);
212BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE);
213BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57);
214BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX);
215BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
216
60667724
SC
217/*
218 * The MMU itself (with a valid role) is the single source of truth for the
219 * MMU. Do not use the regs used to build the MMU/role, nor the vCPU. The
220 * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1,
221 * and the vCPU may be incorrect/irrelevant.
222 */
223#define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name) \
4ac21457 224static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu) \
60667724 225{ \
e5ed0fb0 226 return !!(mmu->cpu_role. base_or_ext . reg##_##name); \
60667724 227}
60667724
SC
228BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
229BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pse);
60667724
SC
230BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smep);
231BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smap);
232BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pke);
233BUILD_MMU_ROLE_ACCESSOR(ext, cr4, la57);
234BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
56b321f9 235BUILD_MMU_ROLE_ACCESSOR(ext, efer, lma);
60667724 236
faf72962
PB
237static inline bool is_cr0_pg(struct kvm_mmu *mmu)
238{
239 return mmu->cpu_role.base.level > 0;
240}
241
242static inline bool is_cr4_pae(struct kvm_mmu *mmu)
243{
244 return !mmu->cpu_role.base.has_4_byte_gpte;
245}
246
594e91a1
SC
247static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
248{
249 struct kvm_mmu_role_regs regs = {
250 .cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
251 .cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
252 .efer = vcpu->arch.efer,
253 };
254
255 return regs;
256}
40ef75a7 257
2fdcc1b3 258static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)
40ef75a7 259{
2fdcc1b3 260 return kvm_read_cr3(vcpu);
40ef75a7
LT
261}
262
2fdcc1b3
PB
263static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,
264 struct kvm_mmu *mmu)
40ef75a7 265{
aefb2f2e 266 if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)
2fdcc1b3 267 return kvm_read_cr3(vcpu);
40ef75a7 268
2fdcc1b3 269 return mmu->get_guest_pgd(vcpu);
40ef75a7
LT
270}
271
8a1300ff 272static inline bool kvm_available_flush_remote_tlbs_range(void)
40ef75a7 273{
0277022a 274#if IS_ENABLED(CONFIG_HYPERV)
8a1300ff 275 return kvm_x86_ops.flush_remote_tlbs_range;
0277022a
SC
276#else
277 return false;
278#endif
40ef75a7
LT
279}
280
1b2dc736
HW
281static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index);
282
283/* Flush the range of guest memory mapped by the given SPTE. */
284static void kvm_flush_remote_tlbs_sptep(struct kvm *kvm, u64 *sptep)
285{
286 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
287 gfn_t gfn = kvm_mmu_page_get_gfn(sp, spte_index(sptep));
288
289 kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);
290}
291
8f79b064
BG
292static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
293 unsigned int access)
294{
c236d962 295 u64 spte = make_mmio_spte(vcpu, gfn, access);
8f79b064 296
c236d962
SC
297 trace_mark_mmio_spte(sptep, gfn, spte);
298 mmu_spte_set(sptep, spte);
ce88decf
XG
299}
300
ce88decf
XG
301static gfn_t get_mmio_spte_gfn(u64 spte)
302{
daa07cbc 303 u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
28a1f3ac 304
8a967d65 305 gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)
28a1f3ac
JS
306 & shadow_nonpresent_or_rsvd_mask;
307
308 return gpa >> PAGE_SHIFT;
ce88decf
XG
309}
310
311static unsigned get_mmio_spte_access(u64 spte)
312{
4af77151 313 return spte & shadow_mmio_access_mask;
ce88decf
XG
314}
315
54bf36aa 316static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
f8f55942 317{
cae7ed3c 318 u64 kvm_gen, spte_gen, gen;
089504c0 319
cae7ed3c
SC
320 gen = kvm_vcpu_memslots(vcpu)->generation;
321 if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
322 return false;
089504c0 323
cae7ed3c 324 kvm_gen = gen & MMIO_SPTE_GEN_MASK;
089504c0
XG
325 spte_gen = get_mmio_spte_generation(spte);
326
327 trace_check_mmio_spte(spte, kvm_gen, spte_gen);
328 return likely(kvm_gen == spte_gen);
f8f55942
XG
329}
330
6aa8b732
AK
331static int is_cpuid_PSE36(void)
332{
333 return 1;
334}
335
603e0651 336#ifdef CONFIG_X86_64
d555c333 337static void __set_spte(u64 *sptep, u64 spte)
e663ee64 338{
b19ee2ff 339 WRITE_ONCE(*sptep, spte);
e663ee64
AK
340}
341
603e0651 342static void __update_clear_spte_fast(u64 *sptep, u64 spte)
a9221dd5 343{
b19ee2ff 344 WRITE_ONCE(*sptep, spte);
603e0651
XG
345}
346
347static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
348{
349 return xchg(sptep, spte);
350}
c2a2ac2b
XG
351
352static u64 __get_spte_lockless(u64 *sptep)
353{
6aa7de05 354 return READ_ONCE(*sptep);
c2a2ac2b 355}
a9221dd5 356#else
603e0651
XG
357union split_spte {
358 struct {
359 u32 spte_low;
360 u32 spte_high;
361 };
362 u64 spte;
363};
a9221dd5 364
c2a2ac2b
XG
365static void count_spte_clear(u64 *sptep, u64 spte)
366{
57354682 367 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
c2a2ac2b
XG
368
369 if (is_shadow_present_pte(spte))
370 return;
371
372 /* Ensure the spte is completely set before we increase the count */
373 smp_wmb();
374 sp->clear_spte_count++;
375}
376
603e0651
XG
377static void __set_spte(u64 *sptep, u64 spte)
378{
379 union split_spte *ssptep, sspte;
a9221dd5 380
603e0651
XG
381 ssptep = (union split_spte *)sptep;
382 sspte = (union split_spte)spte;
383
384 ssptep->spte_high = sspte.spte_high;
385
386 /*
387 * If we map the spte from nonpresent to present, We should store
388 * the high bits firstly, then set present bit, so cpu can not
389 * fetch this spte while we are setting the spte.
390 */
391 smp_wmb();
392
b19ee2ff 393 WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
a9221dd5
AK
394}
395
603e0651
XG
396static void __update_clear_spte_fast(u64 *sptep, u64 spte)
397{
398 union split_spte *ssptep, sspte;
399
400 ssptep = (union split_spte *)sptep;
401 sspte = (union split_spte)spte;
402
b19ee2ff 403 WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
603e0651
XG
404
405 /*
406 * If we map the spte from present to nonpresent, we should clear
407 * present bit firstly to avoid vcpu fetch the old high bits.
408 */
409 smp_wmb();
410
411 ssptep->spte_high = sspte.spte_high;
c2a2ac2b 412 count_spte_clear(sptep, spte);
603e0651
XG
413}
414
415static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
416{
417 union split_spte *ssptep, sspte, orig;
418
419 ssptep = (union split_spte *)sptep;
420 sspte = (union split_spte)spte;
421
422 /* xchg acts as a barrier before the setting of the high bits */
423 orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
41bc3186
ZJ
424 orig.spte_high = ssptep->spte_high;
425 ssptep->spte_high = sspte.spte_high;
c2a2ac2b 426 count_spte_clear(sptep, spte);
603e0651
XG
427
428 return orig.spte;
429}
c2a2ac2b
XG
430
431/*
432 * The idea using the light way get the spte on x86_32 guest is from
39656e83 433 * gup_get_pte (mm/gup.c).
accaefe0 434 *
aed02fe3 435 * An spte tlb flush may be pending, because kvm_set_pte_rmap
accaefe0
XG
436 * coalesces them and we are running out of the MMU lock. Therefore
437 * we need to protect against in-progress updates of the spte.
438 *
439 * Reading the spte while an update is in progress may get the old value
440 * for the high part of the spte. The race is fine for a present->non-present
441 * change (because the high part of the spte is ignored for non-present spte),
442 * but for a present->present change we must reread the spte.
443 *
444 * All such changes are done in two steps (present->non-present and
445 * non-present->present), hence it is enough to count the number of
446 * present->non-present updates: if it changed while reading the spte,
447 * we might have hit the race. This is done using clear_spte_count.
c2a2ac2b
XG
448 */
449static u64 __get_spte_lockless(u64 *sptep)
450{
57354682 451 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
c2a2ac2b
XG
452 union split_spte spte, *orig = (union split_spte *)sptep;
453 int count;
454
455retry:
456 count = sp->clear_spte_count;
457 smp_rmb();
458
459 spte.spte_low = orig->spte_low;
460 smp_rmb();
461
462 spte.spte_high = orig->spte_high;
463 smp_rmb();
464
465 if (unlikely(spte.spte_low != orig->spte_low ||
466 count != sp->clear_spte_count))
467 goto retry;
468
469 return spte.spte;
470}
603e0651
XG
471#endif
472
1df9f2dc
XG
473/* Rules for using mmu_spte_set:
474 * Set the sptep from nonpresent to present.
475 * Note: the sptep being assigned *must* be either not present
476 * or in a state where the hardware will not attempt to update
477 * the spte.
478 */
479static void mmu_spte_set(u64 *sptep, u64 new_spte)
480{
20ba462d 481 WARN_ON_ONCE(is_shadow_present_pte(*sptep));
1df9f2dc
XG
482 __set_spte(sptep, new_spte);
483}
484
f39a058d
JS
485/*
486 * Update the SPTE (excluding the PFN), but do not track changes in its
487 * accessed/dirty status.
1df9f2dc 488 */
f39a058d 489static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
b79b93f9 490{
c7ba5b48 491 u64 old_spte = *sptep;
4132779b 492
20ba462d 493 WARN_ON_ONCE(!is_shadow_present_pte(new_spte));
115111ef 494 check_spte_writable_invariants(new_spte);
b79b93f9 495
6e7d0354
XG
496 if (!is_shadow_present_pte(old_spte)) {
497 mmu_spte_set(sptep, new_spte);
f39a058d 498 return old_spte;
6e7d0354 499 }
4132779b 500
c7ba5b48 501 if (!spte_has_volatile_bits(old_spte))
603e0651 502 __update_clear_spte_fast(sptep, new_spte);
4132779b 503 else
603e0651 504 old_spte = __update_clear_spte_slow(sptep, new_spte);
4132779b 505
20ba462d 506 WARN_ON_ONCE(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));
83ef6c81 507
f39a058d
JS
508 return old_spte;
509}
510
511/* Rules for using mmu_spte_update:
512 * Update the state bits, it means the mapped pfn is not changed.
513 *
02844ac1
DM
514 * Whenever an MMU-writable SPTE is overwritten with a read-only SPTE, remote
515 * TLBs must be flushed. Otherwise rmap_write_protect will find a read-only
516 * spte, even though the writable spte might be cached on a CPU's TLB.
f39a058d
JS
517 *
518 * Returns true if the TLB needs to be flushed
519 */
520static bool mmu_spte_update(u64 *sptep, u64 new_spte)
521{
522 bool flush = false;
523 u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);
524
525 if (!is_shadow_present_pte(old_spte))
526 return false;
527
c7ba5b48
XG
528 /*
529 * For the spte updated out of mmu-lock is safe, since
6a6256f9 530 * we always atomically update it, see the comments in
c7ba5b48
XG
531 * spte_has_volatile_bits().
532 */
706c9c55 533 if (is_mmu_writable_spte(old_spte) &&
7f31c959 534 !is_writable_pte(new_spte))
83ef6c81 535 flush = true;
4132779b 536
7e71a59b 537 /*
83ef6c81 538 * Flush TLB when accessed/dirty states are changed in the page tables,
7e71a59b
KH
539 * to guarantee consistency between TLB and page tables.
540 */
7e71a59b 541
83ef6c81
JS
542 if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
543 flush = true;
4132779b 544 kvm_set_pfn_accessed(spte_to_pfn(old_spte));
83ef6c81
JS
545 }
546
547 if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
548 flush = true;
4132779b 549 kvm_set_pfn_dirty(spte_to_pfn(old_spte));
83ef6c81 550 }
6e7d0354 551
83ef6c81 552 return flush;
b79b93f9
AK
553}
554
1df9f2dc
XG
555/*
556 * Rules for using mmu_spte_clear_track_bits:
557 * It sets the sptep from present to nonpresent, and track the
558 * state bits, it is used to clear the last level sptep.
7fa2a347 559 * Returns the old PTE.
1df9f2dc 560 */
35d539c3 561static u64 mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
1df9f2dc 562{
ba049e93 563 kvm_pfn_t pfn;
1df9f2dc 564 u64 old_spte = *sptep;
71f51d2c 565 int level = sptep_to_sp(sptep)->role.level;
b14b2690 566 struct page *page;
1df9f2dc 567
54eb3ef5
SC
568 if (!is_shadow_present_pte(old_spte) ||
569 !spte_has_volatile_bits(old_spte))
603e0651 570 __update_clear_spte_fast(sptep, 0ull);
1df9f2dc 571 else
603e0651 572 old_spte = __update_clear_spte_slow(sptep, 0ull);
1df9f2dc 573
afd28fe1 574 if (!is_shadow_present_pte(old_spte))
7fa2a347 575 return old_spte;
1df9f2dc 576
71f51d2c
MZ
577 kvm_update_page_stats(kvm, level, -1);
578
1df9f2dc 579 pfn = spte_to_pfn(old_spte);
86fde74c
XG
580
581 /*
b14b2690
SC
582 * KVM doesn't hold a reference to any pages mapped into the guest, and
583 * instead uses the mmu_notifier to ensure that KVM unmaps any pages
584 * before they are reclaimed. Sanity check that, if the pfn is backed
585 * by a refcounted page, the refcount is elevated.
86fde74c 586 */
b14b2690 587 page = kvm_pfn_to_refcounted_page(pfn);
20ba462d 588 WARN_ON_ONCE(page && !page_count(page));
86fde74c 589
83ef6c81 590 if (is_accessed_spte(old_spte))
1df9f2dc 591 kvm_set_pfn_accessed(pfn);
83ef6c81
JS
592
593 if (is_dirty_spte(old_spte))
1df9f2dc 594 kvm_set_pfn_dirty(pfn);
83ef6c81 595
7fa2a347 596 return old_spte;
1df9f2dc
XG
597}
598
599/*
600 * Rules for using mmu_spte_clear_no_track:
601 * Directly clear spte without caring the state bits of sptep,
602 * it is used to set the upper level spte.
603 */
604static void mmu_spte_clear_no_track(u64 *sptep)
605{
603e0651 606 __update_clear_spte_fast(sptep, 0ull);
1df9f2dc
XG
607}
608
c2a2ac2b
XG
609static u64 mmu_spte_get_lockless(u64 *sptep)
610{
611 return __get_spte_lockless(sptep);
612}
613
f160c7b7
JS
614/* Returns the Accessed status of the PTE and resets it at the same time. */
615static bool mmu_spte_age(u64 *sptep)
616{
617 u64 spte = mmu_spte_get_lockless(sptep);
618
619 if (!is_accessed_spte(spte))
620 return false;
621
ac8d57e5 622 if (spte_ad_enabled(spte)) {
f160c7b7
JS
623 clear_bit((ffs(shadow_accessed_mask) - 1),
624 (unsigned long *)sptep);
625 } else {
626 /*
627 * Capture the dirty status of the page, so that it doesn't get
628 * lost when the SPTE is marked for access tracking.
629 */
630 if (is_writable_pte(spte))
631 kvm_set_pfn_dirty(spte_to_pfn(spte));
632
633 spte = mark_spte_for_access_track(spte);
634 mmu_spte_update_no_track(sptep, spte);
635 }
636
637 return true;
638}
639
78fdd2f0
SC
640static inline bool is_tdp_mmu_active(struct kvm_vcpu *vcpu)
641{
642 return tdp_mmu_enabled && vcpu->arch.mmu->root_role.direct;
643}
644
c2a2ac2b
XG
645static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
646{
78fdd2f0 647 if (is_tdp_mmu_active(vcpu)) {
c5c8c7c5
DM
648 kvm_tdp_mmu_walk_lockless_begin();
649 } else {
650 /*
651 * Prevent page table teardown by making any free-er wait during
652 * kvm_flush_remote_tlbs() IPI to all active vcpus.
653 */
654 local_irq_disable();
36ca7e0a 655
c5c8c7c5
DM
656 /*
657 * Make sure a following spte read is not reordered ahead of the write
658 * to vcpu->mode.
659 */
660 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
661 }
c2a2ac2b
XG
662}
663
664static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
665{
78fdd2f0 666 if (is_tdp_mmu_active(vcpu)) {
c5c8c7c5
DM
667 kvm_tdp_mmu_walk_lockless_end();
668 } else {
669 /*
670 * Make sure the write to vcpu->mode is not reordered in front of
671 * reads to sptes. If it does, kvm_mmu_commit_zap_page() can see us
672 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
673 */
674 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
675 local_irq_enable();
676 }
c2a2ac2b
XG
677}
678
378f5cd6 679static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
714b93da 680{
e2dec939
AK
681 int r;
682
531281ad 683 /* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */
94ce87ef
SC
684 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
685 1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM);
d3d25b04 686 if (r)
284aa868 687 return r;
94ce87ef
SC
688 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
689 PT64_ROOT_MAX_LEVEL);
d3d25b04 690 if (r)
171a90d7 691 return r;
378f5cd6 692 if (maybe_indirect) {
6a97575d 693 r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadowed_info_cache,
94ce87ef 694 PT64_ROOT_MAX_LEVEL);
378f5cd6
SC
695 if (r)
696 return r;
697 }
94ce87ef
SC
698 return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
699 PT64_ROOT_MAX_LEVEL);
714b93da
AK
700}
701
702static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
703{
94ce87ef
SC
704 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);
705 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);
6a97575d 706 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadowed_info_cache);
94ce87ef 707 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
714b93da
AK
708}
709
53c07b18 710static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
714b93da 711{
53c07b18 712 kmem_cache_free(pte_list_desc_cache, pte_list_desc);
714b93da
AK
713}
714
6a97575d
DM
715static bool sp_has_gptes(struct kvm_mmu_page *sp);
716
2032a93d
LJ
717static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
718{
84e5ffd0
LJ
719 if (sp->role.passthrough)
720 return sp->gfn;
721
2032a93d 722 if (!sp->role.direct)
6a97575d 723 return sp->shadowed_translation[index] >> PAGE_SHIFT;
2032a93d 724
2ca3129e 725 return sp->gfn + (index << ((sp->role.level - 1) * SPTE_LEVEL_BITS));
2032a93d
LJ
726}
727
6a97575d
DM
728/*
729 * For leaf SPTEs, fetch the *guest* access permissions being shadowed. Note
730 * that the SPTE itself may have a more constrained access permissions that
731 * what the guest enforces. For example, a guest may create an executable
732 * huge PTE but KVM may disallow execution to mitigate iTLB multihit.
733 */
734static u32 kvm_mmu_page_get_access(struct kvm_mmu_page *sp, int index)
2032a93d 735{
6a97575d
DM
736 if (sp_has_gptes(sp))
737 return sp->shadowed_translation[index] & ACC_ALL;
84e5ffd0 738
6a97575d
DM
739 /*
740 * For direct MMUs (e.g. TDP or non-paging guests) or passthrough SPs,
741 * KVM is not shadowing any guest page tables, so the "guest access
742 * permissions" are just ACC_ALL.
743 *
744 * For direct SPs in indirect MMUs (shadow paging), i.e. when KVM
745 * is shadowing a guest huge page with small pages, the guest access
746 * permissions being shadowed are the access permissions of the huge
747 * page.
748 *
749 * In both cases, sp->role.access contains the correct access bits.
750 */
751 return sp->role.access;
752}
753
72ae5822
SC
754static void kvm_mmu_page_set_translation(struct kvm_mmu_page *sp, int index,
755 gfn_t gfn, unsigned int access)
6a97575d
DM
756{
757 if (sp_has_gptes(sp)) {
758 sp->shadowed_translation[index] = (gfn << PAGE_SHIFT) | access;
e9f2a760
PB
759 return;
760 }
761
6a97575d
DM
762 WARN_ONCE(access != kvm_mmu_page_get_access(sp, index),
763 "access mismatch under %s page %llx (expected %u, got %u)\n",
764 sp->role.passthrough ? "passthrough" : "direct",
765 sp->gfn, kvm_mmu_page_get_access(sp, index), access);
766
767 WARN_ONCE(gfn != kvm_mmu_page_get_gfn(sp, index),
768 "gfn mismatch under %s page %llx (expected %llx, got %llx)\n",
769 sp->role.passthrough ? "passthrough" : "direct",
770 sp->gfn, kvm_mmu_page_get_gfn(sp, index), gfn);
771}
772
72ae5822
SC
773static void kvm_mmu_page_set_access(struct kvm_mmu_page *sp, int index,
774 unsigned int access)
6a97575d
DM
775{
776 gfn_t gfn = kvm_mmu_page_get_gfn(sp, index);
777
778 kvm_mmu_page_set_translation(sp, index, gfn, access);
2032a93d
LJ
779}
780
05da4558 781/*
d4dbf470
TY
782 * Return the pointer to the large page information for a given gfn,
783 * handling slots that are not large page aligned.
05da4558 784 */
d4dbf470 785static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
8ca6f063 786 const struct kvm_memory_slot *slot, int level)
05da4558
MT
787{
788 unsigned long idx;
789
fb03cb6f 790 idx = gfn_to_index(gfn, slot->base_gfn, level);
db3fe4eb 791 return &slot->arch.lpage_info[level - 2][idx];
05da4558
MT
792}
793
90b4fe17
CP
794/*
795 * The most significant bit in disallow_lpage tracks whether or not memory
796 * attributes are mixed, i.e. not identical for all gfns at the current level.
797 * The lower order bits are used to refcount other cases where a hugepage is
798 * disallowed, e.g. if KVM has shadow a page table at the gfn.
799 */
800#define KVM_LPAGE_MIXED_FLAG BIT(31)
801
269e9552 802static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
547ffaed
XG
803 gfn_t gfn, int count)
804{
805 struct kvm_lpage_info *linfo;
90b4fe17 806 int old, i;
547ffaed 807
3bae0459 808 for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
547ffaed 809 linfo = lpage_info_slot(gfn, slot, i);
90b4fe17
CP
810
811 old = linfo->disallow_lpage;
547ffaed 812 linfo->disallow_lpage += count;
90b4fe17 813 WARN_ON_ONCE((old ^ linfo->disallow_lpage) & KVM_LPAGE_MIXED_FLAG);
547ffaed
XG
814 }
815}
816
269e9552 817void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
547ffaed
XG
818{
819 update_gfn_disallow_lpage_count(slot, gfn, 1);
820}
821
269e9552 822void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
547ffaed
XG
823{
824 update_gfn_disallow_lpage_count(slot, gfn, -1);
825}
826
3ed1a478 827static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
05da4558 828{
699023e2 829 struct kvm_memslots *slots;
d25797b2 830 struct kvm_memory_slot *slot;
3ed1a478 831 gfn_t gfn;
05da4558 832
56ca57f9 833 kvm->arch.indirect_shadow_pages++;
3ed1a478 834 gfn = sp->gfn;
699023e2
PB
835 slots = kvm_memslots_for_spte_role(kvm, sp->role);
836 slot = __gfn_to_memslot(slots, gfn);
56ca57f9
XG
837
838 /* the non-leaf shadow pages are keeping readonly. */
3bae0459 839 if (sp->role.level > PG_LEVEL_4K)
96316a06 840 return __kvm_write_track_add_gfn(kvm, slot, gfn);
56ca57f9 841
547ffaed 842 kvm_mmu_gfn_disallow_lpage(slot, gfn);
be911771
DM
843
844 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
4ad980ae 845 kvm_flush_remote_tlbs_gfn(kvm, gfn, PG_LEVEL_4K);
05da4558
MT
846}
847
61f94478 848void track_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
b8e8c830 849{
428e9216
SC
850 /*
851 * If it's possible to replace the shadow page with an NX huge page,
852 * i.e. if the shadow page is the only thing currently preventing KVM
853 * from using a huge page, add the shadow page to the list of "to be
854 * zapped for NX recovery" pages. Note, the shadow page can already be
855 * on the list if KVM is reusing an existing shadow page, i.e. if KVM
856 * links a shadow page at multiple points.
857 */
61f94478 858 if (!list_empty(&sp->possible_nx_huge_page_link))
b8e8c830
PB
859 return;
860
861 ++kvm->stat.nx_lpage_splits;
55c510e2
SC
862 list_add_tail(&sp->possible_nx_huge_page_link,
863 &kvm->arch.possible_nx_huge_pages);
b8e8c830
PB
864}
865
61f94478
SC
866static void account_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
867 bool nx_huge_page_possible)
868{
869 sp->nx_huge_page_disallowed = true;
870
871 if (nx_huge_page_possible)
872 track_possible_nx_huge_page(kvm, sp);
b8e8c830
PB
873}
874
3ed1a478 875static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
05da4558 876{
699023e2 877 struct kvm_memslots *slots;
d25797b2 878 struct kvm_memory_slot *slot;
3ed1a478 879 gfn_t gfn;
05da4558 880
56ca57f9 881 kvm->arch.indirect_shadow_pages--;
3ed1a478 882 gfn = sp->gfn;
699023e2
PB
883 slots = kvm_memslots_for_spte_role(kvm, sp->role);
884 slot = __gfn_to_memslot(slots, gfn);
3bae0459 885 if (sp->role.level > PG_LEVEL_4K)
96316a06 886 return __kvm_write_track_remove_gfn(kvm, slot, gfn);
56ca57f9 887
547ffaed 888 kvm_mmu_gfn_allow_lpage(slot, gfn);
05da4558
MT
889}
890
61f94478 891void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
b8e8c830 892{
55c510e2 893 if (list_empty(&sp->possible_nx_huge_page_link))
428e9216
SC
894 return;
895
b8e8c830 896 --kvm->stat.nx_lpage_splits;
55c510e2 897 list_del_init(&sp->possible_nx_huge_page_link);
b8e8c830
PB
898}
899
61f94478
SC
900static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
901{
902 sp->nx_huge_page_disallowed = false;
903
904 untrack_possible_nx_huge_page(kvm, sp);
b8e8c830
PB
905}
906
f3d90f90
SC
907static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu,
908 gfn_t gfn,
909 bool no_dirty_log)
05da4558
MT
910{
911 struct kvm_memory_slot *slot;
5d163b1c 912
54bf36aa 913 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
91b0d268
PB
914 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
915 return NULL;
044c59c4 916 if (no_dirty_log && kvm_slot_dirty_track_enabled(slot))
91b0d268 917 return NULL;
5d163b1c
XG
918
919 return slot;
920}
921
290fc38d 922/*
018aabb5 923 * About rmap_head encoding:
cd4a4e53 924 *
018aabb5
TY
925 * If the bit zero of rmap_head->val is clear, then it points to the only spte
926 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
53c07b18 927 * pte_list_desc containing more mappings.
018aabb5
TY
928 */
929
930/*
931 * Returns the number of pointers in the rmap chain, not counting the new one.
cd4a4e53 932 */
2ff9039a 933static int pte_list_add(struct kvm_mmu_memory_cache *cache, u64 *spte,
018aabb5 934 struct kvm_rmap_head *rmap_head)
cd4a4e53 935{
53c07b18 936 struct pte_list_desc *desc;
13236e25 937 int count = 0;
cd4a4e53 938
018aabb5 939 if (!rmap_head->val) {
018aabb5
TY
940 rmap_head->val = (unsigned long)spte;
941 } else if (!(rmap_head->val & 1)) {
2ff9039a 942 desc = kvm_mmu_memory_cache_alloc(cache);
018aabb5 943 desc->sptes[0] = (u64 *)rmap_head->val;
d555c333 944 desc->sptes[1] = spte;
13236e25 945 desc->spte_count = 2;
141705b7 946 desc->tail_count = 0;
018aabb5 947 rmap_head->val = (unsigned long)desc | 1;
cb16a7b3 948 ++count;
cd4a4e53 949 } else {
018aabb5 950 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
141705b7
LJ
951 count = desc->tail_count + desc->spte_count;
952
953 /*
954 * If the previous head is full, allocate a new head descriptor
955 * as tail descriptors are always kept full.
956 */
957 if (desc->spte_count == PTE_LIST_EXT) {
958 desc = kvm_mmu_memory_cache_alloc(cache);
959 desc->more = (struct pte_list_desc *)(rmap_head->val & ~1ul);
960 desc->spte_count = 0;
961 desc->tail_count = count;
962 rmap_head->val = (unsigned long)desc | 1;
cd4a4e53 963 }
13236e25 964 desc->sptes[desc->spte_count++] = spte;
cd4a4e53 965 }
53a27b39 966 return count;
cd4a4e53
AK
967}
968
069f30c6
MZ
969static void pte_list_desc_remove_entry(struct kvm *kvm,
970 struct kvm_rmap_head *rmap_head,
f3d90f90 971 struct pte_list_desc *desc, int i)
cd4a4e53 972{
141705b7
LJ
973 struct pte_list_desc *head_desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
974 int j = head_desc->spte_count - 1;
cd4a4e53 975
141705b7
LJ
976 /*
977 * The head descriptor should never be empty. A new head is added only
978 * when adding an entry and the previous head is full, and heads are
979 * removed (this flow) when they become empty.
980 */
52e322ed 981 KVM_BUG_ON_DATA_CORRUPTION(j < 0, kvm);
141705b7
LJ
982
983 /*
984 * Replace the to-be-freed SPTE with the last valid entry from the head
985 * descriptor to ensure that tail descriptors are full at all times.
986 * Note, this also means that tail_count is stable for each descriptor.
987 */
988 desc->sptes[i] = head_desc->sptes[j];
989 head_desc->sptes[j] = NULL;
990 head_desc->spte_count--;
991 if (head_desc->spte_count)
cd4a4e53 992 return;
141705b7
LJ
993
994 /*
995 * The head descriptor is empty. If there are no tail descriptors,
54aa699e 996 * nullify the rmap head to mark the list as empty, else point the rmap
141705b7
LJ
997 * head at the next descriptor, i.e. the new head.
998 */
999 if (!head_desc->more)
fe3c2b4c 1000 rmap_head->val = 0;
cd4a4e53 1001 else
141705b7
LJ
1002 rmap_head->val = (unsigned long)head_desc->more | 1;
1003 mmu_free_pte_list_desc(head_desc);
cd4a4e53
AK
1004}
1005
069f30c6
MZ
1006static void pte_list_remove(struct kvm *kvm, u64 *spte,
1007 struct kvm_rmap_head *rmap_head)
cd4a4e53 1008{
53c07b18 1009 struct pte_list_desc *desc;
cd4a4e53
AK
1010 int i;
1011
52e322ed
SC
1012 if (KVM_BUG_ON_DATA_CORRUPTION(!rmap_head->val, kvm))
1013 return;
1014
1015 if (!(rmap_head->val & 1)) {
1016 if (KVM_BUG_ON_DATA_CORRUPTION((u64 *)rmap_head->val != spte, kvm))
1017 return;
1018
018aabb5 1019 rmap_head->val = 0;
cd4a4e53 1020 } else {
018aabb5 1021 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
cd4a4e53 1022 while (desc) {
13236e25 1023 for (i = 0; i < desc->spte_count; ++i) {
d555c333 1024 if (desc->sptes[i] == spte) {
069f30c6
MZ
1025 pte_list_desc_remove_entry(kvm, rmap_head,
1026 desc, i);
cd4a4e53
AK
1027 return;
1028 }
018aabb5 1029 }
cd4a4e53
AK
1030 desc = desc->more;
1031 }
52e322ed
SC
1032
1033 KVM_BUG_ON_DATA_CORRUPTION(true, kvm);
cd4a4e53
AK
1034 }
1035}
1036
9202aee8
SC
1037static void kvm_zap_one_rmap_spte(struct kvm *kvm,
1038 struct kvm_rmap_head *rmap_head, u64 *sptep)
e7912386 1039{
71f51d2c 1040 mmu_spte_clear_track_bits(kvm, sptep);
069f30c6 1041 pte_list_remove(kvm, sptep, rmap_head);
e7912386
WY
1042}
1043
9202aee8
SC
1044/* Return true if at least one SPTE was zapped, false otherwise */
1045static bool kvm_zap_all_rmap_sptes(struct kvm *kvm,
1046 struct kvm_rmap_head *rmap_head)
a75b5404
PX
1047{
1048 struct pte_list_desc *desc, *next;
1049 int i;
1050
1051 if (!rmap_head->val)
1052 return false;
1053
1054 if (!(rmap_head->val & 1)) {
71f51d2c 1055 mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
a75b5404
PX
1056 goto out;
1057 }
1058
1059 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1060
1061 for (; desc; desc = next) {
1062 for (i = 0; i < desc->spte_count; i++)
71f51d2c 1063 mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
a75b5404
PX
1064 next = desc->more;
1065 mmu_free_pte_list_desc(desc);
1066 }
1067out:
1068 /* rmap_head is meaningless now, remember to reset it */
1069 rmap_head->val = 0;
1070 return true;
1071}
1072
3bcd0662
PX
1073unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
1074{
1075 struct pte_list_desc *desc;
3bcd0662
PX
1076
1077 if (!rmap_head->val)
1078 return 0;
1079 else if (!(rmap_head->val & 1))
1080 return 1;
1081
1082 desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
141705b7 1083 return desc->tail_count + desc->spte_count;
3bcd0662
PX
1084}
1085
93e083d4
DM
1086static struct kvm_rmap_head *gfn_to_rmap(gfn_t gfn, int level,
1087 const struct kvm_memory_slot *slot)
53c07b18 1088{
77d11309 1089 unsigned long idx;
53c07b18 1090
77d11309 1091 idx = gfn_to_index(gfn, slot->base_gfn, level);
3bae0459 1092 return &slot->arch.rmap[level - PG_LEVEL_4K][idx];
53c07b18
XG
1093}
1094
53c07b18
XG
1095static void rmap_remove(struct kvm *kvm, u64 *spte)
1096{
601f8af0
DM
1097 struct kvm_memslots *slots;
1098 struct kvm_memory_slot *slot;
53c07b18
XG
1099 struct kvm_mmu_page *sp;
1100 gfn_t gfn;
018aabb5 1101 struct kvm_rmap_head *rmap_head;
53c07b18 1102
57354682 1103 sp = sptep_to_sp(spte);
79e48cec 1104 gfn = kvm_mmu_page_get_gfn(sp, spte_index(spte));
601f8af0
DM
1105
1106 /*
68be1306
DM
1107 * Unlike rmap_add, rmap_remove does not run in the context of a vCPU
1108 * so we have to determine which memslots to use based on context
1109 * information in sp->role.
601f8af0
DM
1110 */
1111 slots = kvm_memslots_for_spte_role(kvm, sp->role);
1112
1113 slot = __gfn_to_memslot(slots, gfn);
93e083d4 1114 rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
601f8af0 1115
069f30c6 1116 pte_list_remove(kvm, spte, rmap_head);
53c07b18
XG
1117}
1118
1e3f42f0
TY
1119/*
1120 * Used by the following functions to iterate through the sptes linked by a
1121 * rmap. All fields are private and not assumed to be used outside.
1122 */
1123struct rmap_iterator {
1124 /* private fields */
1125 struct pte_list_desc *desc; /* holds the sptep if not NULL */
1126 int pos; /* index of the sptep */
1127};
1128
1129/*
1130 * Iteration must be started by this function. This should also be used after
1131 * removing/dropping sptes from the rmap link because in such cases the
0a03cbda 1132 * information in the iterator may not be valid.
1e3f42f0
TY
1133 *
1134 * Returns sptep if found, NULL otherwise.
1135 */
018aabb5
TY
1136static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
1137 struct rmap_iterator *iter)
1e3f42f0 1138{
77fbbbd2
TY
1139 u64 *sptep;
1140
018aabb5 1141 if (!rmap_head->val)
1e3f42f0
TY
1142 return NULL;
1143
018aabb5 1144 if (!(rmap_head->val & 1)) {
1e3f42f0 1145 iter->desc = NULL;
77fbbbd2
TY
1146 sptep = (u64 *)rmap_head->val;
1147 goto out;
1e3f42f0
TY
1148 }
1149
018aabb5 1150 iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1e3f42f0 1151 iter->pos = 0;
77fbbbd2
TY
1152 sptep = iter->desc->sptes[iter->pos];
1153out:
1154 BUG_ON(!is_shadow_present_pte(*sptep));
1155 return sptep;
1e3f42f0
TY
1156}
1157
1158/*
1159 * Must be used with a valid iterator: e.g. after rmap_get_first().
1160 *
1161 * Returns sptep if found, NULL otherwise.
1162 */
1163static u64 *rmap_get_next(struct rmap_iterator *iter)
1164{
77fbbbd2
TY
1165 u64 *sptep;
1166
1e3f42f0
TY
1167 if (iter->desc) {
1168 if (iter->pos < PTE_LIST_EXT - 1) {
1e3f42f0
TY
1169 ++iter->pos;
1170 sptep = iter->desc->sptes[iter->pos];
1171 if (sptep)
77fbbbd2 1172 goto out;
1e3f42f0
TY
1173 }
1174
1175 iter->desc = iter->desc->more;
1176
1177 if (iter->desc) {
1178 iter->pos = 0;
1179 /* desc->sptes[0] cannot be NULL */
77fbbbd2
TY
1180 sptep = iter->desc->sptes[iter->pos];
1181 goto out;
1e3f42f0
TY
1182 }
1183 }
1184
1185 return NULL;
77fbbbd2
TY
1186out:
1187 BUG_ON(!is_shadow_present_pte(*sptep));
1188 return sptep;
1e3f42f0
TY
1189}
1190
018aabb5
TY
1191#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_) \
1192 for (_spte_ = rmap_get_first(_rmap_head_, _iter_); \
77fbbbd2 1193 _spte_; _spte_ = rmap_get_next(_iter_))
0d536790 1194
c3707958 1195static void drop_spte(struct kvm *kvm, u64 *sptep)
e4b502ea 1196{
71f51d2c 1197 u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
7fa2a347
SC
1198
1199 if (is_shadow_present_pte(old_spte))
eb45fda4 1200 rmap_remove(kvm, sptep);
be38d276
AK
1201}
1202
03787394 1203static void drop_large_spte(struct kvm *kvm, u64 *sptep, bool flush)
8e22f955 1204{
0cd8dc73 1205 struct kvm_mmu_page *sp;
8e22f955 1206
0cd8dc73 1207 sp = sptep_to_sp(sptep);
20ba462d 1208 WARN_ON_ONCE(sp->role.level == PG_LEVEL_4K);
c3134ce2 1209
0cd8dc73 1210 drop_spte(kvm, sptep);
03787394
PB
1211
1212 if (flush)
1b2dc736 1213 kvm_flush_remote_tlbs_sptep(kvm, sptep);
8e22f955
XG
1214}
1215
1216/*
49fde340 1217 * Write-protect on the specified @sptep, @pt_protect indicates whether
c126d94f 1218 * spte write-protection is caused by protecting shadow page table.
49fde340 1219 *
b4619660 1220 * Note: write protection is difference between dirty logging and spte
49fde340
XG
1221 * protection:
1222 * - for dirty logging, the spte can be set to writable at anytime if
1223 * its dirty bitmap is properly set.
1224 * - for spte protection, the spte can be writable only after unsync-ing
1225 * shadow page.
8e22f955 1226 *
c126d94f 1227 * Return true if tlb need be flushed.
8e22f955 1228 */
c4f138b4 1229static bool spte_write_protect(u64 *sptep, bool pt_protect)
d13bc5b5
XG
1230{
1231 u64 spte = *sptep;
1232
49fde340 1233 if (!is_writable_pte(spte) &&
706c9c55 1234 !(pt_protect && is_mmu_writable_spte(spte)))
d13bc5b5
XG
1235 return false;
1236
49fde340 1237 if (pt_protect)
5fc3424f 1238 spte &= ~shadow_mmu_writable_mask;
d13bc5b5 1239 spte = spte & ~PT_WRITABLE_MASK;
49fde340 1240
c126d94f 1241 return mmu_spte_update(sptep, spte);
d13bc5b5
XG
1242}
1243
1346bbb6
DM
1244static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
1245 bool pt_protect)
98348e95 1246{
1e3f42f0
TY
1247 u64 *sptep;
1248 struct rmap_iterator iter;
d13bc5b5 1249 bool flush = false;
374cbac0 1250
018aabb5 1251 for_each_rmap_spte(rmap_head, &iter, sptep)
c4f138b4 1252 flush |= spte_write_protect(sptep, pt_protect);
855149aa 1253
d13bc5b5 1254 return flush;
a0ed4607
TY
1255}
1256
c4f138b4 1257static bool spte_clear_dirty(u64 *sptep)
f4b4b180
KH
1258{
1259 u64 spte = *sptep;
1260
0fe6370e 1261 KVM_MMU_WARN_ON(!spte_ad_enabled(spte));
f4b4b180 1262 spte &= ~shadow_dirty_mask;
f4b4b180
KH
1263 return mmu_spte_update(sptep, spte);
1264}
1265
1f4e5fc8 1266static bool spte_wrprot_for_clear_dirty(u64 *sptep)
ac8d57e5
PF
1267{
1268 bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
1269 (unsigned long *)sptep);
1f4e5fc8 1270 if (was_writable && !spte_ad_enabled(*sptep))
ac8d57e5
PF
1271 kvm_set_pfn_dirty(spte_to_pfn(*sptep));
1272
1273 return was_writable;
1274}
1275
1276/*
1277 * Gets the GFN ready for another round of dirty logging by clearing the
1278 * - D bit on ad-enabled SPTEs, and
1279 * - W bit on ad-disabled SPTEs.
1280 * Returns true iff any D or W bits were cleared.
1281 */
0a234f5d 1282static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
269e9552 1283 const struct kvm_memory_slot *slot)
f4b4b180
KH
1284{
1285 u64 *sptep;
1286 struct rmap_iterator iter;
1287 bool flush = false;
1288
018aabb5 1289 for_each_rmap_spte(rmap_head, &iter, sptep)
1f4e5fc8
PB
1290 if (spte_ad_need_write_protect(*sptep))
1291 flush |= spte_wrprot_for_clear_dirty(sptep);
ac8d57e5 1292 else
1f4e5fc8 1293 flush |= spte_clear_dirty(sptep);
f4b4b180
KH
1294
1295 return flush;
1296}
1297
5dc99b23 1298/**
3b0f1d01 1299 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
5dc99b23
TY
1300 * @kvm: kvm instance
1301 * @slot: slot to protect
1302 * @gfn_offset: start of the BITS_PER_LONG pages we care about
1303 * @mask: indicates which pages we should protect
1304 *
89212919 1305 * Used when we do not need to care about huge page mappings.
5dc99b23 1306 */
3b0f1d01 1307static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
5dc99b23
TY
1308 struct kvm_memory_slot *slot,
1309 gfn_t gfn_offset, unsigned long mask)
a0ed4607 1310{
018aabb5 1311 struct kvm_rmap_head *rmap_head;
a0ed4607 1312
1f98f2bd 1313 if (tdp_mmu_enabled)
a6a0b05d
BG
1314 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1315 slot->base_gfn + gfn_offset, mask, true);
e2209710
BG
1316
1317 if (!kvm_memslots_have_rmaps(kvm))
1318 return;
1319
5dc99b23 1320 while (mask) {
93e083d4
DM
1321 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1322 PG_LEVEL_4K, slot);
1346bbb6 1323 rmap_write_protect(rmap_head, false);
05da4558 1324
5dc99b23
TY
1325 /* clear the first set bit */
1326 mask &= mask - 1;
1327 }
374cbac0
AK
1328}
1329
f4b4b180 1330/**
ac8d57e5
PF
1331 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
1332 * protect the page if the D-bit isn't supported.
f4b4b180
KH
1333 * @kvm: kvm instance
1334 * @slot: slot to clear D-bit
1335 * @gfn_offset: start of the BITS_PER_LONG pages we care about
1336 * @mask: indicates which pages we should clear D-bit
1337 *
1338 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
1339 */
a018eba5
SC
1340static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
1341 struct kvm_memory_slot *slot,
1342 gfn_t gfn_offset, unsigned long mask)
f4b4b180 1343{
018aabb5 1344 struct kvm_rmap_head *rmap_head;
f4b4b180 1345
1f98f2bd 1346 if (tdp_mmu_enabled)
a6a0b05d
BG
1347 kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
1348 slot->base_gfn + gfn_offset, mask, false);
e2209710
BG
1349
1350 if (!kvm_memslots_have_rmaps(kvm))
1351 return;
1352
f4b4b180 1353 while (mask) {
93e083d4
DM
1354 rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1355 PG_LEVEL_4K, slot);
0a234f5d 1356 __rmap_clear_dirty(kvm, rmap_head, slot);
f4b4b180
KH
1357
1358 /* clear the first set bit */
1359 mask &= mask - 1;
1360 }
1361}
f4b4b180 1362
3b0f1d01
KH
1363/**
1364 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1365 * PT level pages.
1366 *
1367 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1368 * enable dirty logging for them.
1369 *
89212919
KZ
1370 * We need to care about huge page mappings: e.g. during dirty logging we may
1371 * have such mappings.
3b0f1d01
KH
1372 */
1373void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1374 struct kvm_memory_slot *slot,
1375 gfn_t gfn_offset, unsigned long mask)
1376{
89212919
KZ
1377 /*
1378 * Huge pages are NOT write protected when we start dirty logging in
1379 * initially-all-set mode; must write protect them here so that they
1380 * are split to 4K on the first write.
1381 *
1382 * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn
1383 * of memslot has no such restriction, so the range can cross two large
1384 * pages.
1385 */
1386 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
1387 gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);
1388 gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);
1389
cb00a70b 1390 if (READ_ONCE(eager_page_split))
1aa4bb91 1391 kvm_mmu_try_split_huge_pages(kvm, slot, start, end + 1, PG_LEVEL_4K);
cb00a70b 1392
89212919
KZ
1393 kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);
1394
1395 /* Cross two large pages? */
1396 if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) !=
1397 ALIGN(end << PAGE_SHIFT, PMD_SIZE))
1398 kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
1399 PG_LEVEL_2M);
1400 }
1401
1402 /* Now handle 4K PTEs. */
a018eba5
SC
1403 if (kvm_x86_ops.cpu_dirty_log_size)
1404 kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
88178fd4
KH
1405 else
1406 kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
3b0f1d01
KH
1407}
1408
fb04a1ed
PX
1409int kvm_cpu_dirty_log_size(void)
1410{
6dd03800 1411 return kvm_x86_ops.cpu_dirty_log_size;
fb04a1ed
PX
1412}
1413
aeecee2e 1414bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
3ad93562
KZ
1415 struct kvm_memory_slot *slot, u64 gfn,
1416 int min_level)
95d4c16c 1417{
018aabb5 1418 struct kvm_rmap_head *rmap_head;
5dc99b23 1419 int i;
2f84569f 1420 bool write_protected = false;
95d4c16c 1421
e2209710
BG
1422 if (kvm_memslots_have_rmaps(kvm)) {
1423 for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
93e083d4 1424 rmap_head = gfn_to_rmap(gfn, i, slot);
1346bbb6 1425 write_protected |= rmap_write_protect(rmap_head, true);
e2209710 1426 }
5dc99b23
TY
1427 }
1428
1f98f2bd 1429 if (tdp_mmu_enabled)
46044f72 1430 write_protected |=
3ad93562 1431 kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
46044f72 1432
5dc99b23 1433 return write_protected;
95d4c16c
TY
1434}
1435
cf48f9e2 1436static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
aeecee2e
XG
1437{
1438 struct kvm_memory_slot *slot;
1439
1440 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
3ad93562 1441 return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
aeecee2e
XG
1442}
1443
f8480721
SC
1444static bool __kvm_zap_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1445 const struct kvm_memory_slot *slot)
e930bffe 1446{
9202aee8 1447 return kvm_zap_all_rmap_sptes(kvm, rmap_head);
6a49f85c
XG
1448}
1449
f8480721
SC
1450static bool kvm_zap_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1451 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1452 pte_t unused)
6a49f85c 1453{
f8480721 1454 return __kvm_zap_rmap(kvm, rmap_head, slot);
e930bffe
AA
1455}
1456
aed02fe3
SC
1457static bool kvm_set_pte_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1458 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1459 pte_t pte)
3da0dd43 1460{
1e3f42f0
TY
1461 u64 *sptep;
1462 struct rmap_iterator iter;
98a26b69 1463 bool need_flush = false;
1e3f42f0 1464 u64 new_spte;
ba049e93 1465 kvm_pfn_t new_pfn;
3da0dd43 1466
20ba462d 1467 WARN_ON_ONCE(pte_huge(pte));
3039bcc7 1468 new_pfn = pte_pfn(pte);
1e3f42f0 1469
0d536790 1470restart:
018aabb5 1471 for_each_rmap_spte(rmap_head, &iter, sptep) {
98a26b69 1472 need_flush = true;
1e3f42f0 1473
3039bcc7 1474 if (pte_write(pte)) {
9202aee8 1475 kvm_zap_one_rmap_spte(kvm, rmap_head, sptep);
0d536790 1476 goto restart;
3da0dd43 1477 } else {
cb3eedab
PB
1478 new_spte = kvm_mmu_changed_pte_notifier_make_spte(
1479 *sptep, new_pfn);
1e3f42f0 1480
71f51d2c 1481 mmu_spte_clear_track_bits(kvm, sptep);
1e3f42f0 1482 mmu_spte_set(sptep, new_spte);
3da0dd43
IE
1483 }
1484 }
1e3f42f0 1485
8a1300ff 1486 if (need_flush && kvm_available_flush_remote_tlbs_range()) {
9ffe9265 1487 kvm_flush_remote_tlbs_gfn(kvm, gfn, level);
98a26b69 1488 return false;
3cc5ea94
LT
1489 }
1490
0cf853c5 1491 return need_flush;
3da0dd43
IE
1492}
1493
6ce1f4e2
XG
1494struct slot_rmap_walk_iterator {
1495 /* input fields. */
269e9552 1496 const struct kvm_memory_slot *slot;
6ce1f4e2
XG
1497 gfn_t start_gfn;
1498 gfn_t end_gfn;
1499 int start_level;
1500 int end_level;
1501
1502 /* output fields. */
1503 gfn_t gfn;
018aabb5 1504 struct kvm_rmap_head *rmap;
6ce1f4e2
XG
1505 int level;
1506
1507 /* private field. */
018aabb5 1508 struct kvm_rmap_head *end_rmap;
6ce1f4e2
XG
1509};
1510
f3d90f90
SC
1511static void rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator,
1512 int level)
6ce1f4e2
XG
1513{
1514 iterator->level = level;
1515 iterator->gfn = iterator->start_gfn;
93e083d4
DM
1516 iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
1517 iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
6ce1f4e2
XG
1518}
1519
f3d90f90
SC
1520static void slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1521 const struct kvm_memory_slot *slot,
1522 int start_level, int end_level,
1523 gfn_t start_gfn, gfn_t end_gfn)
6ce1f4e2
XG
1524{
1525 iterator->slot = slot;
1526 iterator->start_level = start_level;
1527 iterator->end_level = end_level;
1528 iterator->start_gfn = start_gfn;
1529 iterator->end_gfn = end_gfn;
1530
1531 rmap_walk_init_level(iterator, iterator->start_level);
1532}
1533
1534static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
1535{
1536 return !!iterator->rmap;
1537}
1538
1539static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
1540{
6ba1e04f 1541 while (++iterator->rmap <= iterator->end_rmap) {
6ce1f4e2 1542 iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
6ba1e04f
VS
1543
1544 if (iterator->rmap->val)
1545 return;
6ce1f4e2
XG
1546 }
1547
1548 if (++iterator->level > iterator->end_level) {
1549 iterator->rmap = NULL;
1550 return;
1551 }
1552
1553 rmap_walk_init_level(iterator, iterator->level);
1554}
1555
1556#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_, \
1557 _start_gfn, _end_gfn, _iter_) \
1558 for (slot_rmap_walk_init(_iter_, _slot_, _start_level_, \
1559 _end_level_, _start_gfn, _end_gfn); \
1560 slot_rmap_walk_okay(_iter_); \
1561 slot_rmap_walk_next(_iter_))
1562
3039bcc7
SC
1563typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1564 struct kvm_memory_slot *slot, gfn_t gfn,
1565 int level, pte_t pte);
c1b91493 1566
3039bcc7
SC
1567static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
1568 struct kvm_gfn_range *range,
1569 rmap_handler_t handler)
e930bffe 1570{
6ce1f4e2 1571 struct slot_rmap_walk_iterator iterator;
3039bcc7 1572 bool ret = false;
e930bffe 1573
3039bcc7
SC
1574 for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
1575 range->start, range->end - 1, &iterator)
1576 ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
3e1efe2b 1577 iterator.level, range->arg.pte);
e930bffe 1578
f395302e 1579 return ret;
e930bffe
AA
1580}
1581
3039bcc7 1582bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
84504ef3 1583{
e2209710 1584 bool flush = false;
063afacd 1585
e2209710 1586 if (kvm_memslots_have_rmaps(kvm))
f8480721 1587 flush = kvm_handle_gfn_range(kvm, range, kvm_zap_rmap);
063afacd 1588
1f98f2bd 1589 if (tdp_mmu_enabled)
c7785d85 1590 flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
063afacd 1591
0a3869e1
SC
1592 if (kvm_x86_ops.set_apic_access_page_addr &&
1593 range->slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT)
0a8a5f2c
SC
1594 kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
1595
3039bcc7 1596 return flush;
b3ae2096
TY
1597}
1598
3039bcc7 1599bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
3da0dd43 1600{
e2209710 1601 bool flush = false;
1d8dd6b3 1602
e2209710 1603 if (kvm_memslots_have_rmaps(kvm))
aed02fe3 1604 flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmap);
1d8dd6b3 1605
1f98f2bd 1606 if (tdp_mmu_enabled)
3039bcc7 1607 flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range);
1d8dd6b3 1608
3039bcc7 1609 return flush;
e930bffe
AA
1610}
1611
aed02fe3
SC
1612static bool kvm_age_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1613 struct kvm_memory_slot *slot, gfn_t gfn, int level,
1614 pte_t unused)
e930bffe 1615{
1e3f42f0 1616 u64 *sptep;
3f649ab7 1617 struct rmap_iterator iter;
e930bffe
AA
1618 int young = 0;
1619
f160c7b7
JS
1620 for_each_rmap_spte(rmap_head, &iter, sptep)
1621 young |= mmu_spte_age(sptep);
0d536790 1622
e930bffe
AA
1623 return young;
1624}
1625
aed02fe3
SC
1626static bool kvm_test_age_rmap(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1627 struct kvm_memory_slot *slot, gfn_t gfn,
1628 int level, pte_t unused)
8ee53820 1629{
1e3f42f0
TY
1630 u64 *sptep;
1631 struct rmap_iterator iter;
8ee53820 1632
83ef6c81
JS
1633 for_each_rmap_spte(rmap_head, &iter, sptep)
1634 if (is_accessed_spte(*sptep))
98a26b69
VM
1635 return true;
1636 return false;
8ee53820
AA
1637}
1638
53a27b39
MT
1639#define RMAP_RECYCLE_THRESHOLD 1000
1640
2ff9039a
DM
1641static void __rmap_add(struct kvm *kvm,
1642 struct kvm_mmu_memory_cache *cache,
1643 const struct kvm_memory_slot *slot,
72ae5822 1644 u64 *spte, gfn_t gfn, unsigned int access)
53a27b39 1645{
852e3c19 1646 struct kvm_mmu_page *sp;
68be1306
DM
1647 struct kvm_rmap_head *rmap_head;
1648 int rmap_count;
852e3c19 1649
57354682 1650 sp = sptep_to_sp(spte);
79e48cec 1651 kvm_mmu_page_set_translation(sp, spte_index(spte), gfn, access);
81cb4657
DM
1652 kvm_update_page_stats(kvm, sp->role.level, 1);
1653
93e083d4 1654 rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
2ff9039a 1655 rmap_count = pte_list_add(cache, spte, rmap_head);
53a27b39 1656
604f5332
ML
1657 if (rmap_count > kvm->stat.max_mmu_rmap_size)
1658 kvm->stat.max_mmu_rmap_size = rmap_count;
68be1306 1659 if (rmap_count > RMAP_RECYCLE_THRESHOLD) {
9202aee8 1660 kvm_zap_all_rmap_sptes(kvm, rmap_head);
1b2dc736 1661 kvm_flush_remote_tlbs_gfn(kvm, gfn, sp->role.level);
68be1306 1662 }
53a27b39
MT
1663}
1664
2ff9039a 1665static void rmap_add(struct kvm_vcpu *vcpu, const struct kvm_memory_slot *slot,
72ae5822 1666 u64 *spte, gfn_t gfn, unsigned int access)
2ff9039a
DM
1667{
1668 struct kvm_mmu_memory_cache *cache = &vcpu->arch.mmu_pte_list_desc_cache;
1669
6a97575d 1670 __rmap_add(vcpu->kvm, cache, slot, spte, gfn, access);
2ff9039a
DM
1671}
1672
3039bcc7 1673bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
e930bffe 1674{
e2209710 1675 bool young = false;
3039bcc7 1676
e2209710 1677 if (kvm_memslots_have_rmaps(kvm))
aed02fe3 1678 young = kvm_handle_gfn_range(kvm, range, kvm_age_rmap);
f8e14497 1679
1f98f2bd 1680 if (tdp_mmu_enabled)
3039bcc7 1681 young |= kvm_tdp_mmu_age_gfn_range(kvm, range);
f8e14497
BG
1682
1683 return young;
e930bffe
AA
1684}
1685
3039bcc7 1686bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
8ee53820 1687{
e2209710 1688 bool young = false;
3039bcc7 1689
e2209710 1690 if (kvm_memslots_have_rmaps(kvm))
aed02fe3 1691 young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmap);
f8e14497 1692
1f98f2bd 1693 if (tdp_mmu_enabled)
3039bcc7 1694 young |= kvm_tdp_mmu_test_age_gfn(kvm, range);
f8e14497
BG
1695
1696 return young;
8ee53820
AA
1697}
1698
58da926c 1699static void kvm_mmu_check_sptes_at_free(struct kvm_mmu_page *sp)
6aa8b732 1700{
870d4d4e 1701#ifdef CONFIG_KVM_PROVE_MMU
242a6dd8 1702 int i;
139bdb2d 1703
242a6dd8 1704 for (i = 0; i < SPTE_ENT_PER_PAGE; i++) {
0fe6370e 1705 if (KVM_MMU_WARN_ON(is_shadow_present_pte(sp->spt[i])))
58da926c
SC
1706 pr_err_ratelimited("SPTE %llx (@ %p) for gfn %llx shadow-present at free",
1707 sp->spt[i], &sp->spt[i],
1708 kvm_mmu_page_get_gfn(sp, i));
242a6dd8 1709 }
d6c69ee9 1710#endif
58da926c 1711}
6aa8b732 1712
45221ab6
DH
1713/*
1714 * This value is the sum of all of the kvm instances's
1715 * kvm->arch.n_used_mmu_pages values. We need a global,
1716 * aggregate version in order to make the slab shrinker
1717 * faster
1718 */
d5aaad6f 1719static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr)
45221ab6
DH
1720{
1721 kvm->arch.n_used_mmu_pages += nr;
1722 percpu_counter_add(&kvm_total_used_mmu_pages, nr);
1723}
1724
43a063ca
YA
1725static void kvm_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1726{
1727 kvm_mod_used_mmu_pages(kvm, +1);
1728 kvm_account_pgtable_pages((void *)sp->spt, +1);
1729}
1730
1731static void kvm_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1732{
1733 kvm_mod_used_mmu_pages(kvm, -1);
1734 kvm_account_pgtable_pages((void *)sp->spt, -1);
1735}
1736
87654643 1737static void kvm_mmu_free_shadow_page(struct kvm_mmu_page *sp)
260746c0 1738{
58da926c
SC
1739 kvm_mmu_check_sptes_at_free(sp);
1740
7775834a 1741 hlist_del(&sp->hash_link);
bd4c86ea
XG
1742 list_del(&sp->link);
1743 free_page((unsigned long)sp->spt);
834be0d8 1744 if (!sp->role.direct)
6a97575d 1745 free_page((unsigned long)sp->shadowed_translation);
e8ad9a70 1746 kmem_cache_free(mmu_page_header_cache, sp);
260746c0
AK
1747}
1748
cea0f0e7
AK
1749static unsigned kvm_page_table_hashfn(gfn_t gfn)
1750{
114df303 1751 return hash_64(gfn, KVM_MMU_HASH_SHIFT);
cea0f0e7
AK
1752}
1753
2ff9039a 1754static void mmu_page_add_parent_pte(struct kvm_mmu_memory_cache *cache,
4db35314 1755 struct kvm_mmu_page *sp, u64 *parent_pte)
cea0f0e7 1756{
cea0f0e7
AK
1757 if (!parent_pte)
1758 return;
cea0f0e7 1759
2ff9039a 1760 pte_list_add(cache, parent_pte, &sp->parent_ptes);
cea0f0e7
AK
1761}
1762
069f30c6 1763static void mmu_page_remove_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
cea0f0e7
AK
1764 u64 *parent_pte)
1765{
069f30c6 1766 pte_list_remove(kvm, parent_pte, &sp->parent_ptes);
cea0f0e7
AK
1767}
1768
069f30c6 1769static void drop_parent_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
bcdd9a93
XG
1770 u64 *parent_pte)
1771{
069f30c6 1772 mmu_page_remove_parent_pte(kvm, sp, parent_pte);
1df9f2dc 1773 mmu_spte_clear_no_track(parent_pte);
bcdd9a93
XG
1774}
1775
67052b35 1776static void mark_unsync(u64 *spte);
1047df1f 1777static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
0074ff63 1778{
74c4e63a
TY
1779 u64 *sptep;
1780 struct rmap_iterator iter;
1781
1782 for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
1783 mark_unsync(sptep);
1784 }
0074ff63
MT
1785}
1786
67052b35 1787static void mark_unsync(u64 *spte)
0074ff63 1788{
67052b35 1789 struct kvm_mmu_page *sp;
0074ff63 1790
57354682 1791 sp = sptep_to_sp(spte);
79e48cec 1792 if (__test_and_set_bit(spte_index(spte), sp->unsync_child_bitmap))
0074ff63 1793 return;
1047df1f 1794 if (sp->unsync_children++)
0074ff63 1795 return;
1047df1f 1796 kvm_mmu_mark_parents_unsync(sp);
0074ff63
MT
1797}
1798
60c8aec6
MT
1799#define KVM_PAGE_ARRAY_NR 16
1800
1801struct kvm_mmu_pages {
1802 struct mmu_page_and_offset {
1803 struct kvm_mmu_page *sp;
1804 unsigned int idx;
1805 } page[KVM_PAGE_ARRAY_NR];
1806 unsigned int nr;
1807};
1808
cded19f3
HE
1809static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
1810 int idx)
4731d4c7 1811{
60c8aec6 1812 int i;
4731d4c7 1813
60c8aec6
MT
1814 if (sp->unsync)
1815 for (i=0; i < pvec->nr; i++)
1816 if (pvec->page[i].sp == sp)
1817 return 0;
1818
1819 pvec->page[pvec->nr].sp = sp;
1820 pvec->page[pvec->nr].idx = idx;
1821 pvec->nr++;
1822 return (pvec->nr == KVM_PAGE_ARRAY_NR);
1823}
1824
fd951457
TY
1825static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
1826{
1827 --sp->unsync_children;
20ba462d 1828 WARN_ON_ONCE((int)sp->unsync_children < 0);
fd951457
TY
1829 __clear_bit(idx, sp->unsync_child_bitmap);
1830}
1831
60c8aec6
MT
1832static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1833 struct kvm_mmu_pages *pvec)
1834{
1835 int i, ret, nr_unsync_leaf = 0;
4731d4c7 1836
37178b8b 1837 for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
7a8f1a74 1838 struct kvm_mmu_page *child;
4731d4c7
MT
1839 u64 ent = sp->spt[i];
1840
fd951457
TY
1841 if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
1842 clear_unsync_child_bit(sp, i);
1843 continue;
1844 }
7a8f1a74 1845
5e3edd7e 1846 child = spte_to_child_sp(ent);
7a8f1a74
XG
1847
1848 if (child->unsync_children) {
1849 if (mmu_pages_add(pvec, child, i))
1850 return -ENOSPC;
1851
1852 ret = __mmu_unsync_walk(child, pvec);
fd951457
TY
1853 if (!ret) {
1854 clear_unsync_child_bit(sp, i);
1855 continue;
1856 } else if (ret > 0) {
7a8f1a74 1857 nr_unsync_leaf += ret;
fd951457 1858 } else
7a8f1a74
XG
1859 return ret;
1860 } else if (child->unsync) {
1861 nr_unsync_leaf++;
1862 if (mmu_pages_add(pvec, child, i))
1863 return -ENOSPC;
1864 } else
fd951457 1865 clear_unsync_child_bit(sp, i);
4731d4c7
MT
1866 }
1867
60c8aec6
MT
1868 return nr_unsync_leaf;
1869}
1870
e23d3fef
XG
1871#define INVALID_INDEX (-1)
1872
60c8aec6
MT
1873static int mmu_unsync_walk(struct kvm_mmu_page *sp,
1874 struct kvm_mmu_pages *pvec)
1875{
0a47cd85 1876 pvec->nr = 0;
60c8aec6
MT
1877 if (!sp->unsync_children)
1878 return 0;
1879
e23d3fef 1880 mmu_pages_add(pvec, sp, INVALID_INDEX);
60c8aec6 1881 return __mmu_unsync_walk(sp, pvec);
4731d4c7
MT
1882}
1883
4731d4c7
MT
1884static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1885{
20ba462d 1886 WARN_ON_ONCE(!sp->unsync);
5e1b3ddb 1887 trace_kvm_mmu_sync_page(sp);
4731d4c7
MT
1888 sp->unsync = 0;
1889 --kvm->stat.mmu_unsync;
1890}
1891
83cdb568
SC
1892static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
1893 struct list_head *invalid_list);
7775834a
XG
1894static void kvm_mmu_commit_zap_page(struct kvm *kvm,
1895 struct list_head *invalid_list);
4731d4c7 1896
767d8d8d
LJ
1897static bool sp_has_gptes(struct kvm_mmu_page *sp)
1898{
1899 if (sp->role.direct)
1900 return false;
1901
84e5ffd0
LJ
1902 if (sp->role.passthrough)
1903 return false;
1904
767d8d8d
LJ
1905 return true;
1906}
1907
ac101b7c
SC
1908#define for_each_valid_sp(_kvm, _sp, _list) \
1909 hlist_for_each_entry(_sp, _list, hash_link) \
fac026da 1910 if (is_obsolete_sp((_kvm), (_sp))) { \
f3414bc7 1911 } else
1044b030 1912
767d8d8d 1913#define for_each_gfn_valid_sp_with_gptes(_kvm, _sp, _gfn) \
ac101b7c
SC
1914 for_each_valid_sp(_kvm, _sp, \
1915 &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \
767d8d8d 1916 if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else
7ae680eb 1917
90e44470
LJ
1918static bool kvm_sync_page_check(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1919{
1920 union kvm_mmu_page_role root_role = vcpu->arch.mmu->root_role;
1921
1922 /*
1923 * Ignore various flags when verifying that it's safe to sync a shadow
1924 * page using the current MMU context.
1925 *
1926 * - level: not part of the overall MMU role and will never match as the MMU's
1927 * level tracks the root level
1928 * - access: updated based on the new guest PTE
1929 * - quadrant: not part of the overall MMU role (similar to level)
1930 */
1931 const union kvm_mmu_page_role sync_role_ign = {
1932 .level = 0xf,
1933 .access = 0x7,
1934 .quadrant = 0x3,
1935 .passthrough = 0x1,
1936 };
1937
1938 /*
1939 * Direct pages can never be unsync, and KVM should never attempt to
1940 * sync a shadow page for a different MMU context, e.g. if the role
1941 * differs then the memslot lookup (SMM vs. non-SMM) will be bogus, the
1942 * reserved bits checks will be wrong, etc...
1943 */
c3c6c9fc 1944 if (WARN_ON_ONCE(sp->role.direct || !vcpu->arch.mmu->sync_spte ||
90e44470
LJ
1945 (sp->role.word ^ root_role.word) & ~sync_role_ign.word))
1946 return false;
1947
1948 return true;
1949}
1950
19ace7d6
LJ
1951static int kvm_sync_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int i)
1952{
1953 if (!sp->spt[i])
1954 return 0;
1955
1956 return vcpu->arch.mmu->sync_spte(vcpu, sp, i);
1957}
1958
90e44470
LJ
1959static int __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1960{
c3c6c9fc
LJ
1961 int flush = 0;
1962 int i;
1963
90e44470
LJ
1964 if (!kvm_sync_page_check(vcpu, sp))
1965 return -1;
1966
c3c6c9fc 1967 for (i = 0; i < SPTE_ENT_PER_PAGE; i++) {
19ace7d6 1968 int ret = kvm_sync_spte(vcpu, sp, i);
c3c6c9fc
LJ
1969
1970 if (ret < -1)
1971 return -1;
1972 flush |= ret;
1973 }
1974
1975 /*
1976 * Note, any flush is purely for KVM's correctness, e.g. when dropping
1977 * an existing SPTE or clearing W/A/D bits to ensure an mmu_notifier
1978 * unmap or dirty logging event doesn't fail to flush. The guest is
1979 * responsible for flushing the TLB to ensure any changes in protection
1980 * bits are recognized, i.e. until the guest flushes or page faults on
1981 * a relevant address, KVM is architecturally allowed to let vCPUs use
1982 * cached translations with the old protection bits.
1983 */
1984 return flush;
90e44470
LJ
1985}
1986
8d5678a7 1987static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
479a1efc 1988 struct list_head *invalid_list)
4731d4c7 1989{
90e44470 1990 int ret = __kvm_sync_page(vcpu, sp);
c3e5e415 1991
8d5678a7 1992 if (ret < 0)
d98ba053 1993 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
8d5678a7 1994 return ret;
4731d4c7
MT
1995}
1996
a2113634
SC
1997static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
1998 struct list_head *invalid_list,
1999 bool remote_flush)
2000{
cfd32acf 2001 if (!remote_flush && list_empty(invalid_list))
a2113634
SC
2002 return false;
2003
2004 if (!list_empty(invalid_list))
2005 kvm_mmu_commit_zap_page(kvm, invalid_list);
2006 else
2007 kvm_flush_remote_tlbs(kvm);
2008 return true;
2009}
2010
002c5f73
SC
2011static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
2012{
a955cad8
SC
2013 if (sp->role.invalid)
2014 return true;
2015
fa3e4203 2016 /* TDP MMU pages do not use the MMU generation. */
de0322f5 2017 return !is_tdp_mmu_page(sp) &&
fac026da 2018 unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
002c5f73
SC
2019}
2020
60c8aec6 2021struct mmu_page_path {
2a7266a8
YZ
2022 struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
2023 unsigned int idx[PT64_ROOT_MAX_LEVEL];
4731d4c7
MT
2024};
2025
60c8aec6 2026#define for_each_sp(pvec, sp, parents, i) \
0a47cd85 2027 for (i = mmu_pages_first(&pvec, &parents); \
60c8aec6
MT
2028 i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
2029 i = mmu_pages_next(&pvec, &parents, i))
2030
cded19f3
HE
2031static int mmu_pages_next(struct kvm_mmu_pages *pvec,
2032 struct mmu_page_path *parents,
2033 int i)
60c8aec6
MT
2034{
2035 int n;
2036
2037 for (n = i+1; n < pvec->nr; n++) {
2038 struct kvm_mmu_page *sp = pvec->page[n].sp;
0a47cd85
PB
2039 unsigned idx = pvec->page[n].idx;
2040 int level = sp->role.level;
60c8aec6 2041
0a47cd85 2042 parents->idx[level-1] = idx;
3bae0459 2043 if (level == PG_LEVEL_4K)
0a47cd85 2044 break;
60c8aec6 2045
0a47cd85 2046 parents->parent[level-2] = sp;
60c8aec6
MT
2047 }
2048
2049 return n;
2050}
2051
0a47cd85
PB
2052static int mmu_pages_first(struct kvm_mmu_pages *pvec,
2053 struct mmu_page_path *parents)
2054{
2055 struct kvm_mmu_page *sp;
2056 int level;
2057
2058 if (pvec->nr == 0)
2059 return 0;
2060
20ba462d 2061 WARN_ON_ONCE(pvec->page[0].idx != INVALID_INDEX);
e23d3fef 2062
0a47cd85
PB
2063 sp = pvec->page[0].sp;
2064 level = sp->role.level;
20ba462d 2065 WARN_ON_ONCE(level == PG_LEVEL_4K);
0a47cd85
PB
2066
2067 parents->parent[level-2] = sp;
2068
2069 /* Also set up a sentinel. Further entries in pvec are all
2070 * children of sp, so this element is never overwritten.
2071 */
2072 parents->parent[level-1] = NULL;
2073 return mmu_pages_next(pvec, parents, 0);
2074}
2075
cded19f3 2076static void mmu_pages_clear_parents(struct mmu_page_path *parents)
4731d4c7 2077{
60c8aec6
MT
2078 struct kvm_mmu_page *sp;
2079 unsigned int level = 0;
2080
2081 do {
2082 unsigned int idx = parents->idx[level];
60c8aec6
MT
2083 sp = parents->parent[level];
2084 if (!sp)
2085 return;
2086
20ba462d 2087 WARN_ON_ONCE(idx == INVALID_INDEX);
fd951457 2088 clear_unsync_child_bit(sp, idx);
60c8aec6 2089 level++;
0a47cd85 2090 } while (!sp->unsync_children);
60c8aec6 2091}
4731d4c7 2092
65855ed8
LJ
2093static int mmu_sync_children(struct kvm_vcpu *vcpu,
2094 struct kvm_mmu_page *parent, bool can_yield)
60c8aec6
MT
2095{
2096 int i;
2097 struct kvm_mmu_page *sp;
2098 struct mmu_page_path parents;
2099 struct kvm_mmu_pages pages;
d98ba053 2100 LIST_HEAD(invalid_list);
50c9e6f3 2101 bool flush = false;
60c8aec6 2102
60c8aec6 2103 while (mmu_unsync_walk(parent, &pages)) {
2f84569f 2104 bool protected = false;
b1a36821
MT
2105
2106 for_each_sp(pages, sp, parents, i)
cf48f9e2 2107 protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);
b1a36821 2108
50c9e6f3 2109 if (protected) {
5591c069 2110 kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
50c9e6f3
PB
2111 flush = false;
2112 }
b1a36821 2113
60c8aec6 2114 for_each_sp(pages, sp, parents, i) {
479a1efc 2115 kvm_unlink_unsync_page(vcpu->kvm, sp);
8d5678a7 2116 flush |= kvm_sync_page(vcpu, sp, &invalid_list) > 0;
60c8aec6
MT
2117 mmu_pages_clear_parents(&parents);
2118 }
531810ca 2119 if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
c3e5e415 2120 kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
65855ed8
LJ
2121 if (!can_yield) {
2122 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2123 return -EINTR;
2124 }
2125
531810ca 2126 cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
50c9e6f3
PB
2127 flush = false;
2128 }
60c8aec6 2129 }
50c9e6f3 2130
c3e5e415 2131 kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
65855ed8 2132 return 0;
4731d4c7
MT
2133}
2134
a30f47cb
XG
2135static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2136{
e5691a81 2137 atomic_set(&sp->write_flooding_count, 0);
a30f47cb
XG
2138}
2139
2140static void clear_sp_write_flooding_count(u64 *spte)
2141{
57354682 2142 __clear_sp_write_flooding_count(sptep_to_sp(spte));
a30f47cb
XG
2143}
2144
cbd858b1
DM
2145/*
2146 * The vCPU is required when finding indirect shadow pages; the shadow
2147 * page may already exist and syncing it needs the vCPU pointer in
2148 * order to read guest page tables. Direct shadow pages are never
2149 * unsync, thus @vcpu can be NULL if @role.direct is true.
2150 */
3cc736b3
DM
2151static struct kvm_mmu_page *kvm_mmu_find_shadow_page(struct kvm *kvm,
2152 struct kvm_vcpu *vcpu,
94c81364
DM
2153 gfn_t gfn,
2154 struct hlist_head *sp_list,
2155 union kvm_mmu_page_role role)
cea0f0e7 2156{
9f1a122f 2157 struct kvm_mmu_page *sp;
8d5678a7 2158 int ret;
f3414bc7 2159 int collisions = 0;
2a74003a 2160 LIST_HEAD(invalid_list);
cea0f0e7 2161
3cc736b3 2162 for_each_valid_sp(kvm, sp, sp_list) {
f3414bc7
DM
2163 if (sp->gfn != gfn) {
2164 collisions++;
2165 continue;
2166 }
2167
ddc16abb
SC
2168 if (sp->role.word != role.word) {
2169 /*
2170 * If the guest is creating an upper-level page, zap
2171 * unsync pages for the same gfn. While it's possible
2172 * the guest is using recursive page tables, in all
2173 * likelihood the guest has stopped using the unsync
2174 * page and is installing a completely unrelated page.
2175 * Unsync pages must not be left as is, because the new
2176 * upper-level page will be write-protected.
2177 */
2e65e842 2178 if (role.level > PG_LEVEL_4K && sp->unsync)
3cc736b3 2179 kvm_mmu_prepare_zap_page(kvm, sp,
ddc16abb 2180 &invalid_list);
7ae680eb 2181 continue;
ddc16abb 2182 }
4731d4c7 2183
bb924ca6
DM
2184 /* unsync and write-flooding only apply to indirect SPs. */
2185 if (sp->role.direct)
94c81364 2186 goto out;
fb58a9c3 2187
2a74003a 2188 if (sp->unsync) {
cbd858b1
DM
2189 if (KVM_BUG_ON(!vcpu, kvm))
2190 break;
2191
07dc4f35 2192 /*
479a1efc 2193 * The page is good, but is stale. kvm_sync_page does
07dc4f35
SC
2194 * get the latest guest state, but (unlike mmu_unsync_children)
2195 * it doesn't write-protect the page or mark it synchronized!
2196 * This way the validity of the mapping is ensured, but the
2197 * overhead of write protection is not incurred until the
2198 * guest invalidates the TLB mapping. This allows multiple
2199 * SPs for a single gfn to be unsync.
2200 *
2201 * If the sync fails, the page is zapped. If so, break
2202 * in order to rebuild it.
2a74003a 2203 */
8d5678a7
HW
2204 ret = kvm_sync_page(vcpu, sp, &invalid_list);
2205 if (ret < 0)
2a74003a
PB
2206 break;
2207
20ba462d 2208 WARN_ON_ONCE(!list_empty(&invalid_list));
8d5678a7 2209 if (ret > 0)
3cc736b3 2210 kvm_flush_remote_tlbs(kvm);
2a74003a 2211 }
e02aa901 2212
a30f47cb 2213 __clear_sp_write_flooding_count(sp);
fb58a9c3 2214
f3414bc7 2215 goto out;
7ae680eb 2216 }
47005792 2217
94c81364 2218 sp = NULL;
3cc736b3 2219 ++kvm->stat.mmu_cache_miss;
47005792 2220
94c81364 2221out:
3cc736b3 2222 kvm_mmu_commit_zap_page(kvm, &invalid_list);
94c81364 2223
3cc736b3
DM
2224 if (collisions > kvm->stat.max_mmu_page_hash_collisions)
2225 kvm->stat.max_mmu_page_hash_collisions = collisions;
94c81364
DM
2226 return sp;
2227}
2228
2f8b1b53
DM
2229/* Caches used when allocating a new shadow page. */
2230struct shadow_page_caches {
2231 struct kvm_mmu_memory_cache *page_header_cache;
2232 struct kvm_mmu_memory_cache *shadow_page_cache;
6a97575d 2233 struct kvm_mmu_memory_cache *shadowed_info_cache;
2f8b1b53
DM
2234};
2235
336081fb 2236static struct kvm_mmu_page *kvm_mmu_alloc_shadow_page(struct kvm *kvm,
2f8b1b53 2237 struct shadow_page_caches *caches,
94c81364
DM
2238 gfn_t gfn,
2239 struct hlist_head *sp_list,
2240 union kvm_mmu_page_role role)
2241{
c306aec8
DM
2242 struct kvm_mmu_page *sp;
2243
2f8b1b53
DM
2244 sp = kvm_mmu_memory_cache_alloc(caches->page_header_cache);
2245 sp->spt = kvm_mmu_memory_cache_alloc(caches->shadow_page_cache);
c306aec8 2246 if (!role.direct)
6a97575d 2247 sp->shadowed_translation = kvm_mmu_memory_cache_alloc(caches->shadowed_info_cache);
c306aec8
DM
2248
2249 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
2250
55c510e2 2251 INIT_LIST_HEAD(&sp->possible_nx_huge_page_link);
428e9216 2252
c306aec8
DM
2253 /*
2254 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages()
2255 * depends on valid pages being added to the head of the list. See
2256 * comments in kvm_zap_obsolete_pages().
2257 */
336081fb
DM
2258 sp->mmu_valid_gen = kvm->arch.mmu_valid_gen;
2259 list_add(&sp->link, &kvm->arch.active_mmu_pages);
43a063ca 2260 kvm_account_mmu_page(kvm, sp);
47005792 2261
4db35314
AK
2262 sp->gfn = gfn;
2263 sp->role = role;
ac101b7c 2264 hlist_add_head(&sp->hash_link, sp_list);
be911771 2265 if (sp_has_gptes(sp))
336081fb 2266 account_shadowed(kvm, sp);
ddc16abb 2267
94c81364
DM
2268 return sp;
2269}
2270
cbd858b1 2271/* Note, @vcpu may be NULL if @role.direct is true; see kvm_mmu_find_shadow_page. */
3cc736b3
DM
2272static struct kvm_mmu_page *__kvm_mmu_get_shadow_page(struct kvm *kvm,
2273 struct kvm_vcpu *vcpu,
2f8b1b53
DM
2274 struct shadow_page_caches *caches,
2275 gfn_t gfn,
2276 union kvm_mmu_page_role role)
94c81364
DM
2277{
2278 struct hlist_head *sp_list;
2279 struct kvm_mmu_page *sp;
2280 bool created = false;
2281
3cc736b3 2282 sp_list = &kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
94c81364 2283
3cc736b3 2284 sp = kvm_mmu_find_shadow_page(kvm, vcpu, gfn, sp_list, role);
94c81364
DM
2285 if (!sp) {
2286 created = true;
3cc736b3 2287 sp = kvm_mmu_alloc_shadow_page(kvm, caches, gfn, sp_list, role);
94c81364
DM
2288 }
2289
2290 trace_kvm_mmu_get_page(sp, created);
4db35314 2291 return sp;
cea0f0e7
AK
2292}
2293
2f8b1b53
DM
2294static struct kvm_mmu_page *kvm_mmu_get_shadow_page(struct kvm_vcpu *vcpu,
2295 gfn_t gfn,
2296 union kvm_mmu_page_role role)
2297{
2298 struct shadow_page_caches caches = {
2299 .page_header_cache = &vcpu->arch.mmu_page_header_cache,
2300 .shadow_page_cache = &vcpu->arch.mmu_shadow_page_cache,
6a97575d 2301 .shadowed_info_cache = &vcpu->arch.mmu_shadowed_info_cache,
2f8b1b53
DM
2302 };
2303
3cc736b3 2304 return __kvm_mmu_get_shadow_page(vcpu->kvm, vcpu, &caches, gfn, role);
2f8b1b53
DM
2305}
2306
39944ab9
SC
2307static union kvm_mmu_page_role kvm_mmu_child_role(u64 *sptep, bool direct,
2308 unsigned int access)
2e65e842
DM
2309{
2310 struct kvm_mmu_page *parent_sp = sptep_to_sp(sptep);
2311 union kvm_mmu_page_role role;
2312
2313 role = parent_sp->role;
2314 role.level--;
2315 role.access = access;
2316 role.direct = direct;
2317 role.passthrough = 0;
2318
2319 /*
2320 * If the guest has 4-byte PTEs then that means it's using 32-bit,
2321 * 2-level, non-PAE paging. KVM shadows such guests with PAE paging
2322 * (i.e. 8-byte PTEs). The difference in PTE size means that KVM must
2323 * shadow each guest page table with multiple shadow page tables, which
2324 * requires extra bookkeeping in the role.
2325 *
2326 * Specifically, to shadow the guest's page directory (which covers a
2327 * 4GiB address space), KVM uses 4 PAE page directories, each mapping
2328 * 1GiB of the address space. @role.quadrant encodes which quarter of
2329 * the address space each maps.
2330 *
2331 * To shadow the guest's page tables (which each map a 4MiB region), KVM
2332 * uses 2 PAE page tables, each mapping a 2MiB region. For these,
2333 * @role.quadrant encodes which half of the region they map.
2334 *
39944ab9
SC
2335 * Concretely, a 4-byte PDE consumes bits 31:22, while an 8-byte PDE
2336 * consumes bits 29:21. To consume bits 31:30, KVM's uses 4 shadow
2337 * PDPTEs; those 4 PAE page directories are pre-allocated and their
2338 * quadrant is assigned in mmu_alloc_root(). A 4-byte PTE consumes
2339 * bits 21:12, while an 8-byte PTE consumes bits 20:12. To consume
2340 * bit 21 in the PTE (the child here), KVM propagates that bit to the
2341 * quadrant, i.e. sets quadrant to '0' or '1'. The parent 8-byte PDE
2342 * covers bit 21 (see above), thus the quadrant is calculated from the
2343 * _least_ significant bit of the PDE index.
2e65e842
DM
2344 */
2345 if (role.has_4_byte_gpte) {
2346 WARN_ON_ONCE(role.level != PG_LEVEL_4K);
79e48cec 2347 role.quadrant = spte_index(sptep) & 1;
2e65e842
DM
2348 }
2349
2350 return role;
2351}
2352
2353static struct kvm_mmu_page *kvm_mmu_get_child_sp(struct kvm_vcpu *vcpu,
2354 u64 *sptep, gfn_t gfn,
2355 bool direct, unsigned int access)
2356{
2357 union kvm_mmu_page_role role;
2358
0cd8dc73
PB
2359 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep))
2360 return ERR_PTR(-EEXIST);
2361
2e65e842 2362 role = kvm_mmu_child_role(sptep, direct, access);
87654643 2363 return kvm_mmu_get_shadow_page(vcpu, gfn, role);
2e65e842
DM
2364}
2365
7eb77e9f
JS
2366static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
2367 struct kvm_vcpu *vcpu, hpa_t root,
2368 u64 addr)
2d11123a
AK
2369{
2370 iterator->addr = addr;
7eb77e9f 2371 iterator->shadow_addr = root;
a972e29c 2372 iterator->level = vcpu->arch.mmu->root_role.level;
81407ca5 2373
12ec33a7 2374 if (iterator->level >= PT64_ROOT_4LEVEL &&
4d25502a 2375 vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&
347a0d0d 2376 !vcpu->arch.mmu->root_role.direct)
12ec33a7 2377 iterator->level = PT32E_ROOT_LEVEL;
81407ca5 2378
2d11123a 2379 if (iterator->level == PT32E_ROOT_LEVEL) {
7eb77e9f
JS
2380 /*
2381 * prev_root is currently only used for 64-bit hosts. So only
2382 * the active root_hpa is valid here.
2383 */
b9e5603c 2384 BUG_ON(root != vcpu->arch.mmu->root.hpa);
7eb77e9f 2385
2d11123a 2386 iterator->shadow_addr
44dd3ffa 2387 = vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2ca3129e 2388 iterator->shadow_addr &= SPTE_BASE_ADDR_MASK;
2d11123a
AK
2389 --iterator->level;
2390 if (!iterator->shadow_addr)
2391 iterator->level = 0;
2392 }
2393}
2394
7eb77e9f
JS
2395static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
2396 struct kvm_vcpu *vcpu, u64 addr)
2397{
b9e5603c 2398 shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,
7eb77e9f
JS
2399 addr);
2400}
2401
2d11123a
AK
2402static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
2403{
3bae0459 2404 if (iterator->level < PG_LEVEL_4K)
2d11123a 2405 return false;
4d88954d 2406
2ca3129e 2407 iterator->index = SPTE_INDEX(iterator->addr, iterator->level);
2d11123a
AK
2408 iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
2409 return true;
2410}
2411
c2a2ac2b
XG
2412static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
2413 u64 spte)
2d11123a 2414{
3e44dce4 2415 if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
052331be
XG
2416 iterator->level = 0;
2417 return;
2418 }
2419
2ca3129e 2420 iterator->shadow_addr = spte & SPTE_BASE_ADDR_MASK;
2d11123a
AK
2421 --iterator->level;
2422}
2423
c2a2ac2b
XG
2424static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
2425{
bb606a9b 2426 __shadow_walk_next(iterator, *iterator->sptep);
c2a2ac2b
XG
2427}
2428
0cd8dc73
PB
2429static void __link_shadow_page(struct kvm *kvm,
2430 struct kvm_mmu_memory_cache *cache, u64 *sptep,
03787394 2431 struct kvm_mmu_page *sp, bool flush)
cc4674d0
BG
2432{
2433 u64 spte;
2434
2435 BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2436
0cd8dc73
PB
2437 /*
2438 * If an SPTE is present already, it must be a leaf and therefore
03787394
PB
2439 * a large one. Drop it, and flush the TLB if needed, before
2440 * installing sp.
0cd8dc73
PB
2441 */
2442 if (is_shadow_present_pte(*sptep))
03787394 2443 drop_large_spte(kvm, sptep, flush);
0cd8dc73 2444
cc4674d0
BG
2445 spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
2446
1df9f2dc 2447 mmu_spte_set(sptep, spte);
98bba238 2448
2ff9039a 2449 mmu_page_add_parent_pte(cache, sp, sptep);
98bba238 2450
c4a48868
LJ
2451 /*
2452 * The non-direct sub-pagetable must be updated before linking. For
2453 * L1 sp, the pagetable is updated via kvm_sync_page() in
2454 * kvm_mmu_find_shadow_page() without write-protecting the gfn,
2455 * so sp->unsync can be true or false. For higher level non-direct
2456 * sp, the pagetable is updated/synced via mmu_sync_children() in
2457 * FNAME(fetch)(), so sp->unsync_children can only be false.
2458 * WARN_ON_ONCE() if anything happens unexpectedly.
2459 */
2460 if (WARN_ON_ONCE(sp->unsync_children) || sp->unsync)
98bba238 2461 mark_unsync(sptep);
32ef26a3
AK
2462}
2463
2ff9039a
DM
2464static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
2465 struct kvm_mmu_page *sp)
2466{
03787394 2467 __link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true);
2ff9039a
DM
2468}
2469
a357bd22
AK
2470static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2471 unsigned direct_access)
2472{
2473 if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
2474 struct kvm_mmu_page *child;
2475
2476 /*
2477 * For the direct sp, if the guest pte's dirty bit
2478 * changed form clean to dirty, it will corrupt the
2479 * sp's access: allow writable in the read-only sp,
2480 * so we should update the spte at this point to get
2481 * a new sp with the correct access.
2482 */
5e3edd7e 2483 child = spte_to_child_sp(*sptep);
a357bd22
AK
2484 if (child->role.access == direct_access)
2485 return;
2486
069f30c6 2487 drop_parent_pte(vcpu->kvm, child, sptep);
3cdf9374 2488 kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);
a357bd22
AK
2489 }
2490}
2491
2de4085c
BG
2492/* Returns the number of zapped non-leaf child shadow pages. */
2493static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
2494 u64 *spte, struct list_head *invalid_list)
38e3b2b2
XG
2495{
2496 u64 pte;
2497 struct kvm_mmu_page *child;
2498
2499 pte = *spte;
2500 if (is_shadow_present_pte(pte)) {
505aef8f 2501 if (is_last_spte(pte, sp->role.level)) {
c3707958 2502 drop_spte(kvm, spte);
505aef8f 2503 } else {
5e3edd7e 2504 child = spte_to_child_sp(pte);
069f30c6 2505 drop_parent_pte(kvm, child, spte);
2de4085c
BG
2506
2507 /*
2508 * Recursively zap nested TDP SPs, parentless SPs are
2509 * unlikely to be used again in the near future. This
2510 * avoids retaining a large number of stale nested SPs.
2511 */
2512 if (tdp_enabled && invalid_list &&
2513 child->role.guest_mode && !child->parent_ptes.val)
2514 return kvm_mmu_prepare_zap_page(kvm, child,
2515 invalid_list);
38e3b2b2 2516 }
ace569e0 2517 } else if (is_mmio_spte(pte)) {
ce88decf 2518 mmu_spte_clear_no_track(spte);
ace569e0 2519 }
2de4085c 2520 return 0;
38e3b2b2
XG
2521}
2522
2de4085c
BG
2523static int kvm_mmu_page_unlink_children(struct kvm *kvm,
2524 struct kvm_mmu_page *sp,
2525 struct list_head *invalid_list)
a436036b 2526{
2de4085c 2527 int zapped = 0;
697fe2e2 2528 unsigned i;
697fe2e2 2529
2ca3129e 2530 for (i = 0; i < SPTE_ENT_PER_PAGE; ++i)
2de4085c
BG
2531 zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);
2532
2533 return zapped;
a436036b
AK
2534}
2535
069f30c6 2536static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
a436036b 2537{
1e3f42f0
TY
2538 u64 *sptep;
2539 struct rmap_iterator iter;
a436036b 2540
018aabb5 2541 while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
069f30c6 2542 drop_parent_pte(kvm, sp, sptep);
31aa2b44
AK
2543}
2544
60c8aec6 2545static int mmu_zap_unsync_children(struct kvm *kvm,
7775834a
XG
2546 struct kvm_mmu_page *parent,
2547 struct list_head *invalid_list)
4731d4c7 2548{
60c8aec6
MT
2549 int i, zapped = 0;
2550 struct mmu_page_path parents;
2551 struct kvm_mmu_pages pages;
4731d4c7 2552
3bae0459 2553 if (parent->role.level == PG_LEVEL_4K)
4731d4c7 2554 return 0;
60c8aec6 2555
60c8aec6
MT
2556 while (mmu_unsync_walk(parent, &pages)) {
2557 struct kvm_mmu_page *sp;
2558
2559 for_each_sp(pages, sp, parents, i) {
7775834a 2560 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
60c8aec6 2561 mmu_pages_clear_parents(&parents);
77662e00 2562 zapped++;
60c8aec6 2563 }
60c8aec6
MT
2564 }
2565
2566 return zapped;
4731d4c7
MT
2567}
2568
83cdb568
SC
2569static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
2570 struct kvm_mmu_page *sp,
2571 struct list_head *invalid_list,
2572 int *nr_zapped)
31aa2b44 2573{
527d5cd7 2574 bool list_unstable, zapped_root = false;
f691fe1d 2575
47b0c2e4 2576 lockdep_assert_held_write(&kvm->mmu_lock);
7775834a 2577 trace_kvm_mmu_prepare_zap_page(sp);
31aa2b44 2578 ++kvm->stat.mmu_shadow_zapped;
83cdb568 2579 *nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2de4085c 2580 *nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
069f30c6 2581 kvm_mmu_unlink_parents(kvm, sp);
5304b8d3 2582
83cdb568
SC
2583 /* Zapping children means active_mmu_pages has become unstable. */
2584 list_unstable = *nr_zapped;
2585
767d8d8d 2586 if (!sp->role.invalid && sp_has_gptes(sp))
3ed1a478 2587 unaccount_shadowed(kvm, sp);
5304b8d3 2588
4731d4c7
MT
2589 if (sp->unsync)
2590 kvm_unlink_unsync_page(kvm, sp);
4db35314 2591 if (!sp->root_count) {
54a4f023 2592 /* Count self */
83cdb568 2593 (*nr_zapped)++;
f95eec9b
SC
2594
2595 /*
2596 * Already invalid pages (previously active roots) are not on
2597 * the active page list. See list_del() in the "else" case of
2598 * !sp->root_count.
2599 */
2600 if (sp->role.invalid)
2601 list_add(&sp->link, invalid_list);
2602 else
2603 list_move(&sp->link, invalid_list);
43a063ca 2604 kvm_unaccount_mmu_page(kvm, sp);
2e53d63a 2605 } else {
f95eec9b
SC
2606 /*
2607 * Remove the active root from the active page list, the root
2608 * will be explicitly freed when the root_count hits zero.
2609 */
2610 list_del(&sp->link);
05988d72 2611
10605204
SC
2612 /*
2613 * Obsolete pages cannot be used on any vCPUs, see the comment
2614 * in kvm_mmu_zap_all_fast(). Note, is_obsolete_sp() also
2615 * treats invalid shadow pages as being obsolete.
2616 */
527d5cd7 2617 zapped_root = !is_obsolete_sp(kvm, sp);
2e53d63a 2618 }
7775834a 2619
55c510e2
SC
2620 if (sp->nx_huge_page_disallowed)
2621 unaccount_nx_huge_page(kvm, sp);
b8e8c830 2622
7775834a 2623 sp->role.invalid = 1;
527d5cd7
SC
2624
2625 /*
2626 * Make the request to free obsolete roots after marking the root
2627 * invalid, otherwise other vCPUs may not see it as invalid.
2628 */
2629 if (zapped_root)
2630 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
83cdb568
SC
2631 return list_unstable;
2632}
2633
2634static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
2635 struct list_head *invalid_list)
2636{
2637 int nr_zapped;
2638
2639 __kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
2640 return nr_zapped;
a436036b
AK
2641}
2642
7775834a
XG
2643static void kvm_mmu_commit_zap_page(struct kvm *kvm,
2644 struct list_head *invalid_list)
2645{
945315b9 2646 struct kvm_mmu_page *sp, *nsp;
7775834a
XG
2647
2648 if (list_empty(invalid_list))
2649 return;
2650
c142786c 2651 /*
9753f529
LT
2652 * We need to make sure everyone sees our modifications to
2653 * the page tables and see changes to vcpu->mode here. The barrier
2654 * in the kvm_flush_remote_tlbs() achieves this. This pairs
2655 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
2656 *
2657 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
2658 * guest mode and/or lockless shadow page table walks.
c142786c
AK
2659 */
2660 kvm_flush_remote_tlbs(kvm);
c2a2ac2b 2661
945315b9 2662 list_for_each_entry_safe(sp, nsp, invalid_list, link) {
20ba462d 2663 WARN_ON_ONCE(!sp->role.invalid || sp->root_count);
87654643 2664 kvm_mmu_free_shadow_page(sp);
945315b9 2665 }
7775834a
XG
2666}
2667
6b82ef2c
SC
2668static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
2669 unsigned long nr_to_zap)
5da59607 2670{
6b82ef2c
SC
2671 unsigned long total_zapped = 0;
2672 struct kvm_mmu_page *sp, *tmp;
ba7888dd 2673 LIST_HEAD(invalid_list);
6b82ef2c
SC
2674 bool unstable;
2675 int nr_zapped;
5da59607
TY
2676
2677 if (list_empty(&kvm->arch.active_mmu_pages))
ba7888dd
SC
2678 return 0;
2679
6b82ef2c 2680restart:
8fc51726 2681 list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
6b82ef2c
SC
2682 /*
2683 * Don't zap active root pages, the page itself can't be freed
2684 * and zapping it will just force vCPUs to realloc and reload.
2685 */
2686 if (sp->root_count)
2687 continue;
2688
2689 unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
2690 &nr_zapped);
2691 total_zapped += nr_zapped;
2692 if (total_zapped >= nr_to_zap)
ba7888dd
SC
2693 break;
2694
6b82ef2c
SC
2695 if (unstable)
2696 goto restart;
ba7888dd 2697 }
5da59607 2698
6b82ef2c
SC
2699 kvm_mmu_commit_zap_page(kvm, &invalid_list);
2700
2701 kvm->stat.mmu_recycled += total_zapped;
2702 return total_zapped;
2703}
2704
afe8d7e6
SC
2705static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
2706{
2707 if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
2708 return kvm->arch.n_max_mmu_pages -
2709 kvm->arch.n_used_mmu_pages;
2710
2711 return 0;
5da59607
TY
2712}
2713
ba7888dd
SC
2714static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
2715{
6b82ef2c 2716 unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
ba7888dd 2717
6b82ef2c 2718 if (likely(avail >= KVM_MIN_FREE_MMU_PAGES))
ba7888dd
SC
2719 return 0;
2720
6b82ef2c 2721 kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
ba7888dd 2722
6e6ec584
SC
2723 /*
2724 * Note, this check is intentionally soft, it only guarantees that one
2725 * page is available, while the caller may end up allocating as many as
2726 * four pages, e.g. for PAE roots or for 5-level paging. Temporarily
2727 * exceeding the (arbitrary by default) limit will not harm the host,
c4342633 2728 * being too aggressive may unnecessarily kill the guest, and getting an
6e6ec584
SC
2729 * exact count is far more trouble than it's worth, especially in the
2730 * page fault paths.
2731 */
ba7888dd
SC
2732 if (!kvm_mmu_available_pages(vcpu->kvm))
2733 return -ENOSPC;
2734 return 0;
2735}
2736
82ce2c96
IE
2737/*
2738 * Changing the number of mmu pages allocated to the vm
49d5ca26 2739 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
82ce2c96 2740 */
bc8a3d89 2741void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
82ce2c96 2742{
531810ca 2743 write_lock(&kvm->mmu_lock);
b34cb590 2744
49d5ca26 2745 if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
6b82ef2c
SC
2746 kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
2747 goal_nr_mmu_pages);
82ce2c96 2748
49d5ca26 2749 goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
82ce2c96 2750 }
82ce2c96 2751
49d5ca26 2752 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
b34cb590 2753
531810ca 2754 write_unlock(&kvm->mmu_lock);
82ce2c96
IE
2755}
2756
1cb3f3ae 2757int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
a436036b 2758{
4db35314 2759 struct kvm_mmu_page *sp;
d98ba053 2760 LIST_HEAD(invalid_list);
a436036b
AK
2761 int r;
2762
a436036b 2763 r = 0;
531810ca 2764 write_lock(&kvm->mmu_lock);
767d8d8d 2765 for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
7ae680eb 2766 r = 1;
f41d335a 2767 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
7ae680eb 2768 }
d98ba053 2769 kvm_mmu_commit_zap_page(kvm, &invalid_list);
531810ca 2770 write_unlock(&kvm->mmu_lock);
1cb3f3ae 2771
a436036b 2772 return r;
cea0f0e7 2773}
96ad91ae
SC
2774
2775static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2776{
2777 gpa_t gpa;
2778 int r;
2779
347a0d0d 2780 if (vcpu->arch.mmu->root_role.direct)
96ad91ae
SC
2781 return 0;
2782
2783 gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);
2784
2785 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2786
2787 return r;
2788}
cea0f0e7 2789
4d78d0b3 2790static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
9cf5cf5a
XG
2791{
2792 trace_kvm_mmu_unsync_page(sp);
4d78d0b3 2793 ++kvm->stat.mmu_unsync;
9cf5cf5a
XG
2794 sp->unsync = 1;
2795
2796 kvm_mmu_mark_parents_unsync(sp);
9cf5cf5a
XG
2797}
2798
0337f585
SC
2799/*
2800 * Attempt to unsync any shadow pages that can be reached by the specified gfn,
2801 * KVM is creating a writable mapping for said gfn. Returns 0 if all pages
2802 * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
2803 * be write-protected.
2804 */
8283e36a 2805int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
2839180c 2806 gfn_t gfn, bool can_unsync, bool prefetch)
4731d4c7 2807{
5c520e90 2808 struct kvm_mmu_page *sp;
ce25681d 2809 bool locked = false;
4731d4c7 2810
0337f585
SC
2811 /*
2812 * Force write-protection if the page is being tracked. Note, the page
2813 * track machinery is used to write-protect upper-level shadow pages,
2814 * i.e. this guards the role.level == 4K assertion below!
2815 */
7b574863 2816 if (kvm_gfn_is_write_tracked(kvm, slot, gfn))
0337f585 2817 return -EPERM;
9cf5cf5a 2818
0337f585
SC
2819 /*
2820 * The page is not write-tracked, mark existing shadow pages unsync
2821 * unless KVM is synchronizing an unsync SP (can_unsync = false). In
2822 * that case, KVM must complete emulation of the guest TLB flush before
2823 * allowing shadow pages to become unsync (writable by the guest).
2824 */
767d8d8d 2825 for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) {
36a2e677 2826 if (!can_unsync)
0337f585 2827 return -EPERM;
36a2e677 2828
5c520e90
XG
2829 if (sp->unsync)
2830 continue;
9cf5cf5a 2831
2839180c 2832 if (prefetch)
f1c4a88c
LJ
2833 return -EEXIST;
2834
ce25681d
SC
2835 /*
2836 * TDP MMU page faults require an additional spinlock as they
2837 * run with mmu_lock held for read, not write, and the unsync
2838 * logic is not thread safe. Take the spinklock regardless of
2839 * the MMU type to avoid extra conditionals/parameters, there's
2840 * no meaningful penalty if mmu_lock is held for write.
2841 */
2842 if (!locked) {
2843 locked = true;
4d78d0b3 2844 spin_lock(&kvm->arch.mmu_unsync_pages_lock);
ce25681d
SC
2845
2846 /*
2847 * Recheck after taking the spinlock, a different vCPU
2848 * may have since marked the page unsync. A false
e59f75de 2849 * negative on the unprotected check above is not
ce25681d 2850 * possible as clearing sp->unsync _must_ hold mmu_lock
e59f75de 2851 * for write, i.e. unsync cannot transition from 1->0
ce25681d
SC
2852 * while this CPU holds mmu_lock for read (or write).
2853 */
2854 if (READ_ONCE(sp->unsync))
2855 continue;
2856 }
2857
20ba462d 2858 WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);
4d78d0b3 2859 kvm_unsync_page(kvm, sp);
4731d4c7 2860 }
ce25681d 2861 if (locked)
4d78d0b3 2862 spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
3d0c27ad 2863
578e1c4d
JS
2864 /*
2865 * We need to ensure that the marking of unsync pages is visible
2866 * before the SPTE is updated to allow writes because
2867 * kvm_mmu_sync_roots() checks the unsync flags without holding
2868 * the MMU lock and so can race with this. If the SPTE was updated
2869 * before the page had been marked as unsync-ed, something like the
2870 * following could happen:
2871 *
2872 * CPU 1 CPU 2
2873 * ---------------------------------------------------------------------
2874 * 1.2 Host updates SPTE
2875 * to be writable
2876 * 2.1 Guest writes a GPTE for GVA X.
2877 * (GPTE being in the guest page table shadowed
2878 * by the SP from CPU 1.)
2879 * This reads SPTE during the page table walk.
2880 * Since SPTE.W is read as 1, there is no
2881 * fault.
2882 *
2883 * 2.2 Guest issues TLB flush.
2884 * That causes a VM Exit.
2885 *
0337f585
SC
2886 * 2.3 Walking of unsync pages sees sp->unsync is
2887 * false and skips the page.
578e1c4d
JS
2888 *
2889 * 2.4 Guest accesses GVA X.
2890 * Since the mapping in the SP was not updated,
2891 * so the old mapping for GVA X incorrectly
2892 * gets used.
2893 * 1.1 Host marks SP
2894 * as unsync
2895 * (sp->unsync = true)
2896 *
2897 * The write barrier below ensures that 1.1 happens before 1.2 and thus
264d3dc1
LJ
2898 * the situation in 2.4 does not arise. It pairs with the read barrier
2899 * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
578e1c4d
JS
2900 */
2901 smp_wmb();
2902
0337f585 2903 return 0;
4731d4c7
MT
2904}
2905
8a9f566a
DM
2906static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
2907 u64 *sptep, unsigned int pte_access, gfn_t gfn,
a12f4381 2908 kvm_pfn_t pfn, struct kvm_page_fault *fault)
1e73f9dd 2909{
d786c778 2910 struct kvm_mmu_page *sp = sptep_to_sp(sptep);
eb5cd7ff 2911 int level = sp->role.level;
1e73f9dd 2912 int was_rmapped = 0;
c4371c2a 2913 int ret = RET_PF_FIXED;
c2a4eadf 2914 bool flush = false;
ad67e480 2915 bool wrprot;
d786c778 2916 u64 spte;
1e73f9dd 2917
a12f4381
PB
2918 /* Prefetching always gets a writable pfn. */
2919 bool host_writable = !fault || fault->map_writable;
2839180c 2920 bool prefetch = !fault || fault->prefetch;
a12f4381 2921 bool write_fault = fault && fault->write;
1e73f9dd 2922
a54aa15c 2923 if (unlikely(is_noslot_pfn(pfn))) {
1075d41e 2924 vcpu->stat.pf_mmio_spte_created++;
a54aa15c
SC
2925 mark_mmio_spte(vcpu, sptep, gfn, pte_access);
2926 return RET_PF_EMULATE;
2927 }
2928
afd28fe1 2929 if (is_shadow_present_pte(*sptep)) {
1e73f9dd
MT
2930 /*
2931 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
2932 * the parent of the now unreachable PTE.
2933 */
3bae0459 2934 if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
1e73f9dd 2935 struct kvm_mmu_page *child;
d555c333 2936 u64 pte = *sptep;
1e73f9dd 2937
5e3edd7e 2938 child = spte_to_child_sp(pte);
069f30c6 2939 drop_parent_pte(vcpu->kvm, child, sptep);
c2a4eadf 2940 flush = true;
d555c333 2941 } else if (pfn != spte_to_pfn(*sptep)) {
c3707958 2942 drop_spte(vcpu->kvm, sptep);
c2a4eadf 2943 flush = true;
6bed6b9e
JR
2944 } else
2945 was_rmapped = 1;
1e73f9dd 2946 }
852e3c19 2947
2839180c 2948 wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch,
7158bee4 2949 true, host_writable, &spte);
d786c778
PB
2950
2951 if (*sptep == spte) {
2952 ret = RET_PF_SPURIOUS;
2953 } else {
d786c778 2954 flush |= mmu_spte_update(sptep, spte);
5959ff4a 2955 trace_kvm_mmu_set_spte(level, gfn, sptep);
d786c778
PB
2956 }
2957
ad67e480 2958 if (wrprot) {
1e73f9dd 2959 if (write_fault)
9b8ebbdb 2960 ret = RET_PF_EMULATE;
a378b4e6 2961 }
c3134ce2 2962
d786c778 2963 if (flush)
4ad980ae 2964 kvm_flush_remote_tlbs_gfn(vcpu->kvm, gfn, level);
1e73f9dd 2965
4293ddb7 2966 if (!was_rmapped) {
d786c778 2967 WARN_ON_ONCE(ret == RET_PF_SPURIOUS);
6a97575d
DM
2968 rmap_add(vcpu, slot, sptep, gfn, pte_access);
2969 } else {
2970 /* Already rmapped but the pte_access bits may have changed. */
79e48cec 2971 kvm_mmu_page_set_access(sp, spte_index(sptep), pte_access);
1c4f1fd6 2972 }
cb9aaa30 2973
9b8ebbdb 2974 return ret;
1c4f1fd6
AK
2975}
2976
957ed9ef
XG
2977static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
2978 struct kvm_mmu_page *sp,
2979 u64 *start, u64 *end)
2980{
2981 struct page *pages[PTE_PREFETCH_NUM];
d9ef13c2 2982 struct kvm_memory_slot *slot;
0a2b64c5 2983 unsigned int access = sp->role.access;
957ed9ef
XG
2984 int i, ret;
2985 gfn_t gfn;
2986
79e48cec 2987 gfn = kvm_mmu_page_get_gfn(sp, spte_index(start));
d9ef13c2
PB
2988 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
2989 if (!slot)
957ed9ef
XG
2990 return -1;
2991
d9ef13c2 2992 ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
957ed9ef
XG
2993 if (ret <= 0)
2994 return -1;
2995
43fdcda9 2996 for (i = 0; i < ret; i++, gfn++, start++) {
8a9f566a 2997 mmu_set_spte(vcpu, slot, start, access, gfn,
a12f4381 2998 page_to_pfn(pages[i]), NULL);
43fdcda9
JS
2999 put_page(pages[i]);
3000 }
957ed9ef
XG
3001
3002 return 0;
3003}
3004
3005static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
3006 struct kvm_mmu_page *sp, u64 *sptep)
3007{
3008 u64 *spte, *start = NULL;
3009 int i;
3010
20ba462d 3011 WARN_ON_ONCE(!sp->role.direct);
957ed9ef 3012
79e48cec 3013 i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1);
957ed9ef
XG
3014 spte = sp->spt + i;
3015
3016 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
c3707958 3017 if (is_shadow_present_pte(*spte) || spte == sptep) {
957ed9ef
XG
3018 if (!start)
3019 continue;
3020 if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
c6cecc4b 3021 return;
957ed9ef
XG
3022 start = NULL;
3023 } else if (!start)
3024 start = spte;
3025 }
c6cecc4b
SC
3026 if (start)
3027 direct_pte_prefetch_many(vcpu, sp, start, spte);
957ed9ef
XG
3028}
3029
3030static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
3031{
3032 struct kvm_mmu_page *sp;
3033
57354682 3034 sp = sptep_to_sp(sptep);
ac8d57e5 3035
957ed9ef 3036 /*
ac8d57e5
PF
3037 * Without accessed bits, there's no way to distinguish between
3038 * actually accessed translations and prefetched, so disable pte
3039 * prefetch if accessed bits aren't available.
957ed9ef 3040 */
ac8d57e5 3041 if (sp_ad_disabled(sp))
957ed9ef
XG
3042 return;
3043
3bae0459 3044 if (sp->role.level > PG_LEVEL_4K)
957ed9ef
XG
3045 return;
3046
4a42d848
DS
3047 /*
3048 * If addresses are being invalidated, skip prefetching to avoid
3049 * accidentally prefetching those addresses.
3050 */
20ec3ebd 3051 if (unlikely(vcpu->kvm->mmu_invalidate_in_progress))
4a42d848
DS
3052 return;
3053
957ed9ef
XG
3054 __direct_pte_prefetch(vcpu, sp, sptep);
3055}
3056
65e3b446
SC
3057/*
3058 * Lookup the mapping level for @gfn in the current mm.
3059 *
3060 * WARNING! Use of host_pfn_mapping_level() requires the caller and the end
3061 * consumer to be tied into KVM's handlers for MMU notifier events!
3062 *
3063 * There are several ways to safely use this helper:
3064 *
8569992d 3065 * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
65e3b446
SC
3066 * consuming it. In this case, mmu_lock doesn't need to be held during the
3067 * lookup, but it does need to be held while checking the MMU notifier.
3068 *
3069 * - Hold mmu_lock AND ensure there is no in-progress MMU notifier invalidation
3070 * event for the hva. This can be done by explicit checking the MMU notifier
3071 * or by ensuring that KVM already has a valid mapping that covers the hva.
3072 *
3073 * - Do not use the result to install new mappings, e.g. use the host mapping
3074 * level only to decide whether or not to zap an entry. In this case, it's
3075 * not required to hold mmu_lock (though it's highly likely the caller will
3076 * want to hold mmu_lock anyways, e.g. to modify SPTEs).
3077 *
3078 * Note! The lookup can still race with modifications to host page tables, but
3079 * the above "rules" ensure KVM will not _consume_ the result of the walk if a
3080 * race with the primary MMU occurs.
3081 */
a8ac499b 3082static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
8ca6f063 3083 const struct kvm_memory_slot *slot)
db543216 3084{
284dc493 3085 int level = PG_LEVEL_4K;
db543216 3086 unsigned long hva;
44187235 3087 unsigned long flags;
44187235
MZ
3088 pgd_t pgd;
3089 p4d_t p4d;
3090 pud_t pud;
3091 pmd_t pmd;
db543216 3092
293e306e
SC
3093 /*
3094 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
3095 * is not solely for performance, it's also necessary to avoid the
3096 * "writable" check in __gfn_to_hva_many(), which will always fail on
3097 * read-only memslots due to gfn_to_hva() assuming writes. Earlier
3098 * page fault steps have already verified the guest isn't writing a
3099 * read-only memslot.
3100 */
db543216
SC
3101 hva = __gfn_to_hva_memslot(slot, gfn);
3102
44187235 3103 /*
65e3b446
SC
3104 * Disable IRQs to prevent concurrent tear down of host page tables,
3105 * e.g. if the primary MMU promotes a P*D to a huge page and then frees
3106 * the original page table.
44187235
MZ
3107 */
3108 local_irq_save(flags);
3109
65e3b446
SC
3110 /*
3111 * Read each entry once. As above, a non-leaf entry can be promoted to
3112 * a huge page _during_ this walk. Re-reading the entry could send the
e72c7c2b 3113 * walk into the weeks, e.g. p*d_leaf() returns false (sees the old
65e3b446
SC
3114 * value) and then p*d_offset() walks into the target huge page instead
3115 * of the old page table (sees the new value).
3116 */
44187235
MZ
3117 pgd = READ_ONCE(*pgd_offset(kvm->mm, hva));
3118 if (pgd_none(pgd))
3119 goto out;
3120
3121 p4d = READ_ONCE(*p4d_offset(&pgd, hva));
3122 if (p4d_none(p4d) || !p4d_present(p4d))
3123 goto out;
db543216 3124
44187235
MZ
3125 pud = READ_ONCE(*pud_offset(&p4d, hva));
3126 if (pud_none(pud) || !pud_present(pud))
3127 goto out;
3128
0a845e0f 3129 if (pud_leaf(pud)) {
44187235
MZ
3130 level = PG_LEVEL_1G;
3131 goto out;
3132 }
3133
3134 pmd = READ_ONCE(*pmd_offset(&pud, hva));
3135 if (pmd_none(pmd) || !pmd_present(pmd))
3136 goto out;
3137
2f709f7b 3138 if (pmd_leaf(pmd))
44187235
MZ
3139 level = PG_LEVEL_2M;
3140
3141out:
3142 local_irq_restore(flags);
db543216
SC
3143 return level;
3144}
3145
8dd2eee9
CP
3146static int __kvm_mmu_max_mapping_level(struct kvm *kvm,
3147 const struct kvm_memory_slot *slot,
3148 gfn_t gfn, int max_level, bool is_private)
1b6d9d9e
SC
3149{
3150 struct kvm_lpage_info *linfo;
ec607a56 3151 int host_level;
1b6d9d9e
SC
3152
3153 max_level = min(max_level, max_huge_page_level);
3154 for ( ; max_level > PG_LEVEL_4K; max_level--) {
3155 linfo = lpage_info_slot(gfn, slot, max_level);
3156 if (!linfo->disallow_lpage)
3157 break;
3158 }
3159
8dd2eee9
CP
3160 if (is_private)
3161 return max_level;
3162
1b6d9d9e
SC
3163 if (max_level == PG_LEVEL_4K)
3164 return PG_LEVEL_4K;
3165
a8ac499b 3166 host_level = host_pfn_mapping_level(kvm, gfn, slot);
ec607a56 3167 return min(host_level, max_level);
1b6d9d9e
SC
3168}
3169
8dd2eee9
CP
3170int kvm_mmu_max_mapping_level(struct kvm *kvm,
3171 const struct kvm_memory_slot *slot, gfn_t gfn,
3172 int max_level)
3173{
3174 bool is_private = kvm_slot_can_be_private(slot) &&
3175 kvm_mem_is_private(kvm, gfn);
3176
3177 return __kvm_mmu_max_mapping_level(kvm, slot, gfn, max_level, is_private);
3178}
3179
73a3c659 3180void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
0885904d 3181{
e710c5f6 3182 struct kvm_memory_slot *slot = fault->slot;
17eff019
SC
3183 kvm_pfn_t mask;
3184
73a3c659 3185 fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
3cf06612 3186
73a3c659
PB
3187 if (unlikely(fault->max_level == PG_LEVEL_4K))
3188 return;
17eff019 3189
5d49f08c 3190 if (is_error_noslot_pfn(fault->pfn))
73a3c659 3191 return;
17eff019 3192
e710c5f6 3193 if (kvm_slot_dirty_track_enabled(slot))
73a3c659 3194 return;
293e306e 3195
3cf06612
SC
3196 /*
3197 * Enforce the iTLB multihit workaround after capturing the requested
3198 * level, which will be used to do precise, accurate accounting.
3199 */
8dd2eee9
CP
3200 fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot,
3201 fault->gfn, fault->max_level,
3202 fault->is_private);
73a3c659
PB
3203 if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
3204 return;
0885904d
SC
3205
3206 /*
20ec3ebd 3207 * mmu_invalidate_retry() was successful and mmu_lock is held, so
17eff019 3208 * the pmd can't be split from under us.
0885904d 3209 */
73a3c659
PB
3210 fault->goal_level = fault->req_level;
3211 mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
3212 VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
3213 fault->pfn &= ~mask;
0885904d
SC
3214}
3215
536f0e6a 3216void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level)
b8e8c830 3217{
536f0e6a
PB
3218 if (cur_level > PG_LEVEL_4K &&
3219 cur_level == fault->goal_level &&
b8e8c830 3220 is_shadow_present_pte(spte) &&
76901e56
MZ
3221 !is_large_pte(spte) &&
3222 spte_to_child_sp(spte)->nx_huge_page_disallowed) {
b8e8c830 3223 /*
6c882ef4
DM
3224 * A small SPTE exists for this pfn, but FNAME(fetch),
3225 * direct_map(), or kvm_tdp_mmu_map() would like to create a
3226 * large PTE instead: just force them to go down another level,
3227 * patching back for them into pfn the next 9 bits of the
3228 * address.
b8e8c830 3229 */
536f0e6a
PB
3230 u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
3231 KVM_PAGES_PER_HPAGE(cur_level - 1);
3232 fault->pfn |= fault->gfn & page_mask;
3233 fault->goal_level--;
b8e8c830
PB
3234 }
3235}
3236
6c882ef4 3237static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
140754bc 3238{
3fcf2d1b 3239 struct kvm_shadow_walk_iterator it;
140754bc 3240 struct kvm_mmu_page *sp;
73a3c659 3241 int ret;
43b74355 3242 gfn_t base_gfn = fault->gfn;
6aa8b732 3243
73a3c659 3244 kvm_mmu_hugepage_adjust(vcpu, fault);
4cd071d1 3245
f0066d94 3246 trace_kvm_mmu_spte_requested(fault);
43b74355 3247 for_each_shadow_entry(vcpu, fault->addr, it) {
b8e8c830
PB
3248 /*
3249 * We cannot overwrite existing page tables with an NX
3250 * large page, as the leaf could be executable.
3251 */
73a3c659 3252 if (fault->nx_huge_page_workaround_enabled)
536f0e6a 3253 disallowed_hugepage_adjust(fault, *it.sptep, it.level);
b8e8c830 3254
c667a3ba 3255 base_gfn = gfn_round_for_level(fault->gfn, it.level);
73a3c659 3256 if (it.level == fault->goal_level)
9f652d21 3257 break;
6aa8b732 3258
2e65e842 3259 sp = kvm_mmu_get_child_sp(vcpu, it.sptep, base_gfn, true, ACC_ALL);
0cd8dc73
PB
3260 if (sp == ERR_PTR(-EEXIST))
3261 continue;
03fffc54
SC
3262
3263 link_shadow_page(vcpu, it.sptep, sp);
b5b0977f 3264 if (fault->huge_page_disallowed)
55c510e2 3265 account_nx_huge_page(vcpu->kvm, sp,
428e9216 3266 fault->req_level >= it.level);
9f652d21 3267 }
3fcf2d1b 3268
b1a429fb
SC
3269 if (WARN_ON_ONCE(it.level != fault->goal_level))
3270 return -EFAULT;
3271
8a9f566a 3272 ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
a12f4381 3273 base_gfn, fault->pfn, fault);
12703759
SC
3274 if (ret == RET_PF_SPURIOUS)
3275 return ret;
3276
3fcf2d1b 3277 direct_pte_prefetch(vcpu, it.sptep);
3fcf2d1b 3278 return ret;
6aa8b732
AK
3279}
3280
cd08d178 3281static void kvm_send_hwpoison_signal(struct kvm_memory_slot *slot, gfn_t gfn)
bf998156 3282{
cd08d178
DM
3283 unsigned long hva = gfn_to_hva_memslot(slot, gfn);
3284
3285 send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva, PAGE_SHIFT, current);
bf998156
HY
3286}
3287
cd08d178 3288static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
bf998156 3289{
cd08d178 3290 if (is_sigpending_pfn(fault->pfn)) {
76657687
PX
3291 kvm_handle_signal_exit(vcpu);
3292 return -EINTR;
3293 }
3294
4d8b81ab
XG
3295 /*
3296 * Do not cache the mmio info caused by writing the readonly gfn
3297 * into the spte otherwise read access on readonly gfn also can
3298 * caused mmio page fault and treat it as mmio access.
4d8b81ab 3299 */
cd08d178 3300 if (fault->pfn == KVM_PFN_ERR_RO_FAULT)
9b8ebbdb 3301 return RET_PF_EMULATE;
4d8b81ab 3302
cd08d178
DM
3303 if (fault->pfn == KVM_PFN_ERR_HWPOISON) {
3304 kvm_send_hwpoison_signal(fault->slot, fault->gfn);
9b8ebbdb 3305 return RET_PF_RETRY;
d7c55201 3306 }
edba23e5 3307
2c151b25 3308 return -EFAULT;
bf998156
HY
3309}
3310
354c908c
DM
3311static int kvm_handle_noslot_fault(struct kvm_vcpu *vcpu,
3312 struct kvm_page_fault *fault,
3313 unsigned int access)
d7c55201 3314{
354c908c 3315 gva_t gva = fault->is_tdp ? 0 : fault->addr;
d7c55201 3316
354c908c
DM
3317 vcpu_cache_mmio_info(vcpu, gva, fault->gfn,
3318 access & shadow_mmio_access_mask);
3a13f4fe 3319
354c908c
DM
3320 /*
3321 * If MMIO caching is disabled, emulate immediately without
3322 * touching the shadow page tables as attempting to install an
3323 * MMIO SPTE will just be an expensive nop.
3324 */
3325 if (unlikely(!enable_mmio_caching))
3326 return RET_PF_EMULATE;
3327
3328 /*
3329 * Do not create an MMIO SPTE for a gfn greater than host.MAXPHYADDR,
3330 * any guest that generates such gfns is running nested and is being
3331 * tricked by L0 userspace (you can observe gfn > L1.MAXPHYADDR if and
3332 * only if L1's MAXPHYADDR is inaccurate with respect to the
3333 * hardware's).
3334 */
3335 if (unlikely(fault->gfn > kvm_mmu_max_gfn()))
3336 return RET_PF_EMULATE;
d7c55201 3337
5276c616 3338 return RET_PF_CONTINUE;
d7c55201
XG
3339}
3340
3c8ad5a6 3341static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
c7ba5b48 3342{
1c118b82 3343 /*
5c64aba5
SC
3344 * Page faults with reserved bits set, i.e. faults on MMIO SPTEs, only
3345 * reach the common page fault handler if the SPTE has an invalid MMIO
3346 * generation number. Refreshing the MMIO generation needs to go down
3347 * the slow path. Note, EPT Misconfigs do NOT set the PRESENT flag!
1c118b82 3348 */
3c8ad5a6 3349 if (fault->rsvd)
1c118b82
XG
3350 return false;
3351
c7ba5b48 3352 /*
f160c7b7 3353 * #PF can be fast if:
f160c7b7 3354 *
54275f74
SC
3355 * 1. The shadow page table entry is not present and A/D bits are
3356 * disabled _by KVM_, which could mean that the fault is potentially
3357 * caused by access tracking (if enabled). If A/D bits are enabled
3358 * by KVM, but disabled by L1 for L2, KVM is forced to disable A/D
3359 * bits for L2 and employ access tracking, but the fast page fault
3360 * mechanism only supports direct MMUs.
3361 * 2. The shadow page table entry is present, the access is a write,
3362 * and no reserved bits are set (MMIO SPTEs cannot be "fixed"), i.e.
3363 * the fault was caused by a write-protection violation. If the
3364 * SPTE is MMU-writable (determined later), the fault can be fixed
3365 * by setting the Writable bit, which can be done out of mmu_lock.
c7ba5b48 3366 */
5c64aba5
SC
3367 if (!fault->present)
3368 return !kvm_ad_enabled();
3369
3370 /*
3371 * Note, instruction fetches and writes are mutually exclusive, ignore
3372 * the "exec" flag.
3373 */
3374 return fault->write;
c7ba5b48
XG
3375}
3376
97dceba2
JS
3377/*
3378 * Returns true if the SPTE was fixed successfully. Otherwise,
3379 * someone else modified the SPTE from its original value.
3380 */
f3d90f90
SC
3381static bool fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu,
3382 struct kvm_page_fault *fault,
3383 u64 *sptep, u64 old_spte, u64 new_spte)
c7ba5b48 3384{
9b51a630
KH
3385 /*
3386 * Theoretically we could also set dirty bit (and flush TLB) here in
3387 * order to eliminate unnecessary PML logging. See comments in
3388 * set_spte. But fast_page_fault is very unlikely to happen with PML
3389 * enabled, so we do not do this. This might result in the same GPA
3390 * to be logged in PML buffer again when the write really happens, and
3391 * eventually to be called by mark_page_dirty twice. But it's also no
3392 * harm. This also avoids the TLB flush needed after setting dirty bit
3393 * so non-PML cases won't be impacted.
3394 *
3395 * Compare with set_spte where instead shadow_dirty_mask is set.
3396 */
2db2f46f 3397 if (!try_cmpxchg64(sptep, &old_spte, new_spte))
97dceba2
JS
3398 return false;
3399
e710c5f6
DM
3400 if (is_writable_pte(new_spte) && !is_writable_pte(old_spte))
3401 mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
c7ba5b48
XG
3402
3403 return true;
3404}
3405
3c8ad5a6 3406static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
d3e328f2 3407{
3c8ad5a6 3408 if (fault->exec)
d3e328f2
JS
3409 return is_executable_pte(spte);
3410
3c8ad5a6 3411 if (fault->write)
d3e328f2
JS
3412 return is_writable_pte(spte);
3413
3414 /* Fault was on Read access */
3415 return spte & PT_PRESENT_MASK;
3416}
3417
6e8eb206
DM
3418/*
3419 * Returns the last level spte pointer of the shadow page walk for the given
3420 * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
3421 * walk could be performed, returns NULL and *spte does not contain valid data.
3422 *
3423 * Contract:
3424 * - Must be called between walk_shadow_page_lockless_{begin,end}.
3425 * - The returned sptep must not be used after walk_shadow_page_lockless_end.
3426 */
3427static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
3428{
3429 struct kvm_shadow_walk_iterator iterator;
3430 u64 old_spte;
3431 u64 *sptep = NULL;
3432
3433 for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
3434 sptep = iterator.sptep;
3435 *spte = old_spte;
6e8eb206
DM
3436 }
3437
3438 return sptep;
3439}
3440
c7ba5b48 3441/*
c4371c2a 3442 * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS.
c7ba5b48 3443 */
3c8ad5a6 3444static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
c7ba5b48 3445{
92a476cb 3446 struct kvm_mmu_page *sp;
c4371c2a 3447 int ret = RET_PF_INVALID;
1de9992f
L
3448 u64 spte;
3449 u64 *sptep;
97dceba2 3450 uint retry_count = 0;
c7ba5b48 3451
3c8ad5a6 3452 if (!page_fault_can_be_fast(fault))
c4371c2a 3453 return ret;
c7ba5b48
XG
3454
3455 walk_shadow_page_lockless_begin(vcpu);
c7ba5b48 3456
97dceba2 3457 do {
d3e328f2 3458 u64 new_spte;
c7ba5b48 3459
dfe0ecc6 3460 if (tdp_mmu_enabled)
3c8ad5a6 3461 sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
6e8eb206 3462 else
3c8ad5a6 3463 sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
d162f30a 3464
1de9992f
L
3465 /*
3466 * It's entirely possible for the mapping to have been zapped
3467 * by a different task, but the root page should always be
3468 * available as the vCPU holds a reference to its root(s).
3469 */
3470 if (WARN_ON_ONCE(!sptep))
3471 spte = REMOVED_SPTE;
3472
ec89e643
SC
3473 if (!is_shadow_present_pte(spte))
3474 break;
3475
6e8eb206 3476 sp = sptep_to_sp(sptep);
97dceba2
JS
3477 if (!is_last_spte(spte, sp->role.level))
3478 break;
c7ba5b48 3479
97dceba2 3480 /*
f160c7b7
JS
3481 * Check whether the memory access that caused the fault would
3482 * still cause it if it were to be performed right now. If not,
3483 * then this is a spurious fault caused by TLB lazily flushed,
3484 * or some other CPU has already fixed the PTE after the
3485 * current CPU took the fault.
97dceba2
JS
3486 *
3487 * Need not check the access of upper level table entries since
3488 * they are always ACC_ALL.
3489 */
3c8ad5a6 3490 if (is_access_allowed(fault, spte)) {
c4371c2a 3491 ret = RET_PF_SPURIOUS;
d3e328f2
JS
3492 break;
3493 }
f160c7b7 3494
d3e328f2
JS
3495 new_spte = spte;
3496
54275f74
SC
3497 /*
3498 * KVM only supports fixing page faults outside of MMU lock for
3499 * direct MMUs, nested MMUs are always indirect, and KVM always
3500 * uses A/D bits for non-nested MMUs. Thus, if A/D bits are
3501 * enabled, the SPTE can't be an access-tracked SPTE.
3502 */
3503 if (unlikely(!kvm_ad_enabled()) && is_access_track_spte(spte))
d3e328f2
JS
3504 new_spte = restore_acc_track_spte(new_spte);
3505
3506 /*
54275f74
SC
3507 * To keep things simple, only SPTEs that are MMU-writable can
3508 * be made fully writable outside of mmu_lock, e.g. only SPTEs
3509 * that were write-protected for dirty-logging or access
3510 * tracking are handled here. Don't bother checking if the
3511 * SPTE is writable to prioritize running with A/D bits enabled.
3512 * The is_access_allowed() check above handles the common case
3513 * of the fault being spurious, and the SPTE is known to be
3514 * shadow-present, i.e. except for access tracking restoration
3515 * making the new SPTE writable, the check is wasteful.
d3e328f2 3516 */
706c9c55 3517 if (fault->write && is_mmu_writable_spte(spte)) {
d3e328f2 3518 new_spte |= PT_WRITABLE_MASK;
f160c7b7
JS
3519
3520 /*
10c30de0
JS
3521 * Do not fix write-permission on the large spte when
3522 * dirty logging is enabled. Since we only dirty the
3523 * first page into the dirty-bitmap in
d3e328f2
JS
3524 * fast_pf_fix_direct_spte(), other pages are missed
3525 * if its slot has dirty logging enabled.
3526 *
3527 * Instead, we let the slow page fault path create a
3528 * normal spte to fix the access.
f160c7b7 3529 */
10c30de0
JS
3530 if (sp->role.level > PG_LEVEL_4K &&
3531 kvm_slot_dirty_track_enabled(fault->slot))
f160c7b7 3532 break;
97dceba2 3533 }
c7ba5b48 3534
f160c7b7 3535 /* Verify that the fault can be handled in the fast path */
d3e328f2 3536 if (new_spte == spte ||
3c8ad5a6 3537 !is_access_allowed(fault, new_spte))
97dceba2
JS
3538 break;
3539
3540 /*
3541 * Currently, fast page fault only works for direct mapping
3542 * since the gfn is not stable for indirect shadow page. See
3ecad8c2 3543 * Documentation/virt/kvm/locking.rst to get more detail.
97dceba2 3544 */
e710c5f6 3545 if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) {
c4371c2a 3546 ret = RET_PF_FIXED;
97dceba2 3547 break;
c4371c2a 3548 }
97dceba2
JS
3549
3550 if (++retry_count > 4) {
8d20bd63 3551 pr_warn_once("Fast #PF retrying more than 4 times.\n");
97dceba2
JS
3552 break;
3553 }
3554
97dceba2 3555 } while (true);
c126d94f 3556
f0066d94 3557 trace_fast_page_fault(vcpu, fault, sptep, spte, ret);
c7ba5b48
XG
3558 walk_shadow_page_lockless_end(vcpu);
3559
1075d41e
SC
3560 if (ret != RET_PF_INVALID)
3561 vcpu->stat.pf_fast++;
3562
c4371c2a 3563 return ret;
c7ba5b48
XG
3564}
3565
74b566e6
JS
3566static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
3567 struct list_head *invalid_list)
17ac10ad 3568{
4db35314 3569 struct kvm_mmu_page *sp;
17ac10ad 3570
74b566e6 3571 if (!VALID_PAGE(*root_hpa))
7b53aa56 3572 return;
35af577a 3573
c5f2d564 3574 sp = root_to_sp(*root_hpa);
20ba462d 3575 if (WARN_ON_ONCE(!sp))
9191b8f0 3576 return;
02c00b3a 3577
576a15de
SC
3578 if (is_tdp_mmu_page(sp)) {
3579 lockdep_assert_held_read(&kvm->mmu_lock);
5f3c8c91 3580 kvm_tdp_mmu_put_root(kvm, sp);
576a15de
SC
3581 } else {
3582 lockdep_assert_held_write(&kvm->mmu_lock);
3583 if (!--sp->root_count && sp->role.invalid)
3584 kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3585 }
17ac10ad 3586
74b566e6
JS
3587 *root_hpa = INVALID_PAGE;
3588}
3589
08fb59d8 3590/* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
0c1c92f1 3591void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
6a82cd1c 3592 ulong roots_to_free)
74b566e6 3593{
576a15de 3594 bool is_tdp_mmu = tdp_mmu_enabled && mmu->root_role.direct;
74b566e6
JS
3595 int i;
3596 LIST_HEAD(invalid_list);
594bef79 3597 bool free_active_root;
74b566e6 3598
f94db0c8
SC
3599 WARN_ON_ONCE(roots_to_free & ~KVM_MMU_ROOTS_ALL);
3600
b94742c9 3601 BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
74b566e6 3602
08fb59d8 3603 /* Before acquiring the MMU lock, see if we need to do any real work. */
594bef79
PB
3604 free_active_root = (roots_to_free & KVM_MMU_ROOT_CURRENT)
3605 && VALID_PAGE(mmu->root.hpa);
3606
3607 if (!free_active_root) {
b94742c9
JS
3608 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3609 if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
3610 VALID_PAGE(mmu->prev_roots[i].hpa))
3611 break;
3612
3613 if (i == KVM_MMU_NUM_PREV_ROOTS)
3614 return;
3615 }
35af577a 3616
576a15de
SC
3617 if (is_tdp_mmu)
3618 read_lock(&kvm->mmu_lock);
3619 else
3620 write_lock(&kvm->mmu_lock);
17ac10ad 3621
b94742c9
JS
3622 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
3623 if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
4d710de9 3624 mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
b94742c9 3625 &invalid_list);
7c390d35 3626
08fb59d8 3627 if (free_active_root) {
0e3223d8
SC
3628 if (kvm_mmu_is_dummy_root(mmu->root.hpa)) {
3629 /* Nothing to cleanup for dummy roots. */
3630 } else if (root_to_sp(mmu->root.hpa)) {
b9e5603c 3631 mmu_free_root_page(kvm, &mmu->root.hpa, &invalid_list);
04d45551 3632 } else if (mmu->pae_root) {
c834e5e4
SC
3633 for (i = 0; i < 4; ++i) {
3634 if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))
3635 continue;
3636
3637 mmu_free_root_page(kvm, &mmu->pae_root[i],
3638 &invalid_list);
3639 mmu->pae_root[i] = INVALID_PAE_ROOT;
3640 }
08fb59d8 3641 }
b9e5603c
PB
3642 mmu->root.hpa = INVALID_PAGE;
3643 mmu->root.pgd = 0;
17ac10ad 3644 }
74b566e6 3645
576a15de
SC
3646 if (is_tdp_mmu) {
3647 read_unlock(&kvm->mmu_lock);
3648 WARN_ON_ONCE(!list_empty(&invalid_list));
3649 } else {
3650 kvm_mmu_commit_zap_page(kvm, &invalid_list);
3651 write_unlock(&kvm->mmu_lock);
3652 }
17ac10ad 3653}
74b566e6 3654EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
17ac10ad 3655
0c1c92f1 3656void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu)
25b62c62
SC
3657{
3658 unsigned long roots_to_free = 0;
c5f2d564 3659 struct kvm_mmu_page *sp;
25b62c62
SC
3660 hpa_t root_hpa;
3661 int i;
3662
3663 /*
3664 * This should not be called while L2 is active, L2 can't invalidate
3665 * _only_ its own roots, e.g. INVVPID unconditionally exits.
3666 */
7a458f0e 3667 WARN_ON_ONCE(mmu->root_role.guest_mode);
25b62c62
SC
3668
3669 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
3670 root_hpa = mmu->prev_roots[i].hpa;
3671 if (!VALID_PAGE(root_hpa))
3672 continue;
3673
c5f2d564
SC
3674 sp = root_to_sp(root_hpa);
3675 if (!sp || sp->role.guest_mode)
25b62c62
SC
3676 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
3677 }
3678
0c1c92f1 3679 kvm_mmu_free_roots(kvm, mmu, roots_to_free);
25b62c62
SC
3680}
3681EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots);
3682
2e65e842 3683static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, int quadrant,
86938ab6 3684 u8 level)
651dd37a 3685{
2e65e842 3686 union kvm_mmu_page_role role = vcpu->arch.mmu->root_role;
651dd37a 3687 struct kvm_mmu_page *sp;
8123f265 3688
2e65e842 3689 role.level = level;
7f497775 3690 role.quadrant = quadrant;
2e65e842 3691
7f497775
DM
3692 WARN_ON_ONCE(quadrant && !role.has_4_byte_gpte);
3693 WARN_ON_ONCE(role.direct && role.has_4_byte_gpte);
2e65e842 3694
87654643 3695 sp = kvm_mmu_get_shadow_page(vcpu, gfn, role);
8123f265
SC
3696 ++sp->root_count;
3697
8123f265
SC
3698 return __pa(sp->spt);
3699}
3700
3701static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
3702{
b37233c9 3703 struct kvm_mmu *mmu = vcpu->arch.mmu;
a972e29c 3704 u8 shadow_root_level = mmu->root_role.level;
8123f265 3705 hpa_t root;
7ebaf15e 3706 unsigned i;
4a38162e
PB
3707 int r;
3708
f5238c2a
SC
3709 if (tdp_mmu_enabled)
3710 return kvm_tdp_mmu_alloc_root(vcpu);
3711
4a38162e
PB
3712 write_lock(&vcpu->kvm->mmu_lock);
3713 r = make_mmu_pages_available(vcpu);
3714 if (r < 0)
3715 goto out_unlock;
651dd37a 3716
f5238c2a 3717 if (shadow_root_level >= PT64_ROOT_4LEVEL) {
86938ab6 3718 root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level);
b9e5603c 3719 mmu->root.hpa = root;
8123f265 3720 } else if (shadow_root_level == PT32E_ROOT_LEVEL) {
4a38162e
PB
3721 if (WARN_ON_ONCE(!mmu->pae_root)) {
3722 r = -EIO;
3723 goto out_unlock;
3724 }
73ad1606 3725
651dd37a 3726 for (i = 0; i < 4; ++i) {
c834e5e4 3727 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
651dd37a 3728
7f497775 3729 root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), 0,
2e65e842 3730 PT32_ROOT_LEVEL);
17e368d9 3731 mmu->pae_root[i] = root | PT_PRESENT_MASK |
d2263de1 3732 shadow_me_value;
651dd37a 3733 }
b9e5603c 3734 mmu->root.hpa = __pa(mmu->pae_root);
73ad1606
SC
3735 } else {
3736 WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level);
4a38162e
PB
3737 r = -EIO;
3738 goto out_unlock;
73ad1606 3739 }
3651c7fc 3740
b9e5603c
PB
3741 /* root.pgd is ignored for direct MMUs. */
3742 mmu->root.pgd = 0;
4a38162e
PB
3743out_unlock:
3744 write_unlock(&vcpu->kvm->mmu_lock);
3745 return r;
651dd37a
JR
3746}
3747
1e76a3ce
DS
3748static int mmu_first_shadow_root_alloc(struct kvm *kvm)
3749{
3750 struct kvm_memslots *slots;
3751 struct kvm_memory_slot *slot;
a54d8066 3752 int r = 0, i, bkt;
1e76a3ce
DS
3753
3754 /*
3755 * Check if this is the first shadow root being allocated before
3756 * taking the lock.
3757 */
3758 if (kvm_shadow_root_allocated(kvm))
3759 return 0;
3760
3761 mutex_lock(&kvm->slots_arch_lock);
3762
3763 /* Recheck, under the lock, whether this is the first shadow root. */
3764 if (kvm_shadow_root_allocated(kvm))
3765 goto out_unlock;
3766
3767 /*
3768 * Check if anything actually needs to be allocated, e.g. all metadata
3769 * will be allocated upfront if TDP is disabled.
3770 */
3771 if (kvm_memslots_have_rmaps(kvm) &&
3772 kvm_page_track_write_tracking_enabled(kvm))
3773 goto out_success;
3774
eed52e43 3775 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
1e76a3ce 3776 slots = __kvm_memslots(kvm, i);
a54d8066 3777 kvm_for_each_memslot(slot, bkt, slots) {
1e76a3ce
DS
3778 /*
3779 * Both of these functions are no-ops if the target is
3780 * already allocated, so unconditionally calling both
3781 * is safe. Intentionally do NOT free allocations on
3782 * failure to avoid having to track which allocations
3783 * were made now versus when the memslot was created.
3784 * The metadata is guaranteed to be freed when the slot
3785 * is freed, and will be kept/used if userspace retries
3786 * KVM_RUN instead of killing the VM.
3787 */
3788 r = memslot_rmap_alloc(slot, slot->npages);
3789 if (r)
3790 goto out_unlock;
3791 r = kvm_page_track_write_tracking_alloc(slot);
3792 if (r)
3793 goto out_unlock;
3794 }
3795 }
3796
3797 /*
3798 * Ensure that shadow_root_allocated becomes true strictly after
3799 * all the related pointers are set.
3800 */
3801out_success:
3802 smp_store_release(&kvm->arch.shadow_root_allocated, true);
3803
3804out_unlock:
3805 mutex_unlock(&kvm->slots_arch_lock);
3806 return r;
3807}
3808
651dd37a 3809static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
17ac10ad 3810{
b37233c9 3811 struct kvm_mmu *mmu = vcpu->arch.mmu;
6e0918ae 3812 u64 pdptrs[4], pm_mask;
be01e8e2 3813 gfn_t root_gfn, root_pgd;
7f497775 3814 int quadrant, i, r;
8123f265 3815 hpa_t root;
3bb65a22 3816
2fdcc1b3 3817 root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
a130066f 3818 root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
17ac10ad 3819
0e3223d8
SC
3820 if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
3821 mmu->root.hpa = kvm_mmu_get_dummy_root();
3822 return 0;
3823 }
651dd37a 3824
4a38162e
PB
3825 /*
3826 * On SVM, reading PDPTRs might access guest memory, which might fault
3827 * and thus might sleep. Grab the PDPTRs before acquiring mmu_lock.
3828 */
4d25502a 3829 if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
6e0918ae
SC
3830 for (i = 0; i < 4; ++i) {
3831 pdptrs[i] = mmu->get_pdptr(vcpu, i);
3832 if (!(pdptrs[i] & PT_PRESENT_MASK))
3833 continue;
3834
0e3223d8
SC
3835 if (!kvm_vcpu_is_visible_gfn(vcpu, pdptrs[i] >> PAGE_SHIFT))
3836 pdptrs[i] = 0;
6e0918ae
SC
3837 }
3838 }
3839
1e76a3ce 3840 r = mmu_first_shadow_root_alloc(vcpu->kvm);
d501f747
BG
3841 if (r)
3842 return r;
3843
4a38162e
PB
3844 write_lock(&vcpu->kvm->mmu_lock);
3845 r = make_mmu_pages_available(vcpu);
3846 if (r < 0)
3847 goto out_unlock;
3848
651dd37a
JR
3849 /*
3850 * Do we shadow a long mode page table? If so we need to
3851 * write-protect the guests page table root.
3852 */
4d25502a 3853 if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
8123f265 3854 root = mmu_alloc_root(vcpu, root_gfn, 0,
86938ab6 3855 mmu->root_role.level);
b9e5603c 3856 mmu->root.hpa = root;
be01e8e2 3857 goto set_root_pgd;
17ac10ad 3858 }
f87f9288 3859
4a38162e
PB
3860 if (WARN_ON_ONCE(!mmu->pae_root)) {
3861 r = -EIO;
3862 goto out_unlock;
3863 }
73ad1606 3864
651dd37a
JR
3865 /*
3866 * We shadow a 32 bit page table. This may be a legacy 2-level
81407ca5
JR
3867 * or a PAE 3-level page table. In either case we need to be aware that
3868 * the shadow page table may be a PAE or a long mode page table.
651dd37a 3869 */
e54f1ff2 3870 pm_mask = PT_PRESENT_MASK | shadow_me_value;
a972e29c 3871 if (mmu->root_role.level >= PT64_ROOT_4LEVEL) {
81407ca5
JR
3872 pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;
3873
03ca4589 3874 if (WARN_ON_ONCE(!mmu->pml4_root)) {
4a38162e
PB
3875 r = -EIO;
3876 goto out_unlock;
3877 }
03ca4589 3878 mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;
cb0f722a 3879
a972e29c 3880 if (mmu->root_role.level == PT64_ROOT_5LEVEL) {
cb0f722a
WH
3881 if (WARN_ON_ONCE(!mmu->pml5_root)) {
3882 r = -EIO;
3883 goto out_unlock;
3884 }
3885 mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;
3886 }
04d45551
SC
3887 }
3888
17ac10ad 3889 for (i = 0; i < 4; ++i) {
c834e5e4 3890 WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
6e6ec584 3891
4d25502a 3892 if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
6e0918ae 3893 if (!(pdptrs[i] & PT_PRESENT_MASK)) {
c834e5e4 3894 mmu->pae_root[i] = INVALID_PAE_ROOT;
417726a3
AK
3895 continue;
3896 }
6e0918ae 3897 root_gfn = pdptrs[i] >> PAGE_SHIFT;
5a7388c2 3898 }
8facbbff 3899
7f497775
DM
3900 /*
3901 * If shadowing 32-bit non-PAE page tables, each PAE page
3902 * directory maps one quarter of the guest's non-PAE page
3903 * directory. Othwerise each PAE page direct shadows one guest
3904 * PAE page directory so that quadrant should be 0.
3905 */
3906 quadrant = (mmu->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
3907
3908 root = mmu_alloc_root(vcpu, root_gfn, quadrant, PT32_ROOT_LEVEL);
b37233c9 3909 mmu->pae_root[i] = root | pm_mask;
17ac10ad 3910 }
81407ca5 3911
a972e29c 3912 if (mmu->root_role.level == PT64_ROOT_5LEVEL)
b9e5603c 3913 mmu->root.hpa = __pa(mmu->pml5_root);
a972e29c 3914 else if (mmu->root_role.level == PT64_ROOT_4LEVEL)
b9e5603c 3915 mmu->root.hpa = __pa(mmu->pml4_root);
ba0a194f 3916 else
b9e5603c 3917 mmu->root.hpa = __pa(mmu->pae_root);
81407ca5 3918
be01e8e2 3919set_root_pgd:
b9e5603c 3920 mmu->root.pgd = root_pgd;
4a38162e
PB
3921out_unlock:
3922 write_unlock(&vcpu->kvm->mmu_lock);
ad7dc69a 3923
c6c937d6 3924 return r;
17ac10ad
AK
3925}
3926
748e52b9
SC
3927static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
3928{
3929 struct kvm_mmu *mmu = vcpu->arch.mmu;
a972e29c 3930 bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;
cb0f722a
WH
3931 u64 *pml5_root = NULL;
3932 u64 *pml4_root = NULL;
3933 u64 *pae_root;
81407ca5
JR
3934
3935 /*
748e52b9
SC
3936 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
3937 * tables are allocated and initialized at root creation as there is no
3938 * equivalent level in the guest's NPT to shadow. Allocate the tables
3939 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
81407ca5 3940 */
347a0d0d
PB
3941 if (mmu->root_role.direct ||
3942 mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
a972e29c 3943 mmu->root_role.level < PT64_ROOT_4LEVEL)
748e52b9 3944 return 0;
81407ca5 3945
a717a780
SC
3946 /*
3947 * NPT, the only paging mode that uses this horror, uses a fixed number
3948 * of levels for the shadow page tables, e.g. all MMUs are 4-level or
3949 * all MMus are 5-level. Thus, this can safely require that pml5_root
3950 * is allocated if the other roots are valid and pml5 is needed, as any
3951 * prior MMU would also have required pml5.
3952 */
3953 if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))
748e52b9 3954 return 0;
81407ca5 3955
748e52b9
SC
3956 /*
3957 * The special roots should always be allocated in concert. Yell and
3958 * bail if KVM ends up in a state where only one of the roots is valid.
3959 */
cb0f722a 3960 if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||
a717a780 3961 (need_pml5 && mmu->pml5_root)))
748e52b9 3962 return -EIO;
81407ca5 3963
4a98623d
SC
3964 /*
3965 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and
3966 * doesn't need to be decrypted.
3967 */
748e52b9
SC
3968 pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3969 if (!pae_root)
3970 return -ENOMEM;
81407ca5 3971
cb0f722a 3972#ifdef CONFIG_X86_64
03ca4589 3973 pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
cb0f722a
WH
3974 if (!pml4_root)
3975 goto err_pml4;
3976
a717a780 3977 if (need_pml5) {
cb0f722a
WH
3978 pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3979 if (!pml5_root)
3980 goto err_pml5;
81407ca5 3981 }
cb0f722a 3982#endif
81407ca5 3983
748e52b9 3984 mmu->pae_root = pae_root;
03ca4589 3985 mmu->pml4_root = pml4_root;
cb0f722a 3986 mmu->pml5_root = pml5_root;
ad7dc69a 3987
8986ecc0 3988 return 0;
cb0f722a
WH
3989
3990#ifdef CONFIG_X86_64
3991err_pml5:
3992 free_page((unsigned long)pml4_root);
3993err_pml4:
3994 free_page((unsigned long)pae_root);
3995 return -ENOMEM;
3996#endif
17ac10ad
AK
3997}
3998
264d3dc1
LJ
3999static bool is_unsync_root(hpa_t root)
4000{
4001 struct kvm_mmu_page *sp;
4002
0e3223d8 4003 if (!VALID_PAGE(root) || kvm_mmu_is_dummy_root(root))
61b05a9f
LJ
4004 return false;
4005
264d3dc1
LJ
4006 /*
4007 * The read barrier orders the CPU's read of SPTE.W during the page table
4008 * walk before the reads of sp->unsync/sp->unsync_children here.
4009 *
4010 * Even if another CPU was marking the SP as unsync-ed simultaneously,
4011 * any guest page table changes are not guaranteed to be visible anyway
4012 * until this VCPU issues a TLB flush strictly after those changes are
4013 * made. We only need to ensure that the other CPU sets these flags
4014 * before any actual changes to the page tables are made. The comments
4015 * in mmu_try_to_unsync_pages() describe what could go wrong if this
4016 * requirement isn't satisfied.
4017 */
4018 smp_rmb();
c5f2d564 4019 sp = root_to_sp(root);
5d6a3221
SC
4020
4021 /*
4022 * PAE roots (somewhat arbitrarily) aren't backed by shadow pages, the
4023 * PDPTEs for a given PAE root need to be synchronized individually.
4024 */
4025 if (WARN_ON_ONCE(!sp))
4026 return false;
4027
264d3dc1
LJ
4028 if (sp->unsync || sp->unsync_children)
4029 return true;
4030
4031 return false;
4032}
4033
578e1c4d 4034void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
0ba73cda
MT
4035{
4036 int i;
4037 struct kvm_mmu_page *sp;
4038
347a0d0d 4039 if (vcpu->arch.mmu->root_role.direct)
81407ca5
JR
4040 return;
4041
b9e5603c 4042 if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))
0ba73cda 4043 return;
6903074c 4044
56f17dd3 4045 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
578e1c4d 4046
4d25502a 4047 if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
b9e5603c 4048 hpa_t root = vcpu->arch.mmu->root.hpa;
578e1c4d 4049
264d3dc1 4050 if (!is_unsync_root(root))
578e1c4d
JS
4051 return;
4052
c5f2d564
SC
4053 sp = root_to_sp(root);
4054
531810ca 4055 write_lock(&vcpu->kvm->mmu_lock);
65855ed8 4056 mmu_sync_children(vcpu, sp, true);
531810ca 4057 write_unlock(&vcpu->kvm->mmu_lock);
0ba73cda
MT
4058 return;
4059 }
578e1c4d 4060
531810ca 4061 write_lock(&vcpu->kvm->mmu_lock);
578e1c4d 4062
0ba73cda 4063 for (i = 0; i < 4; ++i) {
44dd3ffa 4064 hpa_t root = vcpu->arch.mmu->pae_root[i];
0ba73cda 4065
c834e5e4 4066 if (IS_VALID_PAE_ROOT(root)) {
5e3edd7e 4067 sp = spte_to_child_sp(root);
65855ed8 4068 mmu_sync_children(vcpu, sp, true);
0ba73cda
MT
4069 }
4070 }
0ba73cda 4071
531810ca 4072 write_unlock(&vcpu->kvm->mmu_lock);
0ba73cda
MT
4073}
4074
61b05a9f
LJ
4075void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
4076{
4077 unsigned long roots_to_free = 0;
4078 int i;
4079
4080 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
4081 if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
4082 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
4083
4084 /* sync prev_roots by simply freeing them */
0c1c92f1 4085 kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
61b05a9f
LJ
4086}
4087
1f5a21ee 4088static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5b22bbe7 4089 gpa_t vaddr, u64 access,
1f5a21ee 4090 struct x86_exception *exception)
6aa8b732 4091{
ab9ae313
AK
4092 if (exception)
4093 exception->error_code = 0;
c59a0f57 4094 return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception);
6539e738
JR
4095}
4096
ded58749 4097static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
ce88decf 4098{
9034e6e8
PB
4099 /*
4100 * A nested guest cannot use the MMIO cache if it is using nested
4101 * page tables, because cr2 is a nGPA while the cache stores GPAs.
4102 */
4103 if (mmu_is_nested(vcpu))
4104 return false;
4105
ce88decf
XG
4106 if (direct)
4107 return vcpu_match_mmio_gpa(vcpu, addr);
4108
4109 return vcpu_match_mmio_gva(vcpu, addr);
4110}
4111
95fb5b02
BG
4112/*
4113 * Return the level of the lowest level SPTE added to sptes.
4114 * That SPTE may be non-present.
c5c8c7c5
DM
4115 *
4116 * Must be called between walk_shadow_page_lockless_{begin,end}.
95fb5b02 4117 */
39b4d43e 4118static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
ce88decf
XG
4119{
4120 struct kvm_shadow_walk_iterator iterator;
2aa07893 4121 int leaf = -1;
95fb5b02 4122 u64 spte;
ce88decf 4123
39b4d43e
SC
4124 for (shadow_walk_init(&iterator, vcpu, addr),
4125 *root_level = iterator.level;
47ab8751
XG
4126 shadow_walk_okay(&iterator);
4127 __shadow_walk_next(&iterator, spte)) {
95fb5b02 4128 leaf = iterator.level;
47ab8751
XG
4129 spte = mmu_spte_get_lockless(iterator.sptep);
4130
dde81f94 4131 sptes[leaf] = spte;
95fb5b02
BG
4132 }
4133
95fb5b02
BG
4134 return leaf;
4135}
4136
9aa41879 4137/* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
95fb5b02
BG
4138static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
4139{
dde81f94 4140 u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
95fb5b02 4141 struct rsvd_bits_validate *rsvd_check;
39b4d43e 4142 int root, leaf, level;
95fb5b02
BG
4143 bool reserved = false;
4144
c5c8c7c5
DM
4145 walk_shadow_page_lockless_begin(vcpu);
4146
78fdd2f0 4147 if (is_tdp_mmu_active(vcpu))
39b4d43e 4148 leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
95fb5b02 4149 else
39b4d43e 4150 leaf = get_walk(vcpu, addr, sptes, &root);
95fb5b02 4151
c5c8c7c5
DM
4152 walk_shadow_page_lockless_end(vcpu);
4153
2aa07893
SC
4154 if (unlikely(leaf < 0)) {
4155 *sptep = 0ull;
4156 return reserved;
4157 }
4158
9aa41879
SC
4159 *sptep = sptes[leaf];
4160
4161 /*
4162 * Skip reserved bits checks on the terminal leaf if it's not a valid
4163 * SPTE. Note, this also (intentionally) skips MMIO SPTEs, which, by
4164 * design, always have reserved bits set. The purpose of the checks is
4165 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.
4166 */
4167 if (!is_shadow_present_pte(sptes[leaf]))
4168 leaf++;
95fb5b02
BG
4169
4170 rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
4171
9aa41879 4172 for (level = root; level >= leaf; level--)
961f8445 4173 reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
47ab8751 4174
47ab8751 4175 if (reserved) {
bb4cdf3a 4176 pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",
47ab8751 4177 __func__, addr);
95fb5b02 4178 for (level = root; level >= leaf; level--)
bb4cdf3a
SC
4179 pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",
4180 sptes[level], level,
961f8445 4181 get_rsvd_bits(rsvd_check, sptes[level], level));
47ab8751 4182 }
ddce6208 4183
47ab8751 4184 return reserved;
ce88decf
XG
4185}
4186
e08d26f0 4187static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
ce88decf
XG
4188{
4189 u64 spte;
47ab8751 4190 bool reserved;
ce88decf 4191
ded58749 4192 if (mmio_info_in_cache(vcpu, addr, direct))
9b8ebbdb 4193 return RET_PF_EMULATE;
ce88decf 4194
95fb5b02 4195 reserved = get_mmio_spte(vcpu, addr, &spte);
20ba462d 4196 if (WARN_ON_ONCE(reserved))
9b8ebbdb 4197 return -EINVAL;
ce88decf
XG
4198
4199 if (is_mmio_spte(spte)) {
4200 gfn_t gfn = get_mmio_spte_gfn(spte);
0a2b64c5 4201 unsigned int access = get_mmio_spte_access(spte);
ce88decf 4202
54bf36aa 4203 if (!check_mmio_spte(vcpu, spte))
9b8ebbdb 4204 return RET_PF_INVALID;
f8f55942 4205
ce88decf
XG
4206 if (direct)
4207 addr = 0;
4f022648
XG
4208
4209 trace_handle_mmio_page_fault(addr, gfn, access);
ce88decf 4210 vcpu_cache_mmio_info(vcpu, addr, gfn, access);
9b8ebbdb 4211 return RET_PF_EMULATE;
ce88decf
XG
4212 }
4213
ce88decf
XG
4214 /*
4215 * If the page table is zapped by other cpus, let CPU fault again on
4216 * the address.
4217 */
9b8ebbdb 4218 return RET_PF_RETRY;
ce88decf 4219}
ce88decf 4220
3d0c27ad 4221static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
b8a5d551 4222 struct kvm_page_fault *fault)
3d0c27ad 4223{
b8a5d551 4224 if (unlikely(fault->rsvd))
3d0c27ad
XG
4225 return false;
4226
b8a5d551 4227 if (!fault->present || !fault->write)
3d0c27ad
XG
4228 return false;
4229
4230 /*
4231 * guest is writing the page which is write tracked which can
4232 * not be fixed by page fault handler.
4233 */
7b574863 4234 if (kvm_gfn_is_write_tracked(vcpu->kvm, fault->slot, fault->gfn))
3d0c27ad
XG
4235 return true;
4236
4237 return false;
4238}
4239
e5691a81
XG
4240static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
4241{
4242 struct kvm_shadow_walk_iterator iterator;
4243 u64 spte;
4244
e5691a81 4245 walk_shadow_page_lockless_begin(vcpu);
3e44dce4 4246 for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
e5691a81 4247 clear_sp_write_flooding_count(iterator.sptep);
e5691a81
XG
4248 walk_shadow_page_lockless_end(vcpu);
4249}
4250
6f3c1fc5
LZ
4251static u32 alloc_apf_token(struct kvm_vcpu *vcpu)
4252{
4253 /* make sure the token value is not 0 */
4254 u32 id = vcpu->arch.apf.id;
4255
4256 if (id << 12 == 0)
4257 vcpu->arch.apf.id = 1;
4258
4259 return (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
4260}
4261
e8c22266
VK
4262static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
4263 gfn_t gfn)
af585b92
GN
4264{
4265 struct kvm_arch_async_pf arch;
fb67e14f 4266
6f3c1fc5 4267 arch.token = alloc_apf_token(vcpu);
af585b92 4268 arch.gfn = gfn;
347a0d0d 4269 arch.direct_map = vcpu->arch.mmu->root_role.direct;
2fdcc1b3 4270 arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
af585b92 4271
9f1a8526
SC
4272 return kvm_setup_async_pf(vcpu, cr2_or_gpa,
4273 kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
af585b92
GN
4274}
4275
8a009d5b
SC
4276void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
4277{
4278 int r;
4279
4280 if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||
4281 work->wakeup_all)
4282 return;
4283
4284 r = kvm_mmu_reload(vcpu);
4285 if (unlikely(r))
4286 return;
4287
4288 if (!vcpu->arch.mmu->root_role.direct &&
2fdcc1b3 4289 work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))
8a009d5b
SC
4290 return;
4291
258d985f 4292 kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true, NULL);
8a009d5b
SC
4293}
4294
8dd2eee9
CP
4295static inline u8 kvm_max_level_for_order(int order)
4296{
4297 BUILD_BUG_ON(KVM_MAX_HUGEPAGE_LEVEL > PG_LEVEL_1G);
4298
4299 KVM_MMU_WARN_ON(order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G) &&
4300 order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M) &&
4301 order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K));
4302
4303 if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G))
4304 return PG_LEVEL_1G;
4305
4306 if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M))
4307 return PG_LEVEL_2M;
4308
4309 return PG_LEVEL_4K;
4310}
4311
4312static void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
4313 struct kvm_page_fault *fault)
4314{
4315 kvm_prepare_memory_fault_exit(vcpu, fault->gfn << PAGE_SHIFT,
4316 PAGE_SIZE, fault->write, fault->exec,
4317 fault->is_private);
4318}
4319
4320static int kvm_faultin_pfn_private(struct kvm_vcpu *vcpu,
4321 struct kvm_page_fault *fault)
4322{
4323 int max_order, r;
4324
4325 if (!kvm_slot_can_be_private(fault->slot)) {
4326 kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
4327 return -EFAULT;
4328 }
4329
4330 r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,
4331 &max_order);
4332 if (r) {
4333 kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
4334 return r;
4335 }
4336
4337 fault->max_level = min(kvm_max_level_for_order(max_order),
4338 fault->max_level);
4339 fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY);
4340
4341 return RET_PF_CONTINUE;
4342}
4343
ba6e3fe2 4344static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
af585b92 4345{
e710c5f6 4346 struct kvm_memory_slot *slot = fault->slot;
af585b92
GN
4347 bool async;
4348
e0c37868
SC
4349 /*
4350 * Retry the page fault if the gfn hit a memslot that is being deleted
4351 * or moved. This ensures any existing SPTEs for the old memslot will
4352 * be zapped before KVM inserts a new MMIO SPTE for the gfn.
4353 */
4354 if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
5276c616 4355 return RET_PF_RETRY;
e0c37868 4356
9cc13d60
ML
4357 if (!kvm_is_visible_memslot(slot)) {
4358 /* Don't expose private memslots to L2. */
4359 if (is_guest_mode(vcpu)) {
e710c5f6 4360 fault->slot = NULL;
3647cd04
PB
4361 fault->pfn = KVM_PFN_NOSLOT;
4362 fault->map_writable = false;
5276c616 4363 return RET_PF_CONTINUE;
9cc13d60
ML
4364 }
4365 /*
4366 * If the APIC access page exists but is disabled, go directly
4367 * to emulation without caching the MMIO access or creating a
4368 * MMIO SPTE. That way the cache doesn't need to be purged
4369 * when the AVIC is re-enabled.
4370 */
4371 if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
5276c616
SC
4372 !kvm_apicv_activated(vcpu->kvm))
4373 return RET_PF_EMULATE;
3a2936de
JM
4374 }
4375
8dd2eee9
CP
4376 if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) {
4377 kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
4378 return -EFAULT;
4379 }
4380
4381 if (fault->is_private)
4382 return kvm_faultin_pfn_private(vcpu, fault);
4383
3520469d 4384 async = false;
c8b88b33 4385 fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, false, &async,
3647cd04
PB
4386 fault->write, &fault->map_writable,
4387 &fault->hva);
af585b92 4388 if (!async)
5276c616 4389 return RET_PF_CONTINUE; /* *pfn has correct page already */
af585b92 4390
2839180c 4391 if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
3647cd04
PB
4392 trace_kvm_try_async_get_page(fault->addr, fault->gfn);
4393 if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
1685c0f3 4394 trace_kvm_async_pf_repeated_fault(fault->addr, fault->gfn);
af585b92 4395 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
5276c616
SC
4396 return RET_PF_RETRY;
4397 } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) {
4398 return RET_PF_RETRY;
4399 }
af585b92
GN
4400 }
4401
76657687
PX
4402 /*
4403 * Allow gup to bail on pending non-fatal signals when it's also allowed
4404 * to wait for IO. Note, gup always bails if it is unable to quickly
4405 * get a page and a fatal signal, i.e. SIGKILL, is pending.
4406 */
4407 fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, true, NULL,
3647cd04
PB
4408 fault->write, &fault->map_writable,
4409 &fault->hva);
5276c616 4410 return RET_PF_CONTINUE;
af585b92
GN
4411}
4412
354c908c
DM
4413static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
4414 unsigned int access)
ba6e3fe2 4415{
56c3a4e4
DM
4416 int ret;
4417
ba6e3fe2
DM
4418 fault->mmu_seq = vcpu->kvm->mmu_invalidate_seq;
4419 smp_rmb();
4420
d02c357e
SC
4421 /*
4422 * Check for a relevant mmu_notifier invalidation event before getting
4423 * the pfn from the primary MMU, and before acquiring mmu_lock.
4424 *
4425 * For mmu_lock, if there is an in-progress invalidation and the kernel
4426 * allows preemption, the invalidation task may drop mmu_lock and yield
4427 * in response to mmu_lock being contended, which is *very* counter-
4428 * productive as this vCPU can't actually make forward progress until
4429 * the invalidation completes.
4430 *
4431 * Retrying now can also avoid unnessary lock contention in the primary
4432 * MMU, as the primary MMU doesn't necessarily hold a single lock for
4433 * the duration of the invalidation, i.e. faulting in a conflicting pfn
4434 * can cause the invalidation to take longer by holding locks that are
4435 * needed to complete the invalidation.
4436 *
4437 * Do the pre-check even for non-preemtible kernels, i.e. even if KVM
4438 * will never yield mmu_lock in response to contention, as this vCPU is
4439 * *guaranteed* to need to retry, i.e. waiting until mmu_lock is held
4440 * to detect retry guarantees the worst case latency for the vCPU.
4441 */
4442 if (fault->slot &&
4443 mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn))
4444 return RET_PF_RETRY;
4445
56c3a4e4
DM
4446 ret = __kvm_faultin_pfn(vcpu, fault);
4447 if (ret != RET_PF_CONTINUE)
4448 return ret;
4449
4450 if (unlikely(is_error_pfn(fault->pfn)))
cd08d178 4451 return kvm_handle_error_pfn(vcpu, fault);
56c3a4e4 4452
354c908c
DM
4453 if (unlikely(!fault->slot))
4454 return kvm_handle_noslot_fault(vcpu, fault, access);
4455
d02c357e
SC
4456 /*
4457 * Check again for a relevant mmu_notifier invalidation event purely to
4458 * avoid contending mmu_lock. Most invalidations will be detected by
4459 * the previous check, but checking is extremely cheap relative to the
4460 * overall cost of failing to detect the invalidation until after
4461 * mmu_lock is acquired.
4462 */
4463 if (mmu_invalidate_retry_gfn_unsafe(vcpu->kvm, fault->mmu_seq, fault->gfn)) {
4464 kvm_release_pfn_clean(fault->pfn);
4465 return RET_PF_RETRY;
4466 }
4467
56c3a4e4 4468 return RET_PF_CONTINUE;
ba6e3fe2
DM
4469}
4470
a955cad8
SC
4471/*
4472 * Returns true if the page fault is stale and needs to be retried, i.e. if the
4473 * root was invalidated by a memslot update or a relevant mmu_notifier fired.
4474 */
4475static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
ba6e3fe2 4476 struct kvm_page_fault *fault)
a955cad8 4477{
c5f2d564 4478 struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);
18c841e1
SC
4479
4480 /* Special roots, e.g. pae_root, are not backed by shadow pages. */
4481 if (sp && is_obsolete_sp(vcpu->kvm, sp))
4482 return true;
4483
4484 /*
4485 * Roots without an associated shadow page are considered invalid if
4486 * there is a pending request to free obsolete roots. The request is
4487 * only a hint that the current root _may_ be obsolete and needs to be
4488 * reloaded, e.g. if the guest frees a PGD that KVM is tracking as a
4489 * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs
4490 * to reload even if no vCPU is actively using the root.
4491 */
527d5cd7 4492 if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
a955cad8
SC
4493 return true;
4494
d02c357e
SC
4495 /*
4496 * Check for a relevant mmu_notifier invalidation event one last time
4497 * now that mmu_lock is held, as the "unsafe" checks performed without
4498 * holding mmu_lock can get false negatives.
4499 */
a955cad8 4500 return fault->slot &&
8569992d 4501 mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);
a955cad8
SC
4502}
4503
4326e57e 4504static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
6aa8b732 4505{
83f06fa7 4506 int r;
ce88decf 4507
0e3223d8
SC
4508 /* Dummy roots are used only for shadowing bad guest roots. */
4509 if (WARN_ON_ONCE(kvm_mmu_is_dummy_root(vcpu->arch.mmu->root.hpa)))
4510 return RET_PF_RETRY;
4511
b8a5d551 4512 if (page_fault_handle_page_track(vcpu, fault))
9b8ebbdb 4513 return RET_PF_EMULATE;
ce88decf 4514
3c8ad5a6 4515 r = fast_page_fault(vcpu, fault);
6e8eb206
DM
4516 if (r != RET_PF_INVALID)
4517 return r;
83291445 4518
378f5cd6 4519 r = mmu_topup_memory_caches(vcpu, false);
e2dec939
AK
4520 if (r)
4521 return r;
714b93da 4522
354c908c 4523 r = kvm_faultin_pfn(vcpu, fault, ACC_ALL);
5276c616 4524 if (r != RET_PF_CONTINUE)
367fd790 4525 return r;
6aa8b732 4526
367fd790 4527 r = RET_PF_RETRY;
9aa8ab43 4528 write_lock(&vcpu->kvm->mmu_lock);
a2855afc 4529
ba6e3fe2 4530 if (is_page_fault_stale(vcpu, fault))
367fd790 4531 goto out_unlock;
a2855afc 4532
7bd7ded6
SC
4533 r = make_mmu_pages_available(vcpu);
4534 if (r)
367fd790 4535 goto out_unlock;
a955cad8 4536
6c882ef4 4537 r = direct_map(vcpu, fault);
0f90e1c1 4538
367fd790 4539out_unlock:
9aa8ab43 4540 write_unlock(&vcpu->kvm->mmu_lock);
3647cd04 4541 kvm_release_pfn_clean(fault->pfn);
367fd790 4542 return r;
6aa8b732
AK
4543}
4544
c501040a
PB
4545static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
4546 struct kvm_page_fault *fault)
0f90e1c1 4547{
0f90e1c1 4548 /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */
4326e57e
PB
4549 fault->max_level = PG_LEVEL_2M;
4550 return direct_page_fault(vcpu, fault);
0f90e1c1
SC
4551}
4552
1261bfa3 4553int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
d0006530 4554 u64 fault_address, char *insn, int insn_len)
1261bfa3
WL
4555{
4556 int r = 1;
9ce372b3 4557 u32 flags = vcpu->arch.apf.host_apf_flags;
1261bfa3 4558
736c291c
SC
4559#ifndef CONFIG_X86_64
4560 /* A 64-bit CR2 should be impossible on 32-bit KVM. */
4561 if (WARN_ON_ONCE(fault_address >> 32))
4562 return -EFAULT;
4563#endif
4564
c595ceee 4565 vcpu->arch.l1tf_flush_l1d = true;
9ce372b3 4566 if (!flags) {
faa03b39 4567 trace_kvm_page_fault(vcpu, fault_address, error_code);
1261bfa3 4568
d0006530 4569 if (kvm_event_needs_reinjection(vcpu))
1261bfa3
WL
4570 kvm_mmu_unprotect_page_virt(vcpu, fault_address);
4571 r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
4572 insn_len);
9ce372b3 4573 } else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
68fd66f1 4574 vcpu->arch.apf.host_apf_flags = 0;
1261bfa3 4575 local_irq_disable();
6bca69ad 4576 kvm_async_pf_task_wait_schedule(fault_address);
1261bfa3 4577 local_irq_enable();
9ce372b3
VK
4578 } else {
4579 WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
1261bfa3 4580 }
9ce372b3 4581
1261bfa3
WL
4582 return r;
4583}
4584EXPORT_SYMBOL_GPL(kvm_handle_page_fault);
4585
9aa8ab43
DM
4586#ifdef CONFIG_X86_64
4587static int kvm_tdp_mmu_page_fault(struct kvm_vcpu *vcpu,
4588 struct kvm_page_fault *fault)
4589{
4590 int r;
4591
4592 if (page_fault_handle_page_track(vcpu, fault))
4593 return RET_PF_EMULATE;
4594
4595 r = fast_page_fault(vcpu, fault);
4596 if (r != RET_PF_INVALID)
4597 return r;
4598
4599 r = mmu_topup_memory_caches(vcpu, false);
4600 if (r)
4601 return r;
4602
4603 r = kvm_faultin_pfn(vcpu, fault, ACC_ALL);
4604 if (r != RET_PF_CONTINUE)
4605 return r;
4606
4607 r = RET_PF_RETRY;
4608 read_lock(&vcpu->kvm->mmu_lock);
4609
4610 if (is_page_fault_stale(vcpu, fault))
4611 goto out_unlock;
4612
9aa8ab43
DM
4613 r = kvm_tdp_mmu_map(vcpu, fault);
4614
4615out_unlock:
4616 read_unlock(&vcpu->kvm->mmu_lock);
4617 kvm_release_pfn_clean(fault->pfn);
4618 return r;
4619}
4620#endif
4621
1affe455 4622bool __kvm_mmu_honors_guest_mtrrs(bool vm_has_noncoherent_dma)
fb72d167 4623{
d5e90a69 4624 /*
1affe455
YZ
4625 * If host MTRRs are ignored (shadow_memtype_mask is non-zero), and the
4626 * VM has non-coherent DMA (DMA doesn't snoop CPU caches), KVM's ABI is
4627 * to honor the memtype from the guest's MTRRs so that guest accesses
4628 * to memory that is DMA'd aren't cached against the guest's wishes.
d5e90a69
SC
4629 *
4630 * Note, KVM may still ultimately ignore guest MTRRs for certain PFNs,
4631 * e.g. KVM will force UC memtype for host MMIO.
4632 */
1affe455
YZ
4633 return vm_has_noncoherent_dma && shadow_memtype_mask;
4634}
4635
4636int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4637{
4638 /*
4639 * If the guest's MTRRs may be used to compute the "real" memtype,
4640 * restrict the mapping level to ensure KVM uses a consistent memtype
4641 * across the entire mapping.
4642 */
4643 if (kvm_mmu_honors_guest_mtrrs(vcpu->kvm)) {
d5e90a69
SC
4644 for ( ; fault->max_level > PG_LEVEL_4K; --fault->max_level) {
4645 int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
c667a3ba
HW
4646 gfn_t base = gfn_round_for_level(fault->gfn,
4647 fault->max_level);
4326e57e 4648
d5e90a69
SC
4649 if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
4650 break;
4651 }
fd136902 4652 }
852e3c19 4653
9aa8ab43
DM
4654#ifdef CONFIG_X86_64
4655 if (tdp_mmu_enabled)
4656 return kvm_tdp_mmu_page_fault(vcpu, fault);
4657#endif
4658
4326e57e 4659 return direct_page_fault(vcpu, fault);
fb72d167
JR
4660}
4661
84a16226 4662static void nonpaging_init_context(struct kvm_mmu *context)
6aa8b732 4663{
6aa8b732 4664 context->page_fault = nonpaging_page_fault;
6aa8b732 4665 context->gva_to_gpa = nonpaging_gva_to_gpa;
c3c6c9fc 4666 context->sync_spte = NULL;
6aa8b732
AK
4667}
4668
be01e8e2 4669static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd,
0be44352
SC
4670 union kvm_mmu_page_role role)
4671{
c30e000e
SC
4672 struct kvm_mmu_page *sp;
4673
4674 if (!VALID_PAGE(root->hpa))
4675 return false;
4676
4677 if (!role.direct && pgd != root->pgd)
4678 return false;
4679
4680 sp = root_to_sp(root->hpa);
4681 if (WARN_ON_ONCE(!sp))
4682 return false;
4683
4684 return role.word == sp->role.word;
0be44352
SC
4685}
4686
b94742c9 4687/*
5499ea73
PB
4688 * Find out if a previously cached root matching the new pgd/role is available,
4689 * and insert the current root as the MRU in the cache.
4690 * If a matching root is found, it is assigned to kvm_mmu->root and
4691 * true is returned.
4692 * If no match is found, kvm_mmu->root is left invalid, the LRU root is
4693 * evicted to make room for the current root, and false is returned.
b94742c9 4694 */
5499ea73
PB
4695static bool cached_root_find_and_keep_current(struct kvm *kvm, struct kvm_mmu *mmu,
4696 gpa_t new_pgd,
4697 union kvm_mmu_page_role new_role)
b94742c9
JS
4698{
4699 uint i;
b94742c9 4700
b9e5603c 4701 if (is_root_usable(&mmu->root, new_pgd, new_role))
0be44352
SC
4702 return true;
4703
b94742c9 4704 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
5499ea73
PB
4705 /*
4706 * The swaps end up rotating the cache like this:
4707 * C 0 1 2 3 (on entry to the function)
4708 * 0 C 1 2 3
4709 * 1 C 0 2 3
4710 * 2 C 0 1 3
4711 * 3 C 0 1 2 (on exit from the loop)
4712 */
b9e5603c 4713 swap(mmu->root, mmu->prev_roots[i]);
b9e5603c 4714 if (is_root_usable(&mmu->root, new_pgd, new_role))
5499ea73 4715 return true;
b94742c9
JS
4716 }
4717
5499ea73
PB
4718 kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
4719 return false;
b94742c9
JS
4720}
4721
5499ea73
PB
4722/*
4723 * Find out if a previously cached root matching the new pgd/role is available.
4724 * On entry, mmu->root is invalid.
4725 * If a matching root is found, it is assigned to kvm_mmu->root, the LRU entry
4726 * of the cache becomes invalid, and true is returned.
4727 * If no match is found, kvm_mmu->root is left invalid and false is returned.
4728 */
4729static bool cached_root_find_without_current(struct kvm *kvm, struct kvm_mmu *mmu,
4730 gpa_t new_pgd,
4731 union kvm_mmu_page_role new_role)
6aa8b732 4732{
5499ea73
PB
4733 uint i;
4734
4735 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
4736 if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))
4737 goto hit;
7c390d35 4738
5499ea73
PB
4739 return false;
4740
4741hit:
4742 swap(mmu->root, mmu->prev_roots[i]);
4743 /* Bubble up the remaining roots. */
4744 for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)
4745 mmu->prev_roots[i] = mmu->prev_roots[i + 1];
4746 mmu->prev_roots[i].hpa = INVALID_PAGE;
4747 return true;
4748}
4749
4750static bool fast_pgd_switch(struct kvm *kvm, struct kvm_mmu *mmu,
4751 gpa_t new_pgd, union kvm_mmu_page_role new_role)
4752{
7c390d35 4753 /*
0e3223d8
SC
4754 * Limit reuse to 64-bit hosts+VMs without "special" roots in order to
4755 * avoid having to deal with PDPTEs and other complexities.
7c390d35 4756 */
c5f2d564 4757 if (VALID_PAGE(mmu->root.hpa) && !root_to_sp(mmu->root.hpa))
5499ea73 4758 kvm_mmu_free_roots(kvm, mmu, KVM_MMU_ROOT_CURRENT);
7c390d35 4759
5499ea73
PB
4760 if (VALID_PAGE(mmu->root.hpa))
4761 return cached_root_find_and_keep_current(kvm, mmu, new_pgd, new_role);
4762 else
4763 return cached_root_find_without_current(kvm, mmu, new_pgd, new_role);
6aa8b732
AK
4764}
4765
d2e5f333 4766void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
6aa8b732 4767{
0c1c92f1 4768 struct kvm_mmu *mmu = vcpu->arch.mmu;
7a458f0e 4769 union kvm_mmu_page_role new_role = mmu->root_role;
0c1c92f1 4770
a7e48ef7
WL
4771 /*
4772 * Return immediately if no usable root was found, kvm_mmu_reload()
4773 * will establish a valid root prior to the next VM-Enter.
4774 */
4775 if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role))
b869855b 4776 return;
b869855b
SC
4777
4778 /*
4779 * It's possible that the cached previous root page is obsolete because
4780 * of a change in the MMU generation number. However, changing the
527d5cd7
SC
4781 * generation number is accompanied by KVM_REQ_MMU_FREE_OBSOLETE_ROOTS,
4782 * which will free the root set here and allocate a new one.
b869855b
SC
4783 */
4784 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
4785
b5129100 4786 if (force_flush_and_sync_on_reuse) {
b869855b
SC
4787 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
4788 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
b5129100 4789 }
b869855b
SC
4790
4791 /*
4792 * The last MMIO access's GVA and GPA are cached in the VCPU. When
4793 * switching to a new CR3, that GVA->GPA mapping may no longer be
4794 * valid. So clear any cached MMIO info even when we don't need to sync
4795 * the shadow page tables.
4796 */
4797 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
4798
daa5b6c1
BG
4799 /*
4800 * If this is a direct root page, it doesn't have a write flooding
4801 * count. Otherwise, clear the write flooding count.
4802 */
c30e000e
SC
4803 if (!new_role.direct) {
4804 struct kvm_mmu_page *sp = root_to_sp(vcpu->arch.mmu->root.hpa);
4805
4806 if (!WARN_ON_ONCE(!sp))
4807 __clear_sp_write_flooding_count(sp);
4808 }
6aa8b732 4809}
be01e8e2 4810EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
0aab33e4 4811
54bf36aa 4812static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
c3e5e415 4813 unsigned int access)
ce88decf
XG
4814{
4815 if (unlikely(is_mmio_spte(*sptep))) {
4816 if (gfn != get_mmio_spte_gfn(*sptep)) {
4817 mmu_spte_clear_no_track(sptep);
4818 return true;
4819 }
4820
54bf36aa 4821 mark_mmio_spte(vcpu, sptep, gfn, access);
ce88decf
XG
4822 return true;
4823 }
4824
4825 return false;
4826}
4827
37406aaa
NHE
4828#define PTTYPE_EPT 18 /* arbitrary */
4829#define PTTYPE PTTYPE_EPT
4830#include "paging_tmpl.h"
4831#undef PTTYPE
4832
6aa8b732
AK
4833#define PTTYPE 64
4834#include "paging_tmpl.h"
4835#undef PTTYPE
4836
4837#define PTTYPE 32
4838#include "paging_tmpl.h"
4839#undef PTTYPE
4840
f3d90f90
SC
4841static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
4842 u64 pa_bits_rsvd, int level, bool nx,
4843 bool gbpages, bool pse, bool amd)
82725b20 4844{
5f7dde7b 4845 u64 gbpages_bit_rsvd = 0;
a0c0feb5 4846 u64 nonleaf_bit8_rsvd = 0;
5b7f575c 4847 u64 high_bits_rsvd;
82725b20 4848
a0a64f50 4849 rsvd_check->bad_mt_xwr = 0;
25d92081 4850
6dc98b86 4851 if (!gbpages)
5f7dde7b 4852 gbpages_bit_rsvd = rsvd_bits(7, 7);
a0c0feb5 4853
5b7f575c
SC
4854 if (level == PT32E_ROOT_LEVEL)
4855 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
4856 else
4857 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4858
4859 /* Note, NX doesn't exist in PDPTEs, this is handled below. */
4860 if (!nx)
4861 high_bits_rsvd |= rsvd_bits(63, 63);
4862
a0c0feb5
PB
4863 /*
4864 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
4865 * leaf entries) on AMD CPUs only.
4866 */
6fec2144 4867 if (amd)
a0c0feb5
PB
4868 nonleaf_bit8_rsvd = rsvd_bits(8, 8);
4869
6dc98b86 4870 switch (level) {
82725b20
DE
4871 case PT32_ROOT_LEVEL:
4872 /* no rsvd bits for 2 level 4K page table entries */
a0a64f50
XG
4873 rsvd_check->rsvd_bits_mask[0][1] = 0;
4874 rsvd_check->rsvd_bits_mask[0][0] = 0;
4875 rsvd_check->rsvd_bits_mask[1][0] =
4876 rsvd_check->rsvd_bits_mask[0][0];
f815bce8 4877
6dc98b86 4878 if (!pse) {
a0a64f50 4879 rsvd_check->rsvd_bits_mask[1][1] = 0;
f815bce8
XG
4880 break;
4881 }
4882
82725b20
DE
4883 if (is_cpuid_PSE36())
4884 /* 36bits PSE 4MB page */
a0a64f50 4885 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
82725b20
DE
4886 else
4887 /* 32 bits PSE 4MB page */
a0a64f50 4888 rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
82725b20
DE
4889 break;
4890 case PT32E_ROOT_LEVEL:
5b7f575c
SC
4891 rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
4892 high_bits_rsvd |
4893 rsvd_bits(5, 8) |
4894 rsvd_bits(1, 2); /* PDPTE */
4895 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd; /* PDE */
4896 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd; /* PTE */
4897 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4898 rsvd_bits(13, 20); /* large page */
a0a64f50
XG
4899 rsvd_check->rsvd_bits_mask[1][0] =
4900 rsvd_check->rsvd_bits_mask[0][0];
82725b20 4901 break;
855feb67 4902 case PT64_ROOT_5LEVEL:
5b7f575c
SC
4903 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
4904 nonleaf_bit8_rsvd |
4905 rsvd_bits(7, 7);
855feb67
YZ
4906 rsvd_check->rsvd_bits_mask[1][4] =
4907 rsvd_check->rsvd_bits_mask[0][4];
df561f66 4908 fallthrough;
2a7266a8 4909 case PT64_ROOT_4LEVEL:
5b7f575c
SC
4910 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
4911 nonleaf_bit8_rsvd |
4912 rsvd_bits(7, 7);
4913 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
4914 gbpages_bit_rsvd;
4915 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
4916 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
a0a64f50
XG
4917 rsvd_check->rsvd_bits_mask[1][3] =
4918 rsvd_check->rsvd_bits_mask[0][3];
5b7f575c
SC
4919 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
4920 gbpages_bit_rsvd |
4921 rsvd_bits(13, 29);
4922 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
4923 rsvd_bits(13, 20); /* large page */
a0a64f50
XG
4924 rsvd_check->rsvd_bits_mask[1][0] =
4925 rsvd_check->rsvd_bits_mask[0][0];
82725b20
DE
4926 break;
4927 }
4928}
4929
c919e881
KH
4930static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
4931 struct kvm_mmu *context)
6dc98b86 4932{
b705a277 4933 __reset_rsvds_bits_mask(&context->guest_rsvd_check,
5b7f575c 4934 vcpu->arch.reserved_gpa_bits,
4d25502a 4935 context->cpu_role.base.level, is_efer_nx(context),
ccf31d6e 4936 guest_can_use(vcpu, X86_FEATURE_GBPAGES),
4e9c0d80 4937 is_cr4_pse(context),
fd706c9b 4938 guest_cpuid_is_amd_compatible(vcpu));
6dc98b86
XG
4939}
4940
f3d90f90
SC
4941static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4942 u64 pa_bits_rsvd, bool execonly,
4943 int huge_page_level)
25d92081 4944{
5b7f575c 4945 u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
84ea5c09 4946 u64 large_1g_rsvd = 0, large_2m_rsvd = 0;
951f9fd7 4947 u64 bad_mt_xwr;
25d92081 4948
84ea5c09
LJ
4949 if (huge_page_level < PG_LEVEL_1G)
4950 large_1g_rsvd = rsvd_bits(7, 7);
4951 if (huge_page_level < PG_LEVEL_2M)
4952 large_2m_rsvd = rsvd_bits(7, 7);
4953
5b7f575c
SC
4954 rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
4955 rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
84ea5c09
LJ
4956 rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
4957 rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
5b7f575c 4958 rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
25d92081
YZ
4959
4960 /* large page */
855feb67 4961 rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
a0a64f50 4962 rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
84ea5c09
LJ
4963 rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
4964 rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
a0a64f50 4965 rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
25d92081 4966
951f9fd7
PB
4967 bad_mt_xwr = 0xFFull << (2 * 8); /* bits 3..5 must not be 2 */
4968 bad_mt_xwr |= 0xFFull << (3 * 8); /* bits 3..5 must not be 3 */
4969 bad_mt_xwr |= 0xFFull << (7 * 8); /* bits 3..5 must not be 7 */
4970 bad_mt_xwr |= REPEAT_BYTE(1ull << 2); /* bits 0..2 must not be 010 */
4971 bad_mt_xwr |= REPEAT_BYTE(1ull << 6); /* bits 0..2 must not be 110 */
4972 if (!execonly) {
4973 /* bits 0..2 must not be 100 unless VMX capabilities allow it */
4974 bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
25d92081 4975 }
951f9fd7 4976 rsvd_check->bad_mt_xwr = bad_mt_xwr;
25d92081
YZ
4977}
4978
81b8eebb 4979static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
84ea5c09 4980 struct kvm_mmu *context, bool execonly, int huge_page_level)
81b8eebb
XG
4981{
4982 __reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
84ea5c09
LJ
4983 vcpu->arch.reserved_gpa_bits, execonly,
4984 huge_page_level);
81b8eebb
XG
4985}
4986
6f8e65a6
SC
4987static inline u64 reserved_hpa_bits(void)
4988{
4989 return rsvd_bits(shadow_phys_bits, 63);
4990}
4991
c258b62b
XG
4992/*
4993 * the page table on host is the shadow page table for the page
4994 * table in guest or amd nested guest, its mmu features completely
4995 * follow the features in guest.
4996 */
16be1d12
SC
4997static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
4998 struct kvm_mmu *context)
c258b62b 4999{
8c985b2d
SC
5000 /* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */
5001 bool is_amd = true;
5002 /* KVM doesn't use 2-level page tables for the shadow MMU. */
5003 bool is_pse = false;
ea2800dd
BS
5004 struct rsvd_bits_validate *shadow_zero_check;
5005 int i;
5f0b8199 5006
a972e29c 5007 WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);
8c985b2d 5008
ea2800dd 5009 shadow_zero_check = &context->shadow_zero_check;
b705a277 5010 __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
a972e29c 5011 context->root_role.level,
7a458f0e 5012 context->root_role.efer_nx,
ccf31d6e
SC
5013 guest_can_use(vcpu, X86_FEATURE_GBPAGES),
5014 is_pse, is_amd);
ea2800dd
BS
5015
5016 if (!shadow_me_mask)
5017 return;
5018
a972e29c 5019 for (i = context->root_role.level; --i >= 0;) {
e54f1ff2
KH
5020 /*
5021 * So far shadow_me_value is a constant during KVM's life
5022 * time. Bits in shadow_me_value are allowed to be set.
5023 * Bits in shadow_me_mask but not in shadow_me_value are
5024 * not allowed to be set.
5025 */
5026 shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;
5027 shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;
5028 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;
5029 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;
ea2800dd
BS
5030 }
5031
c258b62b 5032}
c258b62b 5033
6fec2144
PB
5034static inline bool boot_cpu_is_amd(void)
5035{
5036 WARN_ON_ONCE(!tdp_enabled);
5037 return shadow_x_mask == 0;
5038}
5039
c258b62b
XG
5040/*
5041 * the direct page table on host, use as much mmu features as
5042 * possible, however, kvm currently does not do execution-protection.
5043 */
f3d90f90 5044static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
c258b62b 5045{
ea2800dd
BS
5046 struct rsvd_bits_validate *shadow_zero_check;
5047 int i;
5048
5049 shadow_zero_check = &context->shadow_zero_check;
5050
6fec2144 5051 if (boot_cpu_is_amd())
b705a277 5052 __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
6c6ab524 5053 context->root_role.level, true,
b8291adc 5054 boot_cpu_has(X86_FEATURE_GBPAGES),
8c985b2d 5055 false, true);
c258b62b 5056 else
ea2800dd 5057 __reset_rsvds_bits_mask_ept(shadow_zero_check,
84ea5c09
LJ
5058 reserved_hpa_bits(), false,
5059 max_huge_page_level);
c258b62b 5060
ea2800dd
BS
5061 if (!shadow_me_mask)
5062 return;
5063
a972e29c 5064 for (i = context->root_role.level; --i >= 0;) {
ea2800dd
BS
5065 shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
5066 shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
5067 }
c258b62b
XG
5068}
5069
5070/*
5071 * as the comments in reset_shadow_zero_bits_mask() except it
5072 * is the shadow page table for intel nested guest.
5073 */
5074static void
e8f6e738 5075reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
c258b62b
XG
5076{
5077 __reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
84ea5c09
LJ
5078 reserved_hpa_bits(), execonly,
5079 max_huge_page_level);
c258b62b
XG
5080}
5081
09f037aa
PB
5082#define BYTE_MASK(access) \
5083 ((1 & (access) ? 2 : 0) | \
5084 (2 & (access) ? 4 : 0) | \
5085 (3 & (access) ? 8 : 0) | \
5086 (4 & (access) ? 16 : 0) | \
5087 (5 & (access) ? 32 : 0) | \
5088 (6 & (access) ? 64 : 0) | \
5089 (7 & (access) ? 128 : 0))
5090
5091
c596f147 5092static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
97d64b78 5093{
09f037aa
PB
5094 unsigned byte;
5095
5096 const u8 x = BYTE_MASK(ACC_EXEC_MASK);
5097 const u8 w = BYTE_MASK(ACC_WRITE_MASK);
5098 const u8 u = BYTE_MASK(ACC_USER_MASK);
5099
c596f147
SC
5100 bool cr4_smep = is_cr4_smep(mmu);
5101 bool cr4_smap = is_cr4_smap(mmu);
5102 bool cr0_wp = is_cr0_wp(mmu);
90599c28 5103 bool efer_nx = is_efer_nx(mmu);
97d64b78 5104
97d64b78 5105 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
09f037aa
PB
5106 unsigned pfec = byte << 1;
5107
97ec8c06 5108 /*
09f037aa
PB
5109 * Each "*f" variable has a 1 bit for each UWX value
5110 * that causes a fault with the given PFEC.
97ec8c06 5111 */
97d64b78 5112
09f037aa 5113 /* Faults from writes to non-writable pages */
a6a6d3b1 5114 u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
09f037aa 5115 /* Faults from user mode accesses to supervisor pages */
a6a6d3b1 5116 u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
09f037aa 5117 /* Faults from fetches of non-executable pages*/
a6a6d3b1 5118 u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
09f037aa
PB
5119 /* Faults from kernel mode fetches of user pages */
5120 u8 smepf = 0;
5121 /* Faults from kernel mode accesses of user pages */
5122 u8 smapf = 0;
5123
5124 if (!ept) {
5125 /* Faults from kernel mode accesses to user pages */
5126 u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
5127
5128 /* Not really needed: !nx will cause pte.nx to fault */
90599c28 5129 if (!efer_nx)
09f037aa
PB
5130 ff = 0;
5131
5132 /* Allow supervisor writes if !cr0.wp */
5133 if (!cr0_wp)
5134 wf = (pfec & PFERR_USER_MASK) ? wf : 0;
5135
5136 /* Disallow supervisor fetches of user code if cr4.smep */
5137 if (cr4_smep)
5138 smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;
5139
5140 /*
5141 * SMAP:kernel-mode data accesses from user-mode
5142 * mappings should fault. A fault is considered
5143 * as a SMAP violation if all of the following
39337ad1 5144 * conditions are true:
09f037aa
PB
5145 * - X86_CR4_SMAP is set in CR4
5146 * - A user page is accessed
5147 * - The access is not a fetch
4f4aa80e
LJ
5148 * - The access is supervisor mode
5149 * - If implicit supervisor access or X86_EFLAGS_AC is clear
09f037aa 5150 *
94b4a2f1
LJ
5151 * Here, we cover the first four conditions.
5152 * The fifth is computed dynamically in permission_fault();
09f037aa
PB
5153 * PFERR_RSVD_MASK bit will be set in PFEC if the access is
5154 * *not* subject to SMAP restrictions.
5155 */
5156 if (cr4_smap)
5157 smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
97d64b78 5158 }
09f037aa
PB
5159
5160 mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
97d64b78
AK
5161 }
5162}
5163
2d344105
HH
5164/*
5165* PKU is an additional mechanism by which the paging controls access to
5166* user-mode addresses based on the value in the PKRU register. Protection
5167* key violations are reported through a bit in the page fault error code.
5168* Unlike other bits of the error code, the PK bit is not known at the
5169* call site of e.g. gva_to_gpa; it must be computed directly in
5170* permission_fault based on two bits of PKRU, on some machine state (CR4,
5171* CR0, EFER, CPL), and on other bits of the error code and the page tables.
5172*
5173* In particular the following conditions come from the error code, the
5174* page tables and the machine state:
5175* - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
5176* - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
5177* - PK is always zero if U=0 in the page tables
5178* - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
5179*
5180* The PKRU bitmask caches the result of these four conditions. The error
5181* code (minus the P bit) and the page table's U bit form an index into the
5182* PKRU bitmask. Two bits of the PKRU bitmask are then extracted and ANDed
5183* with the two bits of the PKRU register corresponding to the protection key.
5184* For the first three conditions above the bits will be 00, thus masking
5185* away both AD and WD. For all reads or if the last condition holds, WD
5186* only will be masked away.
5187*/
2e4c0661 5188static void update_pkru_bitmask(struct kvm_mmu *mmu)
2d344105
HH
5189{
5190 unsigned bit;
5191 bool wp;
5192
a3ca5281
CQ
5193 mmu->pkru_mask = 0;
5194
5195 if (!is_cr4_pke(mmu))
2d344105 5196 return;
2d344105 5197
2e4c0661 5198 wp = is_cr0_wp(mmu);
2d344105
HH
5199
5200 for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
5201 unsigned pfec, pkey_bits;
5202 bool check_pkey, check_write, ff, uf, wf, pte_user;
5203
5204 pfec = bit << 1;
5205 ff = pfec & PFERR_FETCH_MASK;
5206 uf = pfec & PFERR_USER_MASK;
5207 wf = pfec & PFERR_WRITE_MASK;
5208
5209 /* PFEC.RSVD is replaced by ACC_USER_MASK. */
5210 pte_user = pfec & PFERR_RSVD_MASK;
5211
5212 /*
5213 * Only need to check the access which is not an
5214 * instruction fetch and is to a user page.
5215 */
5216 check_pkey = (!ff && pte_user);
5217 /*
5218 * write access is controlled by PKRU if it is a
5219 * user access or CR0.WP = 1.
5220 */
5221 check_write = check_pkey && wf && (uf || wp);
5222
5223 /* PKRU.AD stops both read and write access. */
5224 pkey_bits = !!check_pkey;
5225 /* PKRU.WD stops write access. */
5226 pkey_bits |= (!!check_write) << 1;
5227
5228 mmu->pkru_mask |= (pkey_bits & 3) << pfec;
5229 }
5230}
5231
533f9a4b
SC
5232static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
5233 struct kvm_mmu *mmu)
6fd01b71 5234{
533f9a4b
SC
5235 if (!is_cr0_pg(mmu))
5236 return;
6bb69c9b 5237
c919e881 5238 reset_guest_rsvds_bits_mask(vcpu, mmu);
533f9a4b
SC
5239 update_permission_bitmask(mmu, false);
5240 update_pkru_bitmask(mmu);
6fd01b71
AK
5241}
5242
fe660f72 5243static void paging64_init_context(struct kvm_mmu *context)
6aa8b732 5244{
6aa8b732 5245 context->page_fault = paging64_page_fault;
6aa8b732 5246 context->gva_to_gpa = paging64_gva_to_gpa;
c3c6c9fc 5247 context->sync_spte = paging64_sync_spte;
6aa8b732
AK
5248}
5249
84a16226 5250static void paging32_init_context(struct kvm_mmu *context)
6aa8b732 5251{
6aa8b732 5252 context->page_fault = paging32_page_fault;
6aa8b732 5253 context->gva_to_gpa = paging32_gva_to_gpa;
c3c6c9fc 5254 context->sync_spte = paging32_sync_spte;
6aa8b732
AK
5255}
5256
f3d90f90
SC
5257static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
5258 const struct kvm_mmu_role_regs *regs)
e5ed0fb0 5259{
7a7ae829 5260 union kvm_cpu_role role = {0};
e5ed0fb0
PB
5261
5262 role.base.access = ACC_ALL;
5263 role.base.smm = is_smm(vcpu);
5264 role.base.guest_mode = is_guest_mode(vcpu);
5265 role.ext.valid = 1;
5266
5267 if (!____is_cr0_pg(regs)) {
5268 role.base.direct = 1;
5269 return role;
5270 }
5271
5272 role.base.efer_nx = ____is_efer_nx(regs);
5273 role.base.cr0_wp = ____is_cr0_wp(regs);
5274 role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs);
5275 role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs);
5276 role.base.has_4_byte_gpte = !____is_cr4_pae(regs);
60f3cb60
PB
5277
5278 if (____is_efer_lma(regs))
5279 role.base.level = ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL
5280 : PT64_ROOT_4LEVEL;
5281 else if (____is_cr4_pae(regs))
5282 role.base.level = PT32E_ROOT_LEVEL;
5283 else
5284 role.base.level = PT32_ROOT_LEVEL;
e5ed0fb0 5285
e5ed0fb0
PB
5286 role.ext.cr4_smep = ____is_cr4_smep(regs);
5287 role.ext.cr4_smap = ____is_cr4_smap(regs);
5288 role.ext.cr4_pse = ____is_cr4_pse(regs);
5289
5290 /* PKEY and LA57 are active iff long mode is active. */
5291 role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
5292 role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
5293 role.ext.efer_lma = ____is_efer_lma(regs);
5294 return role;
5295}
5296
cf9f4c0e
SC
5297void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
5298 struct kvm_mmu *mmu)
5299{
5300 const bool cr0_wp = kvm_is_cr0_bit_set(vcpu, X86_CR0_WP);
5301
5302 BUILD_BUG_ON((KVM_MMU_CR0_ROLE_BITS & KVM_POSSIBLE_CR0_GUEST_BITS) != X86_CR0_WP);
5303 BUILD_BUG_ON((KVM_MMU_CR4_ROLE_BITS & KVM_POSSIBLE_CR4_GUEST_BITS));
5304
5305 if (is_cr0_wp(mmu) == cr0_wp)
5306 return;
5307
5308 mmu->cpu_role.base.cr0_wp = cr0_wp;
5309 reset_guest_paging_metadata(vcpu, mmu);
5310}
5311
d468d94b
SC
5312static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
5313{
746700d2
WH
5314 /* tdp_root_level is architecture forced level, use it if nonzero */
5315 if (tdp_root_level)
5316 return tdp_root_level;
5317
d468d94b 5318 /* Use 5-level TDP if and only if it's useful/necessary. */
83013059 5319 if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
d468d94b
SC
5320 return 4;
5321
83013059 5322 return max_tdp_level;
d468d94b
SC
5323}
5324
7a458f0e 5325static union kvm_mmu_page_role
8626c120 5326kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
7a7ae829 5327 union kvm_cpu_role cpu_role)
9fa72119 5328{
7a458f0e 5329 union kvm_mmu_page_role role = {0};
9fa72119 5330
7a458f0e
PB
5331 role.access = ACC_ALL;
5332 role.cr0_wp = true;
5333 role.efer_nx = true;
5334 role.smm = cpu_role.base.smm;
5335 role.guest_mode = cpu_role.base.guest_mode;
54275f74 5336 role.ad_disabled = !kvm_ad_enabled();
7a458f0e
PB
5337 role.level = kvm_mmu_get_tdp_level(vcpu);
5338 role.direct = true;
5339 role.has_4_byte_gpte = false;
9fa72119
JS
5340
5341 return role;
5342}
5343
39e7e2bf 5344static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
a7f1de9b 5345 union kvm_cpu_role cpu_role)
fb72d167 5346{
8c008659 5347 struct kvm_mmu *context = &vcpu->arch.root_mmu;
7a458f0e 5348 union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
fb72d167 5349
e5ed0fb0 5350 if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
7a458f0e 5351 root_role.word == context->root_role.word)
7dcd5755
VK
5352 return;
5353
e5ed0fb0 5354 context->cpu_role.as_u64 = cpu_role.as_u64;
7a458f0e 5355 context->root_role.word = root_role.word;
7a02674d 5356 context->page_fault = kvm_tdp_page_fault;
c3c6c9fc 5357 context->sync_spte = NULL;
2fdcc1b3 5358 context->get_guest_pgd = get_guest_cr3;
e4e517b4 5359 context->get_pdptr = kvm_pdptr_read;
cb659db8 5360 context->inject_page_fault = kvm_inject_page_fault;
fb72d167 5361
36f26787 5362 if (!is_cr0_pg(context))
fb72d167 5363 context->gva_to_gpa = nonpaging_gva_to_gpa;
36f26787 5364 else if (is_cr4_pae(context))
4d6931c3 5365 context->gva_to_gpa = paging64_gva_to_gpa;
f4bd6f73 5366 else
4d6931c3 5367 context->gva_to_gpa = paging32_gva_to_gpa;
fb72d167 5368
533f9a4b 5369 reset_guest_paging_metadata(vcpu, context);
e8f6e738 5370 reset_tdp_shadow_zero_bits_mask(context);
fb72d167
JR
5371}
5372
8c008659 5373static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
7a7ae829 5374 union kvm_cpu_role cpu_role,
7a458f0e 5375 union kvm_mmu_page_role root_role)
9fa72119 5376{
e5ed0fb0 5377 if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
7a458f0e 5378 root_role.word == context->root_role.word)
18db1b17 5379 return;
a770f6f2 5380
e5ed0fb0 5381 context->cpu_role.as_u64 = cpu_role.as_u64;
7a458f0e 5382 context->root_role.word = root_role.word;
18db1b17 5383
36f26787 5384 if (!is_cr0_pg(context))
84a16226 5385 nonpaging_init_context(context);
36f26787 5386 else if (is_cr4_pae(context))
fe660f72 5387 paging64_init_context(context);
6aa8b732 5388 else
84a16226 5389 paging32_init_context(context);
a770f6f2 5390
533f9a4b 5391 reset_guest_paging_metadata(vcpu, context);
c258b62b 5392 reset_shadow_zero_bits_mask(vcpu, context);
52fde8df 5393}
0f04a2ac 5394
594e91a1 5395static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
a7f1de9b 5396 union kvm_cpu_role cpu_role)
0f04a2ac 5397{
8c008659 5398 struct kvm_mmu *context = &vcpu->arch.root_mmu;
56b321f9 5399 union kvm_mmu_page_role root_role;
0f04a2ac 5400
56b321f9 5401 root_role = cpu_role.base;
0f04a2ac 5402
56b321f9
PB
5403 /* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */
5404 root_role.level = max_t(u32, root_role.level, PT32E_ROOT_LEVEL);
59505b55 5405
56b321f9
PB
5406 /*
5407 * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role.
5408 * KVM uses NX when TDP is disabled to handle a variety of scenarios,
5409 * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and
5410 * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0.
5411 * The iTLB multi-hit workaround can be toggled at any time, so assume
5412 * NX can be used by any non-nested shadow MMU to avoid having to reset
5413 * MMU contexts.
5414 */
5415 root_role.efer_nx = true;
5416
5417 shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
59505b55
SC
5418}
5419
dbc4739b
SC
5420void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
5421 unsigned long cr4, u64 efer, gpa_t nested_cr3)
0f04a2ac 5422{
8c008659 5423 struct kvm_mmu *context = &vcpu->arch.guest_mmu;
594e91a1
SC
5424 struct kvm_mmu_role_regs regs = {
5425 .cr0 = cr0,
28f091bc 5426 .cr4 = cr4 & ~X86_CR4_PKE,
594e91a1
SC
5427 .efer = efer,
5428 };
7a7ae829 5429 union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
56b321f9
PB
5430 union kvm_mmu_page_role root_role;
5431
5432 /* NPT requires CR0.PG=1. */
5433 WARN_ON_ONCE(cpu_role.base.direct);
5434
5435 root_role = cpu_role.base;
5436 root_role.level = kvm_mmu_get_tdp_level(vcpu);
84e5ffd0
LJ
5437 if (root_role.level == PT64_ROOT_5LEVEL &&
5438 cpu_role.base.level == PT64_ROOT_4LEVEL)
5439 root_role.passthrough = 1;
a506fdd2 5440
7a458f0e 5441 shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
d2e5f333 5442 kvm_mmu_new_pgd(vcpu, nested_cr3);
0f04a2ac
VK
5443}
5444EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu);
52fde8df 5445
7a7ae829 5446static union kvm_cpu_role
a336282d 5447kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
bb1fcc70 5448 bool execonly, u8 level)
9fa72119 5449{
7a7ae829 5450 union kvm_cpu_role role = {0};
14c07ad8 5451
daed87b8
PB
5452 /*
5453 * KVM does not support SMM transfer monitors, and consequently does not
5454 * support the "entry to SMM" control either. role.base.smm is always 0.
5455 */
5456 WARN_ON_ONCE(is_smm(vcpu));
bb1fcc70 5457 role.base.level = level;
bb3b394d 5458 role.base.has_4_byte_gpte = false;
a336282d
VK
5459 role.base.direct = false;
5460 role.base.ad_disabled = !accessed_dirty;
5461 role.base.guest_mode = true;
5462 role.base.access = ACC_ALL;
9fa72119 5463
cd6767c3 5464 role.ext.word = 0;
a336282d 5465 role.ext.execonly = execonly;
cd6767c3 5466 role.ext.valid = 1;
9fa72119
JS
5467
5468 return role;
5469}
5470
ae1e2d10 5471void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
cc022ae1
LJ
5472 int huge_page_level, bool accessed_dirty,
5473 gpa_t new_eptp)
155a97a3 5474{
8c008659 5475 struct kvm_mmu *context = &vcpu->arch.guest_mmu;
bb1fcc70 5476 u8 level = vmx_eptp_page_walk_level(new_eptp);
7a7ae829 5477 union kvm_cpu_role new_mode =
a336282d 5478 kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
bb1fcc70 5479 execonly, level);
a336282d 5480
e5ed0fb0
PB
5481 if (new_mode.as_u64 != context->cpu_role.as_u64) {
5482 /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
5483 context->cpu_role.as_u64 = new_mode.as_u64;
7a458f0e 5484 context->root_role.word = new_mode.base.word;
3cffc89d 5485
3cffc89d
PB
5486 context->page_fault = ept_page_fault;
5487 context->gva_to_gpa = ept_gva_to_gpa;
c3c6c9fc 5488 context->sync_spte = ept_sync_spte;
347a0d0d 5489
3cffc89d
PB
5490 update_permission_bitmask(context, true);
5491 context->pkru_mask = 0;
5492 reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
5493 reset_ept_shadow_zero_bits_mask(context, execonly);
5494 }
3dc773e7 5495
d2e5f333 5496 kvm_mmu_new_pgd(vcpu, new_eptp);
155a97a3
NHE
5497}
5498EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);
5499
39e7e2bf 5500static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
a7f1de9b 5501 union kvm_cpu_role cpu_role)
52fde8df 5502{
8c008659 5503 struct kvm_mmu *context = &vcpu->arch.root_mmu;
ad896af0 5504
a7f1de9b 5505 kvm_init_shadow_mmu(vcpu, cpu_role);
929d1cfa 5506
2fdcc1b3 5507 context->get_guest_pgd = get_guest_cr3;
ad896af0
PB
5508 context->get_pdptr = kvm_pdptr_read;
5509 context->inject_page_fault = kvm_inject_page_fault;
6aa8b732
AK
5510}
5511
39e7e2bf 5512static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
a7f1de9b 5513 union kvm_cpu_role new_mode)
02f59dc9
JR
5514{
5515 struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
5516
e5ed0fb0 5517 if (new_mode.as_u64 == g_context->cpu_role.as_u64)
bf627a92
VK
5518 return;
5519
e5ed0fb0 5520 g_context->cpu_role.as_u64 = new_mode.as_u64;
2fdcc1b3 5521 g_context->get_guest_pgd = get_guest_cr3;
e4e517b4 5522 g_context->get_pdptr = kvm_pdptr_read;
02f59dc9
JR
5523 g_context->inject_page_fault = kvm_inject_page_fault;
5524
5efac074
PB
5525 /*
5526 * L2 page tables are never shadowed, so there is no need to sync
5527 * SPTEs.
5528 */
9fd4a4e3 5529 g_context->sync_spte = NULL;
5efac074 5530
02f59dc9 5531 /*
44dd3ffa 5532 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
0af2593b
DM
5533 * L1's nested page tables (e.g. EPT12). The nested translation
5534 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
5535 * L2's page tables as the first level of translation and L1's
5536 * nested page tables as the second level of translation. Basically
5537 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
02f59dc9 5538 */
fa4b5588 5539 if (!is_paging(vcpu))
1f5a21ee 5540 g_context->gva_to_gpa = nonpaging_gva_to_gpa;
fa4b5588 5541 else if (is_long_mode(vcpu))
1f5a21ee 5542 g_context->gva_to_gpa = paging64_gva_to_gpa;
fa4b5588 5543 else if (is_pae(vcpu))
1f5a21ee 5544 g_context->gva_to_gpa = paging64_gva_to_gpa;
fa4b5588 5545 else
1f5a21ee 5546 g_context->gva_to_gpa = paging32_gva_to_gpa;
02f59dc9 5547
533f9a4b 5548 reset_guest_paging_metadata(vcpu, g_context);
02f59dc9
JR
5549}
5550
c9060662 5551void kvm_init_mmu(struct kvm_vcpu *vcpu)
fb72d167 5552{
39e7e2bf 5553 struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
a7f1de9b 5554 union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
39e7e2bf 5555
02f59dc9 5556 if (mmu_is_nested(vcpu))
a7f1de9b 5557 init_kvm_nested_mmu(vcpu, cpu_role);
02f59dc9 5558 else if (tdp_enabled)
a7f1de9b 5559 init_kvm_tdp_mmu(vcpu, cpu_role);
fb72d167 5560 else
a7f1de9b 5561 init_kvm_softmmu(vcpu, cpu_role);
fb72d167 5562}
1c53da3f 5563EXPORT_SYMBOL_GPL(kvm_init_mmu);
fb72d167 5564
49c6f875
SC
5565void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
5566{
5567 /*
5568 * Invalidate all MMU roles to force them to reinitialize as CPUID
5569 * information is factored into reserved bit calculations.
feb627e8
VK
5570 *
5571 * Correctly handling multiple vCPU models with respect to paging and
5572 * physical address properties) in a single VM would require tracking
5573 * all relevant CPUID information in kvm_mmu_page_role. That is very
5574 * undesirable as it would increase the memory requirements for
338068b5
SC
5575 * gfn_write_track (see struct kvm_mmu_page_role comments). For now
5576 * that problem is swept under the rug; KVM's CPUID API is horrific and
feb627e8 5577 * it's all but impossible to solve it without introducing a new API.
49c6f875 5578 */
1bc26cb9
SC
5579 vcpu->arch.root_mmu.root_role.invalid = 1;
5580 vcpu->arch.guest_mmu.root_role.invalid = 1;
5581 vcpu->arch.nested_mmu.root_role.invalid = 1;
e5ed0fb0
PB
5582 vcpu->arch.root_mmu.cpu_role.ext.valid = 0;
5583 vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;
5584 vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;
49c6f875 5585 kvm_mmu_reset_context(vcpu);
63f5a190
SC
5586
5587 /*
feb627e8
VK
5588 * Changing guest CPUID after KVM_RUN is forbidden, see the comment in
5589 * kvm_arch_vcpu_ioctl().
63f5a190 5590 */
fb3146b4 5591 KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm);
49c6f875
SC
5592}
5593
8a3c1a33 5594void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
6aa8b732 5595{
95f93af4 5596 kvm_mmu_unload(vcpu);
c9060662 5597 kvm_init_mmu(vcpu);
17c3ba9d 5598}
8668a3c4 5599EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
17c3ba9d
AK
5600
5601int kvm_mmu_load(struct kvm_vcpu *vcpu)
6aa8b732 5602{
714b93da
AK
5603 int r;
5604
347a0d0d 5605 r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct);
17c3ba9d
AK
5606 if (r)
5607 goto out;
748e52b9 5608 r = mmu_alloc_special_roots(vcpu);
17c3ba9d
AK
5609 if (r)
5610 goto out;
347a0d0d 5611 if (vcpu->arch.mmu->root_role.direct)
6e6ec584
SC
5612 r = mmu_alloc_direct_roots(vcpu);
5613 else
5614 r = mmu_alloc_shadow_roots(vcpu);
8986ecc0
MT
5615 if (r)
5616 goto out;
a91f387b
SC
5617
5618 kvm_mmu_sync_roots(vcpu);
5619
727a7e27 5620 kvm_mmu_load_pgd(vcpu);
db01416b
SC
5621
5622 /*
5623 * Flush any TLB entries for the new root, the provenance of the root
5624 * is unknown. Even if KVM ensures there are no stale TLB entries
5625 * for a freed root, in theory another hypervisor could have left
5626 * stale entries. Flushing on alloc also allows KVM to skip the TLB
5627 * flush when freeing a root (see kvm_tdp_mmu_put_root()).
5628 */
e27bc044 5629 static_call(kvm_x86_flush_tlb_current)(vcpu);
714b93da
AK
5630out:
5631 return r;
6aa8b732 5632}
17c3ba9d
AK
5633
5634void kvm_mmu_unload(struct kvm_vcpu *vcpu)
5635{
0c1c92f1
PB
5636 struct kvm *kvm = vcpu->kvm;
5637
5638 kvm_mmu_free_roots(kvm, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
20ba462d 5639 WARN_ON_ONCE(VALID_PAGE(vcpu->arch.root_mmu.root.hpa));
0c1c92f1 5640 kvm_mmu_free_roots(kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
20ba462d 5641 WARN_ON_ONCE(VALID_PAGE(vcpu->arch.guest_mmu.root.hpa));
6d58f275 5642 vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
17c3ba9d 5643}
6aa8b732 5644
527d5cd7
SC
5645static bool is_obsolete_root(struct kvm *kvm, hpa_t root_hpa)
5646{
5647 struct kvm_mmu_page *sp;
5648
5649 if (!VALID_PAGE(root_hpa))
5650 return false;
5651
5652 /*
5653 * When freeing obsolete roots, treat roots as obsolete if they don't
0e3223d8
SC
5654 * have an associated shadow page, as it's impossible to determine if
5655 * such roots are fresh or stale. This does mean KVM will get false
527d5cd7
SC
5656 * positives and free roots that don't strictly need to be freed, but
5657 * such false positives are relatively rare:
5658 *
0e3223d8
SC
5659 * (a) only PAE paging and nested NPT have roots without shadow pages
5660 * (or any shadow paging flavor with a dummy root, see note below)
527d5cd7
SC
5661 * (b) remote reloads due to a memslot update obsoletes _all_ roots
5662 * (c) KVM doesn't track previous roots for PAE paging, and the guest
5663 * is unlikely to zap an in-use PGD.
0e3223d8
SC
5664 *
5665 * Note! Dummy roots are unique in that they are obsoleted by memslot
5666 * _creation_! See also FNAME(fetch).
527d5cd7 5667 */
c5f2d564 5668 sp = root_to_sp(root_hpa);
527d5cd7
SC
5669 return !sp || is_obsolete_sp(kvm, sp);
5670}
5671
5672static void __kvm_mmu_free_obsolete_roots(struct kvm *kvm, struct kvm_mmu *mmu)
5673{
5674 unsigned long roots_to_free = 0;
5675 int i;
5676
5677 if (is_obsolete_root(kvm, mmu->root.hpa))
5678 roots_to_free |= KVM_MMU_ROOT_CURRENT;
5679
5680 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
cf4a8693 5681 if (is_obsolete_root(kvm, mmu->prev_roots[i].hpa))
527d5cd7
SC
5682 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
5683 }
5684
5685 if (roots_to_free)
5686 kvm_mmu_free_roots(kvm, mmu, roots_to_free);
5687}
5688
5689void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu)
5690{
5691 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
5692 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
5693}
5694
889e5cbc 5695static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
0e0fee5c 5696 int *bytes)
da4a00f0 5697{
0e0fee5c 5698 u64 gentry = 0;
889e5cbc 5699 int r;
72016f3a 5700
72016f3a
AK
5701 /*
5702 * Assume that the pte write on a page table of the same type
49b26e26
XG
5703 * as the current vcpu paging mode since we update the sptes only
5704 * when they have the same mode.
72016f3a 5705 */
889e5cbc 5706 if (is_pae(vcpu) && *bytes == 4) {
72016f3a 5707 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
889e5cbc
XG
5708 *gpa &= ~(gpa_t)7;
5709 *bytes = 8;
08e850c6
AK
5710 }
5711
0e0fee5c
JS
5712 if (*bytes == 4 || *bytes == 8) {
5713 r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
5714 if (r)
5715 gentry = 0;
72016f3a
AK
5716 }
5717
889e5cbc
XG
5718 return gentry;
5719}
5720
5721/*
5722 * If we're seeing too many writes to a page, it may no longer be a page table,
5723 * or we may be forking, in which case it is better to unmap the page.
5724 */
a138fe75 5725static bool detect_write_flooding(struct kvm_mmu_page *sp)
889e5cbc 5726{
a30f47cb
XG
5727 /*
5728 * Skip write-flooding detected for the sp whose level is 1, because
5729 * it can become unsync, then the guest page is not write-protected.
5730 */
3bae0459 5731 if (sp->role.level == PG_LEVEL_4K)
a30f47cb 5732 return false;
3246af0e 5733
e5691a81
XG
5734 atomic_inc(&sp->write_flooding_count);
5735 return atomic_read(&sp->write_flooding_count) >= 3;
889e5cbc
XG
5736}
5737
5738/*
5739 * Misaligned accesses are too much trouble to fix up; also, they usually
5740 * indicate a page is not used as a page table.
5741 */
5742static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
5743 int bytes)
5744{
5745 unsigned offset, pte_size, misaligned;
5746
889e5cbc 5747 offset = offset_in_page(gpa);
bb3b394d 5748 pte_size = sp->role.has_4_byte_gpte ? 4 : 8;
5d9ca30e
XG
5749
5750 /*
5751 * Sometimes, the OS only writes the last one bytes to update status
5752 * bits, for example, in linux, andb instruction is used in clear_bit().
5753 */
5754 if (!(offset & (pte_size - 1)) && bytes == 1)
5755 return false;
5756
889e5cbc
XG
5757 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
5758 misaligned |= bytes < 4;
5759
5760 return misaligned;
5761}
5762
5763static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
5764{
5765 unsigned page_offset, quadrant;
5766 u64 *spte;
5767 int level;
5768
5769 page_offset = offset_in_page(gpa);
5770 level = sp->role.level;
5771 *nspte = 1;
bb3b394d 5772 if (sp->role.has_4_byte_gpte) {
889e5cbc
XG
5773 page_offset <<= 1; /* 32->64 */
5774 /*
5775 * A 32-bit pde maps 4MB while the shadow pdes map
5776 * only 2MB. So we need to double the offset again
5777 * and zap two pdes instead of one.
5778 */
5779 if (level == PT32_ROOT_LEVEL) {
5780 page_offset &= ~7; /* kill rounding error */
5781 page_offset <<= 1;
5782 *nspte = 2;
5783 }
5784 quadrant = page_offset >> PAGE_SHIFT;
5785 page_offset &= ~PAGE_MASK;
5786 if (quadrant != sp->role.quadrant)
5787 return NULL;
5788 }
5789
5790 spte = &sp->spt[page_offset / sizeof(*spte)];
5791 return spte;
5792}
5793
93284446
SC
5794void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
5795 int bytes)
889e5cbc
XG
5796{
5797 gfn_t gfn = gpa >> PAGE_SHIFT;
889e5cbc 5798 struct kvm_mmu_page *sp;
889e5cbc
XG
5799 LIST_HEAD(invalid_list);
5800 u64 entry, gentry, *spte;
5801 int npte;
06152b2d 5802 bool flush = false;
889e5cbc
XG
5803
5804 /*
5805 * If we don't have indirect shadow pages, it means no page is
5806 * write-protected, so we can exit simply.
5807 */
6aa7de05 5808 if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
889e5cbc
XG
5809 return;
5810
531810ca 5811 write_lock(&vcpu->kvm->mmu_lock);
0e0fee5c
JS
5812
5813 gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);
5814
889e5cbc 5815 ++vcpu->kvm->stat.mmu_pte_write;
889e5cbc 5816
767d8d8d 5817 for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) {
a30f47cb 5818 if (detect_write_misaligned(sp, gpa, bytes) ||
a138fe75 5819 detect_write_flooding(sp)) {
b8c67b7a 5820 kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
4cee5764 5821 ++vcpu->kvm->stat.mmu_flooded;
0e7bc4b9
AK
5822 continue;
5823 }
889e5cbc
XG
5824
5825 spte = get_written_sptes(sp, gpa, &npte);
5826 if (!spte)
5827 continue;
5828
ac1b714e 5829 while (npte--) {
79539cec 5830 entry = *spte;
2de4085c 5831 mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
c5e2184d
SC
5832 if (gentry && sp->role.level != PG_LEVEL_4K)
5833 ++vcpu->kvm->stat.mmu_pde_zapped;
1441ca14 5834 if (is_shadow_present_pte(entry))
06152b2d 5835 flush = true;
ac1b714e 5836 ++spte;
9b7a0325 5837 }
9b7a0325 5838 }
06152b2d 5839 kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
531810ca 5840 write_unlock(&vcpu->kvm->mmu_lock);
da4a00f0
AK
5841}
5842
1075d41e 5843int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
dc25e89e 5844 void *insn, int insn_len)
3067714c 5845{
92daa48b 5846 int r, emulation_type = EMULTYPE_PF;
347a0d0d 5847 bool direct = vcpu->arch.mmu->root_role.direct;
3067714c 5848
d09f7112
SC
5849 /*
5850 * IMPLICIT_ACCESS is a KVM-defined flag used to correctly perform SMAP
5851 * checks when emulating instructions that triggers implicit access.
5852 * WARN if hardware generates a fault with an error code that collides
5853 * with the KVM-defined value. Clear the flag and continue on, i.e.
5854 * don't terminate the VM, as KVM can't possibly be relying on a flag
5855 * that KVM doesn't know about.
5856 */
5857 if (WARN_ON_ONCE(error_code & PFERR_IMPLICIT_ACCESS))
5858 error_code &= ~PFERR_IMPLICIT_ACCESS;
5859
20ba462d 5860 if (WARN_ON_ONCE(!VALID_PAGE(vcpu->arch.mmu->root.hpa)))
ddce6208
SC
5861 return RET_PF_RETRY;
5862
9b8ebbdb 5863 r = RET_PF_INVALID;
e9ee956e 5864 if (unlikely(error_code & PFERR_RSVD_MASK)) {
736c291c 5865 r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
472faffa 5866 if (r == RET_PF_EMULATE)
e9ee956e 5867 goto emulate;
e9ee956e 5868 }
3067714c 5869
9b8ebbdb 5870 if (r == RET_PF_INVALID) {
7a02674d 5871 r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
258d985f
SC
5872 lower_32_bits(error_code), false,
5873 &emulation_type);
19025e7b 5874 if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
7b367bc9 5875 return -EIO;
9b8ebbdb
PB
5876 }
5877
3067714c 5878 if (r < 0)
e9ee956e 5879 return r;
83a2ba4c
SC
5880 if (r != RET_PF_EMULATE)
5881 return 1;
3067714c 5882
14727754
TL
5883 /*
5884 * Before emulating the instruction, check if the error code
5885 * was due to a RO violation while translating the guest page.
5886 * This can occur when using nested virtualization with nested
5887 * paging in both guests. If true, we simply unprotect the page
5888 * and resume the guest.
14727754 5889 */
347a0d0d 5890 if (vcpu->arch.mmu->root_role.direct &&
eebed243 5891 (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
736c291c 5892 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa));
14727754
TL
5893 return 1;
5894 }
5895
472faffa
SC
5896 /*
5897 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
5898 * optimistically try to just unprotect the page and let the processor
5899 * re-execute the instruction that caused the page fault. Do not allow
5900 * retrying MMIO emulation, as it's not only pointless but could also
5901 * cause us to enter an infinite loop because the processor will keep
6c3dfeb6
SC
5902 * faulting on the non-existent MMIO address. Retrying an instruction
5903 * from a nested guest is also pointless and dangerous as we are only
5904 * explicitly shadowing L1's page tables, i.e. unprotecting something
5905 * for L1 isn't going to magically fix whatever issue cause L2 to fail.
472faffa 5906 */
736c291c 5907 if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu))
92daa48b 5908 emulation_type |= EMULTYPE_ALLOW_RETRY_PF;
e9ee956e 5909emulate:
736c291c 5910 return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
60fc3d02 5911 insn_len);
3067714c
AK
5912}
5913EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
5914
9fd4a4e3
LJ
5915static void __kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
5916 u64 addr, hpa_t root_hpa)
5917{
5918 struct kvm_shadow_walk_iterator iterator;
5919
5920 vcpu_clear_mmio_info(vcpu, addr);
5921
762b33eb
LX
5922 /*
5923 * Walking and synchronizing SPTEs both assume they are operating in
5924 * the context of the current MMU, and would need to be reworked if
5925 * this is ever used to sync the guest_mmu, e.g. to emulate INVEPT.
5926 */
5927 if (WARN_ON_ONCE(mmu != vcpu->arch.mmu))
5928 return;
5929
9fd4a4e3
LJ
5930 if (!VALID_PAGE(root_hpa))
5931 return;
5932
5933 write_lock(&vcpu->kvm->mmu_lock);
5934 for_each_shadow_entry_using_root(vcpu, root_hpa, addr, iterator) {
5935 struct kvm_mmu_page *sp = sptep_to_sp(iterator.sptep);
5936
5937 if (sp->unsync) {
19ace7d6 5938 int ret = kvm_sync_spte(vcpu, sp, iterator.index);
9fd4a4e3
LJ
5939
5940 if (ret < 0)
5941 mmu_page_zap_pte(vcpu->kvm, sp, iterator.sptep, NULL);
5942 if (ret)
5943 kvm_flush_remote_tlbs_sptep(vcpu->kvm, iterator.sptep);
5944 }
5945
5946 if (!sp->unsync_children)
5947 break;
5948 }
5949 write_unlock(&vcpu->kvm->mmu_lock);
5950}
5951
753b43c9 5952void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
cd42853e 5953 u64 addr, unsigned long roots)
a7052897 5954{
b94742c9 5955 int i;
7eb77e9f 5956
cd42853e
LJ
5957 WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
5958
5efac074
PB
5959 /* It's actually a GPA for vcpu->arch.guest_mmu. */
5960 if (mmu != &vcpu->arch.guest_mmu) {
5961 /* INVLPG on a non-canonical address is a NOP according to the SDM. */
753b43c9 5962 if (is_noncanonical_address(addr, vcpu))
5efac074
PB
5963 return;
5964
753b43c9 5965 static_call(kvm_x86_flush_tlb_gva)(vcpu, addr);
5efac074
PB
5966 }
5967
9fd4a4e3 5968 if (!mmu->sync_spte)
faff8758
JS
5969 return;
5970
cd42853e 5971 if (roots & KVM_MMU_ROOT_CURRENT)
9fd4a4e3 5972 __kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->root.hpa);
956bf353 5973
cd42853e 5974 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
ed335278 5975 if (roots & KVM_MMU_ROOT_PREVIOUS(i))
9fd4a4e3 5976 __kvm_mmu_invalidate_addr(vcpu, mmu, addr, mmu->prev_roots[i].hpa);
5efac074
PB
5977 }
5978}
2c86c444 5979EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_addr);
956bf353 5980
5efac074
PB
5981void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
5982{
cd42853e
LJ
5983 /*
5984 * INVLPG is required to invalidate any global mappings for the VA,
5985 * irrespective of PCID. Blindly sync all roots as it would take
5986 * roughly the same amount of work/time to determine whether any of the
5987 * previous roots have a global mapping.
5988 *
5989 * Mappings not reachable via the current or previous cached roots will
5990 * be synced when switching to that new cr3, so nothing needs to be
5991 * done here for them.
5992 */
5993 kvm_mmu_invalidate_addr(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);
a7052897
MT
5994 ++vcpu->stat.invlpg;
5995}
5996EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
5997
5efac074 5998
eb4b248e
JS
5999void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
6000{
44dd3ffa 6001 struct kvm_mmu *mmu = vcpu->arch.mmu;
9ebc3f51 6002 unsigned long roots = 0;
b94742c9 6003 uint i;
eb4b248e 6004
9ebc3f51
LJ
6005 if (pcid == kvm_get_active_pcid(vcpu))
6006 roots |= KVM_MMU_ROOT_CURRENT;
eb4b248e 6007
b94742c9
JS
6008 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
6009 if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
9ebc3f51
LJ
6010 pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd))
6011 roots |= KVM_MMU_ROOT_PREVIOUS(i);
956bf353 6012 }
ade61e28 6013
9ebc3f51
LJ
6014 if (roots)
6015 kvm_mmu_invalidate_addr(vcpu, mmu, gva, roots);
eb4b248e
JS
6016 ++vcpu->stat.invlpg;
6017
6018 /*
b94742c9
JS
6019 * Mappings not reachable via the current cr3 or the prev_roots will be
6020 * synced when switching to that cr3, so nothing needs to be done here
6021 * for them.
eb4b248e
JS
6022 */
6023}
eb4b248e 6024
746700d2
WH
6025void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
6026 int tdp_max_root_level, int tdp_huge_page_level)
18552672 6027{
bde77235 6028 tdp_enabled = enable_tdp;
746700d2 6029 tdp_root_level = tdp_forced_root_level;
83013059 6030 max_tdp_level = tdp_max_root_level;
703c335d 6031
1f98f2bd
DM
6032#ifdef CONFIG_X86_64
6033 tdp_mmu_enabled = tdp_mmu_allowed && tdp_enabled;
6034#endif
703c335d 6035 /*
1d92d2e8 6036 * max_huge_page_level reflects KVM's MMU capabilities irrespective
703c335d
SC
6037 * of kernel support, e.g. KVM may be capable of using 1GB pages when
6038 * the kernel is not. But, KVM never creates a page size greater than
6039 * what is used by the kernel for any given HVA, i.e. the kernel's
6040 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust().
6041 */
6042 if (tdp_enabled)
1d92d2e8 6043 max_huge_page_level = tdp_huge_page_level;
703c335d 6044 else if (boot_cpu_has(X86_FEATURE_GBPAGES))
1d92d2e8 6045 max_huge_page_level = PG_LEVEL_1G;
703c335d 6046 else
1d92d2e8 6047 max_huge_page_level = PG_LEVEL_2M;
18552672 6048}
bde77235 6049EXPORT_SYMBOL_GPL(kvm_configure_mmu);
85875a13
SC
6050
6051/* The return value indicates if tlb flush on all vcpus is needed. */
727ae377 6052typedef bool (*slot_rmaps_handler) (struct kvm *kvm,
269e9552
HM
6053 struct kvm_rmap_head *rmap_head,
6054 const struct kvm_memory_slot *slot);
85875a13 6055
727ae377
SC
6056static __always_inline bool __walk_slot_rmaps(struct kvm *kvm,
6057 const struct kvm_memory_slot *slot,
6058 slot_rmaps_handler fn,
6059 int start_level, int end_level,
6060 gfn_t start_gfn, gfn_t end_gfn,
6061 bool flush_on_yield, bool flush)
85875a13
SC
6062{
6063 struct slot_rmap_walk_iterator iterator;
85875a13 6064
eddd9e83
SC
6065 lockdep_assert_held_write(&kvm->mmu_lock);
6066
727ae377 6067 for_each_slot_rmap_range(slot, start_level, end_level, start_gfn,
85875a13
SC
6068 end_gfn, &iterator) {
6069 if (iterator.rmap)
727ae377 6070 flush |= fn(kvm, iterator.rmap, slot);
85875a13 6071
531810ca 6072 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
302695a5 6073 if (flush && flush_on_yield) {
8c63e8c2
DM
6074 kvm_flush_remote_tlbs_range(kvm, start_gfn,
6075 iterator.gfn - start_gfn + 1);
85875a13
SC
6076 flush = false;
6077 }
531810ca 6078 cond_resched_rwlock_write(&kvm->mmu_lock);
85875a13
SC
6079 }
6080 }
6081
85875a13
SC
6082 return flush;
6083}
6084
727ae377
SC
6085static __always_inline bool walk_slot_rmaps(struct kvm *kvm,
6086 const struct kvm_memory_slot *slot,
6087 slot_rmaps_handler fn,
6088 int start_level, int end_level,
6089 bool flush_on_yield)
85875a13 6090{
727ae377
SC
6091 return __walk_slot_rmaps(kvm, slot, fn, start_level, end_level,
6092 slot->base_gfn, slot->base_gfn + slot->npages - 1,
6093 flush_on_yield, false);
85875a13
SC
6094}
6095
727ae377
SC
6096static __always_inline bool walk_slot_rmaps_4k(struct kvm *kvm,
6097 const struct kvm_memory_slot *slot,
6098 slot_rmaps_handler fn,
6099 bool flush_on_yield)
85875a13 6100{
727ae377 6101 return walk_slot_rmaps(kvm, slot, fn, PG_LEVEL_4K, PG_LEVEL_4K, flush_on_yield);
85875a13
SC
6102}
6103
1cfff4d9 6104static void free_mmu_pages(struct kvm_mmu *mmu)
6aa8b732 6105{
4a98623d
SC
6106 if (!tdp_enabled && mmu->pae_root)
6107 set_memory_encrypted((unsigned long)mmu->pae_root, 1);
1cfff4d9 6108 free_page((unsigned long)mmu->pae_root);
03ca4589 6109 free_page((unsigned long)mmu->pml4_root);
cb0f722a 6110 free_page((unsigned long)mmu->pml5_root);
6aa8b732
AK
6111}
6112
04d28e37 6113static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
6aa8b732 6114{
17ac10ad 6115 struct page *page;
6aa8b732
AK
6116 int i;
6117
b9e5603c
PB
6118 mmu->root.hpa = INVALID_PAGE;
6119 mmu->root.pgd = 0;
04d28e37
SC
6120 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
6121 mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
6122
27f4fca2
LJ
6123 /* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */
6124 if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)
6125 return 0;
6126
17ac10ad 6127 /*
b6b80c78
SC
6128 * When using PAE paging, the four PDPTEs are treated as 'root' pages,
6129 * while the PDP table is a per-vCPU construct that's allocated at MMU
6130 * creation. When emulating 32-bit mode, cr3 is only 32 bits even on
6131 * x86_64. Therefore we need to allocate the PDP table in the first
04d45551
SC
6132 * 4GB of memory, which happens to fit the DMA32 zone. TDP paging
6133 * generally doesn't use PAE paging and can skip allocating the PDP
6134 * table. The main exception, handled here, is SVM's 32-bit NPT. The
6135 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
84432316 6136 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().
17ac10ad 6137 */
d468d94b 6138 if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
b6b80c78
SC
6139 return 0;
6140
254272ce 6141 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
17ac10ad 6142 if (!page)
d7fa6ab2
WY
6143 return -ENOMEM;
6144
1cfff4d9 6145 mmu->pae_root = page_address(page);
4a98623d
SC
6146
6147 /*
6148 * CR3 is only 32 bits when PAE paging is used, thus it's impossible to
6149 * get the CPU to treat the PDPTEs as encrypted. Decrypt the page so
6150 * that KVM's writes and the CPU's reads get along. Note, this is
6151 * only necessary when using shadow paging, as 64-bit NPT can get at
6152 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported
6153 * by 32-bit kernels (when KVM itself uses 32-bit NPT).
6154 */
6155 if (!tdp_enabled)
6156 set_memory_decrypted((unsigned long)mmu->pae_root, 1);
6157 else
e54f1ff2 6158 WARN_ON_ONCE(shadow_me_value);
4a98623d 6159
17ac10ad 6160 for (i = 0; i < 4; ++i)
c834e5e4 6161 mmu->pae_root[i] = INVALID_PAE_ROOT;
17ac10ad 6162
6aa8b732 6163 return 0;
6aa8b732
AK
6164}
6165
8018c27b 6166int kvm_mmu_create(struct kvm_vcpu *vcpu)
6aa8b732 6167{
1cfff4d9 6168 int ret;
b94742c9 6169
5962bfb7 6170 vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;
5f6078f9
SC
6171 vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;
6172
5962bfb7 6173 vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;
5f6078f9 6174 vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;
5962bfb7 6175
96880883
SC
6176 vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
6177
44dd3ffa
VK
6178 vcpu->arch.mmu = &vcpu->arch.root_mmu;
6179 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
6aa8b732 6180
04d28e37 6181 ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
1cfff4d9
JP
6182 if (ret)
6183 return ret;
6184
04d28e37 6185 ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
1cfff4d9
JP
6186 if (ret)
6187 goto fail_allocate_root;
6188
6189 return ret;
6190 fail_allocate_root:
6191 free_mmu_pages(&vcpu->arch.guest_mmu);
6192 return ret;
6aa8b732
AK
6193}
6194
fbb158cb 6195#define BATCH_ZAP_PAGES 10
002c5f73
SC
6196static void kvm_zap_obsolete_pages(struct kvm *kvm)
6197{
6198 struct kvm_mmu_page *sp, *node;
fbb158cb 6199 int nr_zapped, batch = 0;
b28cb0cd 6200 bool unstable;
002c5f73
SC
6201
6202restart:
6203 list_for_each_entry_safe_reverse(sp, node,
6204 &kvm->arch.active_mmu_pages, link) {
6205 /*
6206 * No obsolete valid page exists before a newly created page
6207 * since active_mmu_pages is a FIFO list.
6208 */
6209 if (!is_obsolete_sp(kvm, sp))
6210 break;
6211
6212 /*
f95eec9b
SC
6213 * Invalid pages should never land back on the list of active
6214 * pages. Skip the bogus page, otherwise we'll get stuck in an
6215 * infinite loop if the page gets put back on the list (again).
002c5f73 6216 */
20ba462d 6217 if (WARN_ON_ONCE(sp->role.invalid))
002c5f73
SC
6218 continue;
6219
4506ecf4
SC
6220 /*
6221 * No need to flush the TLB since we're only zapping shadow
6222 * pages with an obsolete generation number and all vCPUS have
6223 * loaded a new root, i.e. the shadow pages being zapped cannot
6224 * be in active use by the guest.
6225 */
fbb158cb 6226 if (batch >= BATCH_ZAP_PAGES &&
531810ca 6227 cond_resched_rwlock_write(&kvm->mmu_lock)) {
fbb158cb 6228 batch = 0;
002c5f73
SC
6229 goto restart;
6230 }
6231
b28cb0cd
SC
6232 unstable = __kvm_mmu_prepare_zap_page(kvm, sp,
6233 &kvm->arch.zapped_obsolete_pages, &nr_zapped);
6234 batch += nr_zapped;
6235
6236 if (unstable)
002c5f73
SC
6237 goto restart;
6238 }
6239
4506ecf4 6240 /*
7ae5840e
SC
6241 * Kick all vCPUs (via remote TLB flush) before freeing the page tables
6242 * to ensure KVM is not in the middle of a lockless shadow page table
6243 * walk, which may reference the pages. The remote TLB flush itself is
6244 * not required and is simply a convenient way to kick vCPUs as needed.
6245 * KVM performs a local TLB flush when allocating a new root (see
6246 * kvm_mmu_load()), and the reload in the caller ensure no vCPUs are
6247 * running with an obsolete MMU.
4506ecf4 6248 */
10605204 6249 kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
002c5f73
SC
6250}
6251
6252/*
6253 * Fast invalidate all shadow pages and use lock-break technique
6254 * to zap obsolete pages.
6255 *
6256 * It's required when memslot is being deleted or VM is being
6257 * destroyed, in these cases, we should ensure that KVM MMU does
6258 * not use any resource of the being-deleted slot or all slots
6259 * after calling the function.
6260 */
6261static void kvm_mmu_zap_all_fast(struct kvm *kvm)
6262{
ca333add
SC
6263 lockdep_assert_held(&kvm->slots_lock);
6264
531810ca 6265 write_lock(&kvm->mmu_lock);
14a3c4f4 6266 trace_kvm_mmu_zap_all_fast(kvm);
ca333add
SC
6267
6268 /*
6269 * Toggle mmu_valid_gen between '0' and '1'. Because slots_lock is
6270 * held for the entire duration of zapping obsolete pages, it's
6271 * impossible for there to be multiple invalid generations associated
6272 * with *valid* shadow pages at any given time, i.e. there is exactly
6273 * one valid generation and (at most) one invalid generation.
6274 */
6275 kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
002c5f73 6276
2f6f66cc
SC
6277 /*
6278 * In order to ensure all vCPUs drop their soon-to-be invalid roots,
6279 * invalidating TDP MMU roots must be done while holding mmu_lock for
6280 * write and in the same critical section as making the reload request,
6281 * e.g. before kvm_zap_obsolete_pages() could drop mmu_lock and yield.
b7cccd39 6282 */
1f98f2bd 6283 if (tdp_mmu_enabled)
b7cccd39
BG
6284 kvm_tdp_mmu_invalidate_all_roots(kvm);
6285
4506ecf4
SC
6286 /*
6287 * Notify all vcpus to reload its shadow page table and flush TLB.
6288 * Then all vcpus will switch to new shadow page table with the new
6289 * mmu_valid_gen.
6290 *
6291 * Note: we need to do this under the protection of mmu_lock,
6292 * otherwise, vcpu would purge shadow page but miss tlb flush.
6293 */
527d5cd7 6294 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
4506ecf4 6295
002c5f73 6296 kvm_zap_obsolete_pages(kvm);
faaf05b0 6297
531810ca 6298 write_unlock(&kvm->mmu_lock);
4c6654bd 6299
f28e9c7f
SC
6300 /*
6301 * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
6302 * returning to the caller, e.g. if the zap is in response to a memslot
6303 * deletion, mmu_notifier callbacks will be unable to reach the SPTEs
6304 * associated with the deleted memslot once the update completes, and
6305 * Deferring the zap until the final reference to the root is put would
6306 * lead to use-after-free.
6307 */
1f98f2bd 6308 if (tdp_mmu_enabled)
4c6654bd 6309 kvm_tdp_mmu_zap_invalidated_roots(kvm);
002c5f73
SC
6310}
6311
10605204
SC
6312static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
6313{
6314 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
6315}
6316
0df9dab8 6317void kvm_mmu_init_vm(struct kvm *kvm)
1bad2b2a 6318{
a1a39128
PB
6319 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
6320 INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
55c510e2 6321 INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages);
ce25681d
SC
6322 spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
6323
0df9dab8
SC
6324 if (tdp_mmu_enabled)
6325 kvm_mmu_init_tdp_mmu(kvm);
fe5db27d 6326
ada51a9d
DM
6327 kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;
6328 kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;
6329
6330 kvm->arch.split_shadow_page_cache.gfp_zero = __GFP_ZERO;
6331
6332 kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;
6333 kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;
1bad2b2a
XG
6334}
6335
ada51a9d
DM
6336static void mmu_free_vm_memory_caches(struct kvm *kvm)
6337{
6338 kvm_mmu_free_memory_cache(&kvm->arch.split_desc_cache);
6339 kvm_mmu_free_memory_cache(&kvm->arch.split_page_header_cache);
6340 kvm_mmu_free_memory_cache(&kvm->arch.split_shadow_page_cache);
6341}
6342
13d268ca 6343void kvm_mmu_uninit_vm(struct kvm *kvm)
1bad2b2a 6344{
09732d2b
DM
6345 if (tdp_mmu_enabled)
6346 kvm_mmu_uninit_tdp_mmu(kvm);
ada51a9d
DM
6347
6348 mmu_free_vm_memory_caches(kvm);
1bad2b2a
XG
6349}
6350
2833eda0 6351static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
21fa3246
SC
6352{
6353 const struct kvm_memory_slot *memslot;
6354 struct kvm_memslots *slots;
f4209439 6355 struct kvm_memslot_iter iter;
21fa3246
SC
6356 bool flush = false;
6357 gfn_t start, end;
f4209439 6358 int i;
21fa3246
SC
6359
6360 if (!kvm_memslots_have_rmaps(kvm))
6361 return flush;
6362
eed52e43 6363 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
21fa3246 6364 slots = __kvm_memslots(kvm, i);
f4209439
MS
6365
6366 kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {
6367 memslot = iter.slot;
21fa3246
SC
6368 start = max(gfn_start, memslot->base_gfn);
6369 end = min(gfn_end, memslot->base_gfn + memslot->npages);
f4209439 6370 if (WARN_ON_ONCE(start >= end))
21fa3246
SC
6371 continue;
6372
727ae377
SC
6373 flush = __walk_slot_rmaps(kvm, memslot, __kvm_zap_rmap,
6374 PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
6375 start, end - 1, true, flush);
21fa3246
SC
6376 }
6377 }
6378
6379 return flush;
6380}
6381
88f58535
ML
6382/*
6383 * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end
6384 * (not including it)
6385 */
efdfe536
XG
6386void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
6387{
21fa3246 6388 bool flush;
efdfe536 6389
f4209439
MS
6390 if (WARN_ON_ONCE(gfn_end <= gfn_start))
6391 return;
6392
5a324c24
SC
6393 write_lock(&kvm->mmu_lock);
6394
8569992d
CP
6395 kvm_mmu_invalidate_begin(kvm);
6396
6397 kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
edb298c6 6398
2833eda0 6399 flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
efdfe536 6400
441a5dfc
PB
6401 if (tdp_mmu_enabled)
6402 flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
5a324c24
SC
6403
6404 if (flush)
8c63e8c2 6405 kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
5a324c24 6406
8569992d 6407 kvm_mmu_invalidate_end(kvm);
edb298c6 6408
5a324c24 6409 write_unlock(&kvm->mmu_lock);
efdfe536
XG
6410}
6411
018aabb5 6412static bool slot_rmap_write_protect(struct kvm *kvm,
0a234f5d 6413 struct kvm_rmap_head *rmap_head,
269e9552 6414 const struct kvm_memory_slot *slot)
d77aa73c 6415{
1346bbb6 6416 return rmap_write_protect(rmap_head, false);
d77aa73c
XG
6417}
6418
1c91cad4 6419void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
269e9552 6420 const struct kvm_memory_slot *memslot,
3c9bd400 6421 int start_level)
6aa8b732 6422{
e2209710
BG
6423 if (kvm_memslots_have_rmaps(kvm)) {
6424 write_lock(&kvm->mmu_lock);
727ae377
SC
6425 walk_slot_rmaps(kvm, memslot, slot_rmap_write_protect,
6426 start_level, KVM_MAX_HUGEPAGE_LEVEL, false);
e2209710
BG
6427 write_unlock(&kvm->mmu_lock);
6428 }
198c74f4 6429
1f98f2bd 6430 if (tdp_mmu_enabled) {
24ae4cfa 6431 read_lock(&kvm->mmu_lock);
b64d740e 6432 kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
24ae4cfa
BG
6433 read_unlock(&kvm->mmu_lock);
6434 }
6aa8b732 6435}
37a7d8b0 6436
ada51a9d
DM
6437static inline bool need_topup(struct kvm_mmu_memory_cache *cache, int min)
6438{
6439 return kvm_mmu_memory_cache_nr_free_objects(cache) < min;
6440}
6441
6442static bool need_topup_split_caches_or_resched(struct kvm *kvm)
6443{
6444 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock))
6445 return true;
6446
6447 /*
6448 * In the worst case, SPLIT_DESC_CACHE_MIN_NR_OBJECTS descriptors are needed
6449 * to split a single huge page. Calculating how many are actually needed
6450 * is possible but not worth the complexity.
6451 */
6452 return need_topup(&kvm->arch.split_desc_cache, SPLIT_DESC_CACHE_MIN_NR_OBJECTS) ||
6453 need_topup(&kvm->arch.split_page_header_cache, 1) ||
6454 need_topup(&kvm->arch.split_shadow_page_cache, 1);
6455}
6456
6457static int topup_split_caches(struct kvm *kvm)
6458{
b9b71f43
SC
6459 /*
6460 * Allocating rmap list entries when splitting huge pages for nested
dfd4eb44 6461 * MMUs is uncommon as KVM needs to use a list if and only if there is
b9b71f43 6462 * more than one rmap entry for a gfn, i.e. requires an L1 gfn to be
dfd4eb44
SC
6463 * aliased by multiple L2 gfns and/or from multiple nested roots with
6464 * different roles. Aliasing gfns when using TDP is atypical for VMMs;
6465 * a few gfns are often aliased during boot, e.g. when remapping BIOS,
6466 * but aliasing rarely occurs post-boot or for many gfns. If there is
6467 * only one rmap entry, rmap->val points directly at that one entry and
6468 * doesn't need to allocate a list. Buffer the cache by the default
6469 * capacity so that KVM doesn't have to drop mmu_lock to topup if KVM
b9b71f43
SC
6470 * encounters an aliased gfn or two.
6471 */
6472 const int capacity = SPLIT_DESC_CACHE_MIN_NR_OBJECTS +
6473 KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE;
ada51a9d
DM
6474 int r;
6475
6476 lockdep_assert_held(&kvm->slots_lock);
6477
b9b71f43 6478 r = __kvm_mmu_topup_memory_cache(&kvm->arch.split_desc_cache, capacity,
ada51a9d
DM
6479 SPLIT_DESC_CACHE_MIN_NR_OBJECTS);
6480 if (r)
6481 return r;
6482
6483 r = kvm_mmu_topup_memory_cache(&kvm->arch.split_page_header_cache, 1);
6484 if (r)
6485 return r;
6486
6487 return kvm_mmu_topup_memory_cache(&kvm->arch.split_shadow_page_cache, 1);
6488}
6489
6490static struct kvm_mmu_page *shadow_mmu_get_sp_for_split(struct kvm *kvm, u64 *huge_sptep)
6491{
6492 struct kvm_mmu_page *huge_sp = sptep_to_sp(huge_sptep);
6493 struct shadow_page_caches caches = {};
6494 union kvm_mmu_page_role role;
6495 unsigned int access;
6496 gfn_t gfn;
6497
79e48cec
SC
6498 gfn = kvm_mmu_page_get_gfn(huge_sp, spte_index(huge_sptep));
6499 access = kvm_mmu_page_get_access(huge_sp, spte_index(huge_sptep));
ada51a9d
DM
6500
6501 /*
6502 * Note, huge page splitting always uses direct shadow pages, regardless
6503 * of whether the huge page itself is mapped by a direct or indirect
6504 * shadow page, since the huge page region itself is being directly
6505 * mapped with smaller pages.
6506 */
6507 role = kvm_mmu_child_role(huge_sptep, /*direct=*/true, access);
6508
6509 /* Direct SPs do not require a shadowed_info_cache. */
6510 caches.page_header_cache = &kvm->arch.split_page_header_cache;
6511 caches.shadow_page_cache = &kvm->arch.split_shadow_page_cache;
6512
6513 /* Safe to pass NULL for vCPU since requesting a direct SP. */
6514 return __kvm_mmu_get_shadow_page(kvm, NULL, &caches, gfn, role);
6515}
6516
6517static void shadow_mmu_split_huge_page(struct kvm *kvm,
6518 const struct kvm_memory_slot *slot,
6519 u64 *huge_sptep)
6520
6521{
6522 struct kvm_mmu_memory_cache *cache = &kvm->arch.split_desc_cache;
6523 u64 huge_spte = READ_ONCE(*huge_sptep);
6524 struct kvm_mmu_page *sp;
03787394 6525 bool flush = false;
ada51a9d
DM
6526 u64 *sptep, spte;
6527 gfn_t gfn;
6528 int index;
6529
6530 sp = shadow_mmu_get_sp_for_split(kvm, huge_sptep);
6531
6532 for (index = 0; index < SPTE_ENT_PER_PAGE; index++) {
6533 sptep = &sp->spt[index];
6534 gfn = kvm_mmu_page_get_gfn(sp, index);
6535
6536 /*
6537 * The SP may already have populated SPTEs, e.g. if this huge
6538 * page is aliased by multiple sptes with the same access
6539 * permissions. These entries are guaranteed to map the same
6540 * gfn-to-pfn translation since the SP is direct, so no need to
6541 * modify them.
6542 *
03787394
PB
6543 * However, if a given SPTE points to a lower level page table,
6544 * that lower level page table may only be partially populated.
6545 * Installing such SPTEs would effectively unmap a potion of the
6546 * huge page. Unmapping guest memory always requires a TLB flush
6547 * since a subsequent operation on the unmapped regions would
6548 * fail to detect the need to flush.
ada51a9d 6549 */
03787394
PB
6550 if (is_shadow_present_pte(*sptep)) {
6551 flush |= !is_last_spte(*sptep, sp->role.level);
ada51a9d 6552 continue;
03787394 6553 }
ada51a9d
DM
6554
6555 spte = make_huge_page_split_spte(kvm, huge_spte, sp->role, index);
6556 mmu_spte_set(sptep, spte);
6557 __rmap_add(kvm, cache, slot, sptep, gfn, sp->role.access);
6558 }
6559
03787394 6560 __link_shadow_page(kvm, cache, huge_sptep, sp, flush);
ada51a9d
DM
6561}
6562
6563static int shadow_mmu_try_split_huge_page(struct kvm *kvm,
6564 const struct kvm_memory_slot *slot,
6565 u64 *huge_sptep)
6566{
6567 struct kvm_mmu_page *huge_sp = sptep_to_sp(huge_sptep);
6568 int level, r = 0;
6569 gfn_t gfn;
6570 u64 spte;
6571
6572 /* Grab information for the tracepoint before dropping the MMU lock. */
79e48cec 6573 gfn = kvm_mmu_page_get_gfn(huge_sp, spte_index(huge_sptep));
ada51a9d
DM
6574 level = huge_sp->role.level;
6575 spte = *huge_sptep;
6576
6577 if (kvm_mmu_available_pages(kvm) <= KVM_MIN_FREE_MMU_PAGES) {
6578 r = -ENOSPC;
6579 goto out;
6580 }
6581
6582 if (need_topup_split_caches_or_resched(kvm)) {
6583 write_unlock(&kvm->mmu_lock);
6584 cond_resched();
6585 /*
6586 * If the topup succeeds, return -EAGAIN to indicate that the
6587 * rmap iterator should be restarted because the MMU lock was
6588 * dropped.
6589 */
6590 r = topup_split_caches(kvm) ?: -EAGAIN;
6591 write_lock(&kvm->mmu_lock);
6592 goto out;
6593 }
6594
6595 shadow_mmu_split_huge_page(kvm, slot, huge_sptep);
6596
6597out:
6598 trace_kvm_mmu_split_huge_page(gfn, spte, level, r);
6599 return r;
6600}
6601
6602static bool shadow_mmu_try_split_huge_pages(struct kvm *kvm,
6603 struct kvm_rmap_head *rmap_head,
6604 const struct kvm_memory_slot *slot)
6605{
6606 struct rmap_iterator iter;
6607 struct kvm_mmu_page *sp;
6608 u64 *huge_sptep;
6609 int r;
6610
6611restart:
6612 for_each_rmap_spte(rmap_head, &iter, huge_sptep) {
6613 sp = sptep_to_sp(huge_sptep);
6614
6615 /* TDP MMU is enabled, so rmap only contains nested MMU SPs. */
6616 if (WARN_ON_ONCE(!sp->role.guest_mode))
6617 continue;
6618
6619 /* The rmaps should never contain non-leaf SPTEs. */
6620 if (WARN_ON_ONCE(!is_large_pte(*huge_sptep)))
6621 continue;
6622
6623 /* SPs with level >PG_LEVEL_4K should never by unsync. */
6624 if (WARN_ON_ONCE(sp->unsync))
6625 continue;
6626
6627 /* Don't bother splitting huge pages on invalid SPs. */
6628 if (sp->role.invalid)
6629 continue;
6630
6631 r = shadow_mmu_try_split_huge_page(kvm, slot, huge_sptep);
6632
6633 /*
6634 * The split succeeded or needs to be retried because the MMU
6635 * lock was dropped. Either way, restart the iterator to get it
6636 * back into a consistent state.
6637 */
6638 if (!r || r == -EAGAIN)
6639 goto restart;
6640
6641 /* The split failed and shouldn't be retried (e.g. -ENOMEM). */
6642 break;
6643 }
6644
6645 return false;
6646}
6647
6648static void kvm_shadow_mmu_try_split_huge_pages(struct kvm *kvm,
6649 const struct kvm_memory_slot *slot,
6650 gfn_t start, gfn_t end,
6651 int target_level)
6652{
6653 int level;
6654
6655 /*
6656 * Split huge pages starting with KVM_MAX_HUGEPAGE_LEVEL and working
6657 * down to the target level. This ensures pages are recursively split
6658 * all the way to the target level. There's no need to split pages
6659 * already at the target level.
6660 */
727ae377
SC
6661 for (level = KVM_MAX_HUGEPAGE_LEVEL; level > target_level; level--)
6662 __walk_slot_rmaps(kvm, slot, shadow_mmu_try_split_huge_pages,
6663 level, level, start, end - 1, true, false);
ada51a9d
DM
6664}
6665
cb00a70b
DM
6666/* Must be called with the mmu_lock held in write-mode. */
6667void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
6668 const struct kvm_memory_slot *memslot,
6669 u64 start, u64 end,
6670 int target_level)
6671{
1f98f2bd 6672 if (!tdp_mmu_enabled)
ada51a9d
DM
6673 return;
6674
6675 if (kvm_memslots_have_rmaps(kvm))
6676 kvm_shadow_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level);
6677
6678 kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, false);
cb00a70b
DM
6679
6680 /*
54aa699e 6681 * A TLB flush is unnecessary at this point for the same reasons as in
cb00a70b
DM
6682 * kvm_mmu_slot_try_split_huge_pages().
6683 */
6684}
6685
a3fe5dbd 6686void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
cb00a70b
DM
6687 const struct kvm_memory_slot *memslot,
6688 int target_level)
a3fe5dbd
DM
6689{
6690 u64 start = memslot->base_gfn;
6691 u64 end = start + memslot->npages;
6692
1f98f2bd 6693 if (!tdp_mmu_enabled)
ada51a9d
DM
6694 return;
6695
6696 if (kvm_memslots_have_rmaps(kvm)) {
6697 write_lock(&kvm->mmu_lock);
6698 kvm_shadow_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level);
6699 write_unlock(&kvm->mmu_lock);
a3fe5dbd
DM
6700 }
6701
ada51a9d
DM
6702 read_lock(&kvm->mmu_lock);
6703 kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, true);
6704 read_unlock(&kvm->mmu_lock);
6705
a3fe5dbd
DM
6706 /*
6707 * No TLB flush is necessary here. KVM will flush TLBs after
6708 * write-protecting and/or clearing dirty on the newly split SPTEs to
6709 * ensure that guest writes are reflected in the dirty log before the
6710 * ioctl to enable dirty logging on this memslot completes. Since the
6711 * split SPTEs retain the write and dirty bits of the huge SPTE, it is
6712 * safe for KVM to decide if a TLB flush is necessary based on the split
6713 * SPTEs.
6714 */
6715}
6716
3ea3b7fa 6717static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
0a234f5d 6718 struct kvm_rmap_head *rmap_head,
269e9552 6719 const struct kvm_memory_slot *slot)
3ea3b7fa
WL
6720{
6721 u64 *sptep;
6722 struct rmap_iterator iter;
6723 int need_tlb_flush = 0;
3ea3b7fa
WL
6724 struct kvm_mmu_page *sp;
6725
0d536790 6726restart:
018aabb5 6727 for_each_rmap_spte(rmap_head, &iter, sptep) {
57354682 6728 sp = sptep_to_sp(sptep);
3ea3b7fa
WL
6729
6730 /*
decf6333
XG
6731 * We cannot do huge page mapping for indirect shadow pages,
6732 * which are found on the last rmap (level = 1) when not using
6733 * tdp; such shadow pages are synced with the page table in
6734 * the guest, and the guest page table is using 4K page size
6735 * mapping if the indirect sp has level = 1.
3ea3b7fa 6736 */
5d49f08c 6737 if (sp->role.direct &&
9eba50f8 6738 sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn,
a8ac499b 6739 PG_LEVEL_NUM)) {
9202aee8 6740 kvm_zap_one_rmap_spte(kvm, rmap_head, sptep);
40ef75a7 6741
8a1300ff 6742 if (kvm_available_flush_remote_tlbs_range())
1b2dc736 6743 kvm_flush_remote_tlbs_sptep(kvm, sptep);
40ef75a7
LT
6744 else
6745 need_tlb_flush = 1;
6746
0d536790
XG
6747 goto restart;
6748 }
3ea3b7fa
WL
6749 }
6750
6751 return need_tlb_flush;
6752}
6753
20d49186
DM
6754static void kvm_rmap_zap_collapsible_sptes(struct kvm *kvm,
6755 const struct kvm_memory_slot *slot)
6756{
6757 /*
6758 * Note, use KVM_MAX_HUGEPAGE_LEVEL - 1 since there's no need to zap
6759 * pages that are already mapped at the maximum hugepage level.
6760 */
727ae377
SC
6761 if (walk_slot_rmaps(kvm, slot, kvm_mmu_zap_collapsible_spte,
6762 PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL - 1, true))
619b5072 6763 kvm_flush_remote_tlbs_memslot(kvm, slot);
20d49186
DM
6764}
6765
3ea3b7fa 6766void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
269e9552 6767 const struct kvm_memory_slot *slot)
3ea3b7fa 6768{
e2209710
BG
6769 if (kvm_memslots_have_rmaps(kvm)) {
6770 write_lock(&kvm->mmu_lock);
20d49186 6771 kvm_rmap_zap_collapsible_sptes(kvm, slot);
e2209710
BG
6772 write_unlock(&kvm->mmu_lock);
6773 }
2db6f772 6774
1f98f2bd 6775 if (tdp_mmu_enabled) {
2db6f772 6776 read_lock(&kvm->mmu_lock);
4b85c921 6777 kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot);
2db6f772
BG
6778 read_unlock(&kvm->mmu_lock);
6779 }
3ea3b7fa
WL
6780}
6781
f4b4b180 6782void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
269e9552 6783 const struct kvm_memory_slot *memslot)
f4b4b180 6784{
e2209710
BG
6785 if (kvm_memslots_have_rmaps(kvm)) {
6786 write_lock(&kvm->mmu_lock);
610265ea
DM
6787 /*
6788 * Clear dirty bits only on 4k SPTEs since the legacy MMU only
6789 * support dirty logging at a 4k granularity.
6790 */
727ae377 6791 walk_slot_rmaps_4k(kvm, memslot, __rmap_clear_dirty, false);
e2209710
BG
6792 write_unlock(&kvm->mmu_lock);
6793 }
f4b4b180 6794
1f98f2bd 6795 if (tdp_mmu_enabled) {
24ae4cfa 6796 read_lock(&kvm->mmu_lock);
b64d740e 6797 kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
24ae4cfa
BG
6798 read_unlock(&kvm->mmu_lock);
6799 }
6800
f4b4b180 6801 /*
b64d740e
JS
6802 * The caller will flush the TLBs after this function returns.
6803 *
f4b4b180
KH
6804 * It's also safe to flush TLBs out of mmu lock here as currently this
6805 * function is only used for dirty logging, in which case flushing TLB
6806 * out of mmu lock also guarantees no dirty pages will be lost in
6807 * dirty_bitmap.
6808 */
f4b4b180 6809}
f4b4b180 6810
db0d70e6 6811static void kvm_mmu_zap_all(struct kvm *kvm)
5304b8d3
XG
6812{
6813 struct kvm_mmu_page *sp, *node;
7390de1e 6814 LIST_HEAD(invalid_list);
83cdb568 6815 int ign;
5304b8d3 6816
531810ca 6817 write_lock(&kvm->mmu_lock);
5304b8d3 6818restart:
8a674adc 6819 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
20ba462d 6820 if (WARN_ON_ONCE(sp->role.invalid))
4771450c 6821 continue;
92f58b5c 6822 if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
5304b8d3 6823 goto restart;
531810ca 6824 if (cond_resched_rwlock_write(&kvm->mmu_lock))
5304b8d3
XG
6825 goto restart;
6826 }
6827
4771450c 6828 kvm_mmu_commit_zap_page(kvm, &invalid_list);
faaf05b0 6829
1f98f2bd 6830 if (tdp_mmu_enabled)
faaf05b0
BG
6831 kvm_tdp_mmu_zap_all(kvm);
6832
531810ca 6833 write_unlock(&kvm->mmu_lock);
5304b8d3
XG
6834}
6835
db0d70e6
SC
6836void kvm_arch_flush_shadow_all(struct kvm *kvm)
6837{
6838 kvm_mmu_zap_all(kvm);
6839}
6840
6841void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
6842 struct kvm_memory_slot *slot)
6843{
eeb87272 6844 kvm_mmu_zap_all_fast(kvm);
db0d70e6
SC
6845}
6846
15248258 6847void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
f8f55942 6848{
20ba462d 6849 WARN_ON_ONCE(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
e1359e2b 6850
164bf7e5 6851 gen &= MMIO_SPTE_GEN_MASK;
e1359e2b 6852
f8f55942 6853 /*
e1359e2b
SC
6854 * Generation numbers are incremented in multiples of the number of
6855 * address spaces in order to provide unique generations across all
6856 * address spaces. Strip what is effectively the address space
6857 * modifier prior to checking for a wrap of the MMIO generation so
6858 * that a wrap in any address space is detected.
6859 */
eed52e43 6860 gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1);
e1359e2b 6861
f8f55942 6862 /*
e1359e2b 6863 * The very rare case: if the MMIO generation number has wrapped,
f8f55942 6864 * zap all shadow pages.
f8f55942 6865 */
e1359e2b 6866 if (unlikely(gen == 0)) {
8d20bd63 6867 kvm_debug_ratelimited("zapping shadow pages for mmio generation wraparound\n");
92f58b5c 6868 kvm_mmu_zap_all_fast(kvm);
7a2e8aaf 6869 }
f8f55942
XG
6870}
6871
f3d90f90
SC
6872static unsigned long mmu_shrink_scan(struct shrinker *shrink,
6873 struct shrink_control *sc)
3ee16c81
IE
6874{
6875 struct kvm *kvm;
1495f230 6876 int nr_to_scan = sc->nr_to_scan;
70534a73 6877 unsigned long freed = 0;
3ee16c81 6878
0d9ce162 6879 mutex_lock(&kvm_lock);
3ee16c81
IE
6880
6881 list_for_each_entry(kvm, &vm_list, vm_list) {
3d56cbdf 6882 int idx;
d98ba053 6883 LIST_HEAD(invalid_list);
3ee16c81 6884
35f2d16b
TY
6885 /*
6886 * Never scan more than sc->nr_to_scan VM instances.
6887 * Will not hit this condition practically since we do not try
6888 * to shrink more than one VM and it is very unlikely to see
6889 * !n_used_mmu_pages so many times.
6890 */
6891 if (!nr_to_scan--)
6892 break;
19526396
GN
6893 /*
6894 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
6895 * here. We may skip a VM instance errorneosly, but we do not
6896 * want to shrink a VM that only started to populate its MMU
6897 * anyway.
6898 */
10605204
SC
6899 if (!kvm->arch.n_used_mmu_pages &&
6900 !kvm_has_zapped_obsolete_pages(kvm))
19526396 6901 continue;
19526396 6902
f656ce01 6903 idx = srcu_read_lock(&kvm->srcu);
531810ca 6904 write_lock(&kvm->mmu_lock);
3ee16c81 6905
10605204
SC
6906 if (kvm_has_zapped_obsolete_pages(kvm)) {
6907 kvm_mmu_commit_zap_page(kvm,
6908 &kvm->arch.zapped_obsolete_pages);
6909 goto unlock;
6910 }
6911
ebdb292d 6912 freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
19526396 6913
10605204 6914unlock:
531810ca 6915 write_unlock(&kvm->mmu_lock);
f656ce01 6916 srcu_read_unlock(&kvm->srcu, idx);
19526396 6917
70534a73
DC
6918 /*
6919 * unfair on small ones
6920 * per-vm shrinkers cry out
6921 * sadness comes quickly
6922 */
19526396
GN
6923 list_move_tail(&kvm->vm_list, &vm_list);
6924 break;
3ee16c81 6925 }
3ee16c81 6926
0d9ce162 6927 mutex_unlock(&kvm_lock);
70534a73 6928 return freed;
70534a73
DC
6929}
6930
f3d90f90
SC
6931static unsigned long mmu_shrink_count(struct shrinker *shrink,
6932 struct shrink_control *sc)
70534a73 6933{
45221ab6 6934 return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
3ee16c81
IE
6935}
6936
e5985c40 6937static struct shrinker *mmu_shrinker;
3ee16c81 6938
2ddfd20e 6939static void mmu_destroy_caches(void)
b5a33a75 6940{
c1bd743e
TH
6941 kmem_cache_destroy(pte_list_desc_cache);
6942 kmem_cache_destroy(mmu_page_header_cache);
b5a33a75
AK
6943}
6944
0b210faf
SC
6945static int get_nx_huge_pages(char *buffer, const struct kernel_param *kp)
6946{
6947 if (nx_hugepage_mitigation_hard_disabled)
1d6664fa 6948 return sysfs_emit(buffer, "never\n");
0b210faf
SC
6949
6950 return param_get_bool(buffer, kp);
6951}
6952
b8e8c830
PB
6953static bool get_nx_auto_mode(void)
6954{
6955 /* Return true when CPU has the bug, and mitigations are ON */
6956 return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off();
6957}
6958
6959static void __set_nx_huge_pages(bool val)
6960{
6961 nx_huge_pages = itlb_multihit_kvm_mitigation = val;
6962}
6963
6964static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
6965{
6966 bool old_val = nx_huge_pages;
6967 bool new_val;
6968
0b210faf
SC
6969 if (nx_hugepage_mitigation_hard_disabled)
6970 return -EPERM;
6971
b8e8c830 6972 /* In "auto" mode deploy workaround only if CPU has the bug. */
0b210faf 6973 if (sysfs_streq(val, "off")) {
b8e8c830 6974 new_val = 0;
0b210faf 6975 } else if (sysfs_streq(val, "force")) {
b8e8c830 6976 new_val = 1;
0b210faf 6977 } else if (sysfs_streq(val, "auto")) {
b8e8c830 6978 new_val = get_nx_auto_mode();
0b210faf
SC
6979 } else if (sysfs_streq(val, "never")) {
6980 new_val = 0;
6981
6982 mutex_lock(&kvm_lock);
6983 if (!list_empty(&vm_list)) {
6984 mutex_unlock(&kvm_lock);
6985 return -EBUSY;
6986 }
6987 nx_hugepage_mitigation_hard_disabled = true;
6988 mutex_unlock(&kvm_lock);
6989 } else if (kstrtobool(val, &new_val) < 0) {
b8e8c830 6990 return -EINVAL;
0b210faf 6991 }
b8e8c830
PB
6992
6993 __set_nx_huge_pages(new_val);
6994
6995 if (new_val != old_val) {
6996 struct kvm *kvm;
b8e8c830
PB
6997
6998 mutex_lock(&kvm_lock);
6999
7000 list_for_each_entry(kvm, &vm_list, vm_list) {
ed69a6cb 7001 mutex_lock(&kvm->slots_lock);
b8e8c830 7002 kvm_mmu_zap_all_fast(kvm);
ed69a6cb 7003 mutex_unlock(&kvm->slots_lock);
1aa9b957 7004
55c510e2 7005 wake_up_process(kvm->arch.nx_huge_page_recovery_thread);
b8e8c830
PB
7006 }
7007 mutex_unlock(&kvm_lock);
7008 }
7009
7010 return 0;
7011}
7012
1d0e8480
SC
7013/*
7014 * nx_huge_pages needs to be resolved to true/false when kvm.ko is loaded, as
7015 * its default value of -1 is technically undefined behavior for a boolean.
c3e0c8c2
SC
7016 * Forward the module init call to SPTE code so that it too can handle module
7017 * params that need to be resolved/snapshot.
1d0e8480 7018 */
982bae43 7019void __init kvm_mmu_x86_module_init(void)
b5a33a75 7020{
b8e8c830
PB
7021 if (nx_huge_pages == -1)
7022 __set_nx_huge_pages(get_nx_auto_mode());
c3e0c8c2 7023
1f98f2bd
DM
7024 /*
7025 * Snapshot userspace's desire to enable the TDP MMU. Whether or not the
7026 * TDP MMU is actually enabled is determined in kvm_configure_mmu()
7027 * when the vendor module is loaded.
7028 */
7029 tdp_mmu_allowed = tdp_mmu_enabled;
7030
c3e0c8c2 7031 kvm_mmu_spte_module_init();
1d0e8480
SC
7032}
7033
7034/*
7035 * The bulk of the MMU initialization is deferred until the vendor module is
7036 * loaded as many of the masks/values may be modified by VMX or SVM, i.e. need
7037 * to be reset when a potentially different vendor module is loaded.
7038 */
7039int kvm_mmu_vendor_module_init(void)
7040{
7041 int ret = -ENOMEM;
b8e8c830 7042
36d9594d
VK
7043 /*
7044 * MMU roles use union aliasing which is, generally speaking, an
7045 * undefined behavior. However, we supposedly know how compilers behave
7046 * and the current status quo is unlikely to change. Guardians below are
7047 * supposed to let us know if the assumption becomes false.
7048 */
7049 BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
7050 BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
7a7ae829 7051 BUILD_BUG_ON(sizeof(union kvm_cpu_role) != sizeof(u64));
36d9594d 7052
28a1f3ac 7053 kvm_mmu_reset_all_pte_masks();
f160c7b7 7054
0dbd0546 7055 pte_list_desc_cache = KMEM_CACHE(pte_list_desc, SLAB_ACCOUNT);
53c07b18 7056 if (!pte_list_desc_cache)
ab271bd4 7057 goto out;
b5a33a75 7058
d3d25b04
AK
7059 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
7060 sizeof(struct kvm_mmu_page),
46bea48a 7061 0, SLAB_ACCOUNT, NULL);
d3d25b04 7062 if (!mmu_page_header_cache)
ab271bd4 7063 goto out;
d3d25b04 7064
908c7f19 7065 if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
ab271bd4 7066 goto out;
45bf21a8 7067
e5985c40
QZ
7068 mmu_shrinker = shrinker_alloc(0, "x86-mmu");
7069 if (!mmu_shrinker)
d7c9bfb9 7070 goto out_shrinker;
3ee16c81 7071
e5985c40
QZ
7072 mmu_shrinker->count_objects = mmu_shrink_count;
7073 mmu_shrinker->scan_objects = mmu_shrink_scan;
7074 mmu_shrinker->seeks = DEFAULT_SEEKS * 10;
7075
7076 shrinker_register(mmu_shrinker);
7077
b5a33a75
AK
7078 return 0;
7079
d7c9bfb9
ML
7080out_shrinker:
7081 percpu_counter_destroy(&kvm_total_used_mmu_pages);
ab271bd4 7082out:
3ee16c81 7083 mmu_destroy_caches();
ab271bd4 7084 return ret;
b5a33a75
AK
7085}
7086
c42fffe3
XG
7087void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
7088{
95f93af4 7089 kvm_mmu_unload(vcpu);
1cfff4d9
JP
7090 free_mmu_pages(&vcpu->arch.root_mmu);
7091 free_mmu_pages(&vcpu->arch.guest_mmu);
c42fffe3 7092 mmu_free_memory_caches(vcpu);
b034cf01
XG
7093}
7094
1d0e8480 7095void kvm_mmu_vendor_module_exit(void)
b034cf01
XG
7096{
7097 mmu_destroy_caches();
7098 percpu_counter_destroy(&kvm_total_used_mmu_pages);
e5985c40 7099 shrinker_free(mmu_shrinker);
c42fffe3 7100}
1aa9b957 7101
f47491d7
SC
7102/*
7103 * Calculate the effective recovery period, accounting for '0' meaning "let KVM
7104 * select a halving time of 1 hour". Returns true if recovery is enabled.
7105 */
7106static bool calc_nx_huge_pages_recovery_period(uint *period)
7107{
7108 /*
7109 * Use READ_ONCE to get the params, this may be called outside of the
7110 * param setters, e.g. by the kthread to compute its next timeout.
7111 */
7112 bool enabled = READ_ONCE(nx_huge_pages);
7113 uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
7114
7115 if (!enabled || !ratio)
7116 return false;
7117
7118 *period = READ_ONCE(nx_huge_pages_recovery_period_ms);
7119 if (!*period) {
7120 /* Make sure the period is not less than one second. */
7121 ratio = min(ratio, 3600u);
7122 *period = 60 * 60 * 1000 / ratio;
7123 }
7124 return true;
7125}
7126
4dfe4f40 7127static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
1aa9b957 7128{
4dfe4f40
JS
7129 bool was_recovery_enabled, is_recovery_enabled;
7130 uint old_period, new_period;
1aa9b957
JS
7131 int err;
7132
0b210faf
SC
7133 if (nx_hugepage_mitigation_hard_disabled)
7134 return -EPERM;
7135
f47491d7 7136 was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period);
4dfe4f40 7137
1aa9b957
JS
7138 err = param_set_uint(val, kp);
7139 if (err)
7140 return err;
7141
f47491d7 7142 is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period);
4dfe4f40 7143
f47491d7 7144 if (is_recovery_enabled &&
4dfe4f40 7145 (!was_recovery_enabled || old_period > new_period)) {
1aa9b957
JS
7146 struct kvm *kvm;
7147
7148 mutex_lock(&kvm_lock);
7149
7150 list_for_each_entry(kvm, &vm_list, vm_list)
55c510e2 7151 wake_up_process(kvm->arch.nx_huge_page_recovery_thread);
1aa9b957
JS
7152
7153 mutex_unlock(&kvm_lock);
7154 }
7155
7156 return err;
7157}
7158
55c510e2 7159static void kvm_recover_nx_huge_pages(struct kvm *kvm)
1aa9b957 7160{
ade74e14 7161 unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;
eb298605 7162 struct kvm_memory_slot *slot;
1aa9b957
JS
7163 int rcu_idx;
7164 struct kvm_mmu_page *sp;
7165 unsigned int ratio;
7166 LIST_HEAD(invalid_list);
048f4980 7167 bool flush = false;
1aa9b957
JS
7168 ulong to_zap;
7169
7170 rcu_idx = srcu_read_lock(&kvm->srcu);
531810ca 7171 write_lock(&kvm->mmu_lock);
1aa9b957 7172
bb95dfb9
SC
7173 /*
7174 * Zapping TDP MMU shadow pages, including the remote TLB flush, must
7175 * be done under RCU protection, because the pages are freed via RCU
7176 * callback.
7177 */
7178 rcu_read_lock();
7179
1aa9b957 7180 ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
ade74e14 7181 to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0;
7d919c7a 7182 for ( ; to_zap; --to_zap) {
55c510e2 7183 if (list_empty(&kvm->arch.possible_nx_huge_pages))
7d919c7a
SC
7184 break;
7185
1aa9b957
JS
7186 /*
7187 * We use a separate list instead of just using active_mmu_pages
55c510e2
SC
7188 * because the number of shadow pages that be replaced with an
7189 * NX huge page is expected to be relatively small compared to
7190 * the total number of shadow pages. And because the TDP MMU
7191 * doesn't use active_mmu_pages.
1aa9b957 7192 */
55c510e2 7193 sp = list_first_entry(&kvm->arch.possible_nx_huge_pages,
1aa9b957 7194 struct kvm_mmu_page,
55c510e2
SC
7195 possible_nx_huge_page_link);
7196 WARN_ON_ONCE(!sp->nx_huge_page_disallowed);
eb298605
DM
7197 WARN_ON_ONCE(!sp->role.direct);
7198
eb298605
DM
7199 /*
7200 * Unaccount and do not attempt to recover any NX Huge Pages
7201 * that are being dirty tracked, as they would just be faulted
7202 * back in as 4KiB pages. The NX Huge Pages in this slot will be
7203 * recovered, along with all the other huge pages in the slot,
7204 * when dirty logging is disabled.
6c7b2202
PB
7205 *
7206 * Since gfn_to_memslot() is relatively expensive, it helps to
7207 * skip it if it the test cannot possibly return true. On the
7208 * other hand, if any memslot has logging enabled, chances are
7209 * good that all of them do, in which case unaccount_nx_huge_page()
7210 * is much cheaper than zapping the page.
7211 *
7212 * If a memslot update is in progress, reading an incorrect value
7213 * of kvm->nr_memslots_dirty_logging is not a problem: if it is
7214 * becoming zero, gfn_to_memslot() will be done unnecessarily; if
7215 * it is becoming nonzero, the page will be zapped unnecessarily.
7216 * Either way, this only affects efficiency in racy situations,
7217 * and not correctness.
eb298605 7218 */
6c7b2202
PB
7219 slot = NULL;
7220 if (atomic_read(&kvm->nr_memslots_dirty_logging)) {
817fa998
SC
7221 struct kvm_memslots *slots;
7222
7223 slots = kvm_memslots_for_spte_role(kvm, sp->role);
7224 slot = __gfn_to_memslot(slots, sp->gfn);
6c7b2202
PB
7225 WARN_ON_ONCE(!slot);
7226 }
7227
eb298605
DM
7228 if (slot && kvm_slot_dirty_track_enabled(slot))
7229 unaccount_nx_huge_page(kvm, sp);
7230 else if (is_tdp_mmu_page(sp))
315f02c6 7231 flush |= kvm_tdp_mmu_zap_sp(kvm, sp);
3a056757 7232 else
29cf0f50 7233 kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
3a056757 7234 WARN_ON_ONCE(sp->nx_huge_page_disallowed);
1aa9b957 7235
531810ca 7236 if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
048f4980 7237 kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
bb95dfb9
SC
7238 rcu_read_unlock();
7239
531810ca 7240 cond_resched_rwlock_write(&kvm->mmu_lock);
048f4980 7241 flush = false;
bb95dfb9
SC
7242
7243 rcu_read_lock();
1aa9b957
JS
7244 }
7245 }
048f4980 7246 kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
1aa9b957 7247
bb95dfb9
SC
7248 rcu_read_unlock();
7249
531810ca 7250 write_unlock(&kvm->mmu_lock);
1aa9b957
JS
7251 srcu_read_unlock(&kvm->srcu, rcu_idx);
7252}
7253
55c510e2 7254static long get_nx_huge_page_recovery_timeout(u64 start_time)
1aa9b957 7255{
f47491d7
SC
7256 bool enabled;
7257 uint period;
4dfe4f40 7258
f47491d7 7259 enabled = calc_nx_huge_pages_recovery_period(&period);
4dfe4f40 7260
f47491d7
SC
7261 return enabled ? start_time + msecs_to_jiffies(period) - get_jiffies_64()
7262 : MAX_SCHEDULE_TIMEOUT;
1aa9b957
JS
7263}
7264
55c510e2 7265static int kvm_nx_huge_page_recovery_worker(struct kvm *kvm, uintptr_t data)
1aa9b957
JS
7266{
7267 u64 start_time;
7268 long remaining_time;
7269
7270 while (true) {
7271 start_time = get_jiffies_64();
55c510e2 7272 remaining_time = get_nx_huge_page_recovery_timeout(start_time);
1aa9b957
JS
7273
7274 set_current_state(TASK_INTERRUPTIBLE);
7275 while (!kthread_should_stop() && remaining_time > 0) {
7276 schedule_timeout(remaining_time);
55c510e2 7277 remaining_time = get_nx_huge_page_recovery_timeout(start_time);
1aa9b957
JS
7278 set_current_state(TASK_INTERRUPTIBLE);
7279 }
7280
7281 set_current_state(TASK_RUNNING);
7282
7283 if (kthread_should_stop())
7284 return 0;
7285
55c510e2 7286 kvm_recover_nx_huge_pages(kvm);
1aa9b957
JS
7287 }
7288}
7289
7290int kvm_mmu_post_init_vm(struct kvm *kvm)
7291{
7292 int err;
7293
0b210faf
SC
7294 if (nx_hugepage_mitigation_hard_disabled)
7295 return 0;
7296
55c510e2 7297 err = kvm_vm_create_worker_thread(kvm, kvm_nx_huge_page_recovery_worker, 0,
1aa9b957 7298 "kvm-nx-lpage-recovery",
55c510e2 7299 &kvm->arch.nx_huge_page_recovery_thread);
1aa9b957 7300 if (!err)
55c510e2 7301 kthread_unpark(kvm->arch.nx_huge_page_recovery_thread);
1aa9b957
JS
7302
7303 return err;
7304}
7305
7306void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
7307{
55c510e2
SC
7308 if (kvm->arch.nx_huge_page_recovery_thread)
7309 kthread_stop(kvm->arch.nx_huge_page_recovery_thread);
1aa9b957 7310}
90b4fe17
CP
7311
7312#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
8dd2eee9
CP
7313bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
7314 struct kvm_gfn_range *range)
7315{
7316 /*
7317 * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only
7318 * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM
7319 * can simply ignore such slots. But if userspace is making memory
7320 * PRIVATE, then KVM must prevent the guest from accessing the memory
7321 * as shared. And if userspace is making memory SHARED and this point
7322 * is reached, then at least one page within the range was previously
7323 * PRIVATE, i.e. the slot's possible hugepage ranges are changing.
7324 * Zapping SPTEs in this case ensures KVM will reassess whether or not
7325 * a hugepage can be used for affected ranges.
7326 */
7327 if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
7328 return false;
7329
7330 return kvm_unmap_gfn_range(kvm, range);
7331}
7332
90b4fe17
CP
7333static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7334 int level)
7335{
7336 return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;
7337}
7338
7339static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7340 int level)
7341{
7342 lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;
7343}
7344
7345static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7346 int level)
7347{
7348 lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
7349}
7350
7351static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
7352 gfn_t gfn, int level, unsigned long attrs)
7353{
7354 const unsigned long start = gfn;
7355 const unsigned long end = start + KVM_PAGES_PER_HPAGE(level);
7356
7357 if (level == PG_LEVEL_2M)
7358 return kvm_range_has_memory_attributes(kvm, start, end, attrs);
7359
7360 for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {
7361 if (hugepage_test_mixed(slot, gfn, level - 1) ||
7362 attrs != kvm_get_memory_attributes(kvm, gfn))
7363 return false;
7364 }
7365 return true;
7366}
7367
7368bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
7369 struct kvm_gfn_range *range)
7370{
7371 unsigned long attrs = range->arg.attributes;
7372 struct kvm_memory_slot *slot = range->slot;
7373 int level;
7374
7375 lockdep_assert_held_write(&kvm->mmu_lock);
7376 lockdep_assert_held(&kvm->slots_lock);
7377
7378 /*
7379 * Calculate which ranges can be mapped with hugepages even if the slot
7380 * can't map memory PRIVATE. KVM mustn't create a SHARED hugepage over
7381 * a range that has PRIVATE GFNs, and conversely converting a range to
7382 * SHARED may now allow hugepages.
7383 */
7384 if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
7385 return false;
7386
7387 /*
7388 * The sequence matters here: upper levels consume the result of lower
7389 * level's scanning.
7390 */
7391 for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
7392 gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level);
7393 gfn_t gfn = gfn_round_for_level(range->start, level);
7394
7395 /* Process the head page if it straddles the range. */
7396 if (gfn != range->start || gfn + nr_pages > range->end) {
7397 /*
7398 * Skip mixed tracking if the aligned gfn isn't covered
7399 * by the memslot, KVM can't use a hugepage due to the
7400 * misaligned address regardless of memory attributes.
7401 */
992b54bd
RE
7402 if (gfn >= slot->base_gfn &&
7403 gfn + nr_pages <= slot->base_gfn + slot->npages) {
90b4fe17
CP
7404 if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
7405 hugepage_clear_mixed(slot, gfn, level);
7406 else
7407 hugepage_set_mixed(slot, gfn, level);
7408 }
7409 gfn += nr_pages;
7410 }
7411
7412 /*
7413 * Pages entirely covered by the range are guaranteed to have
7414 * only the attributes which were just set.
7415 */
7416 for ( ; gfn + nr_pages <= range->end; gfn += nr_pages)
7417 hugepage_clear_mixed(slot, gfn, level);
7418
7419 /*
7420 * Process the last tail page if it straddles the range and is
7421 * contained by the memslot. Like the head page, KVM can't
7422 * create a hugepage if the slot size is misaligned.
7423 */
7424 if (gfn < range->end &&
7425 (gfn + nr_pages) <= (slot->base_gfn + slot->npages)) {
7426 if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
7427 hugepage_clear_mixed(slot, gfn, level);
7428 else
7429 hugepage_set_mixed(slot, gfn, level);
7430 }
7431 }
7432 return false;
7433}
7434
7435void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
7436 struct kvm_memory_slot *slot)
7437{
7438 int level;
7439
7440 if (!kvm_arch_has_private_mem(kvm))
7441 return;
7442
7443 for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
7444 /*
7445 * Don't bother tracking mixed attributes for pages that can't
7446 * be huge due to alignment, i.e. process only pages that are
7447 * entirely contained by the memslot.
7448 */
7449 gfn_t end = gfn_round_for_level(slot->base_gfn + slot->npages, level);
7450 gfn_t start = gfn_round_for_level(slot->base_gfn, level);
7451 gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level);
7452 gfn_t gfn;
7453
7454 if (start < slot->base_gfn)
7455 start += nr_pages;
7456
7457 /*
7458 * Unlike setting attributes, every potential hugepage needs to
7459 * be manually checked as the attributes may already be mixed.
7460 */
7461 for (gfn = start; gfn < end; gfn += nr_pages) {
7462 unsigned long attrs = kvm_get_memory_attributes(kvm, gfn);
7463
7464 if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
7465 hugepage_clear_mixed(slot, gfn, level);
7466 else
7467 hugepage_set_mixed(slot, gfn, level);
7468 }
7469 }
7470}
7471#endif