Commit | Line | Data |
---|---|---|
c9180b72 BG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #ifndef __KVM_X86_MMU_TDP_ITER_H | |
4 | #define __KVM_X86_MMU_TDP_ITER_H | |
5 | ||
6 | #include <linux/kvm_host.h> | |
7 | ||
8 | #include "mmu.h" | |
ba3a6120 | 9 | #include "spte.h" |
c9180b72 | 10 | |
0e587aa7 SC |
11 | /* |
12 | * TDP MMU SPTEs are RCU protected to allow paging structures (non-leaf SPTEs) | |
bb95dfb9 SC |
13 | * to be zapped while holding mmu_lock for read, and to allow TLB flushes to be |
14 | * batched without having to collect the list of zapped SPs. Flows that can | |
15 | * remove SPs must service pending TLB flushes prior to dropping RCU protection. | |
0e587aa7 SC |
16 | */ |
17 | static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) | |
18 | { | |
19 | return READ_ONCE(*rcu_dereference(sptep)); | |
20 | } | |
ba3a6120 SC |
21 | |
22 | static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) | |
23 | { | |
24 | return xchg(rcu_dereference(sptep), new_spte); | |
25 | } | |
26 | ||
27 | static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte) | |
0e587aa7 | 28 | { |
ba3a6120 SC |
29 | WRITE_ONCE(*rcu_dereference(sptep), new_spte); |
30 | } | |
31 | ||
41e07665 VS |
32 | /* |
33 | * SPTEs must be modified atomically if they are shadow-present, leaf | |
34 | * SPTEs, and have volatile bits, i.e. has bits that can be set outside | |
35 | * of mmu_lock. The Writable bit can be set by KVM's fast page fault | |
36 | * handler, and Accessed and Dirty bits can be set by the CPU. | |
37 | * | |
38 | * Note, non-leaf SPTEs do have Accessed bits and those bits are | |
39 | * technically volatile, but KVM doesn't consume the Accessed bit of | |
40 | * non-leaf SPTEs, i.e. KVM doesn't care if it clobbers the bit. This | |
41 | * logic needs to be reassessed if KVM were to use non-leaf Accessed | |
42 | * bits, e.g. to skip stepping down into child SPTEs when aging SPTEs. | |
43 | */ | |
44 | static inline bool kvm_tdp_mmu_spte_need_atomic_write(u64 old_spte, int level) | |
45 | { | |
46 | return is_shadow_present_pte(old_spte) && | |
47 | is_last_spte(old_spte, level) && | |
48 | spte_has_volatile_bits(old_spte); | |
49 | } | |
50 | ||
ba3a6120 SC |
51 | static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte, |
52 | u64 new_spte, int level) | |
53 | { | |
41e07665 | 54 | if (kvm_tdp_mmu_spte_need_atomic_write(old_spte, level)) |
ba3a6120 SC |
55 | return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte); |
56 | ||
57 | __kvm_tdp_mmu_write_spte(sptep, new_spte); | |
58 | return old_spte; | |
0e587aa7 SC |
59 | } |
60 | ||
89c313f2 VS |
61 | static inline u64 tdp_mmu_clear_spte_bits(tdp_ptep_t sptep, u64 old_spte, |
62 | u64 mask, int level) | |
63 | { | |
64 | atomic64_t *sptep_atomic; | |
65 | ||
66 | if (kvm_tdp_mmu_spte_need_atomic_write(old_spte, level)) { | |
67 | sptep_atomic = (atomic64_t *)rcu_dereference(sptep); | |
68 | return (u64)atomic64_fetch_and(~mask, sptep_atomic); | |
69 | } | |
70 | ||
71 | __kvm_tdp_mmu_write_spte(sptep, old_spte & ~mask); | |
72 | return old_spte; | |
73 | } | |
74 | ||
c9180b72 BG |
75 | /* |
76 | * A TDP iterator performs a pre-order walk over a TDP paging structure. | |
77 | */ | |
78 | struct tdp_iter { | |
79 | /* | |
80 | * The iterator will traverse the paging structure towards the mapping | |
81 | * for this GFN. | |
82 | */ | |
74953d35 | 83 | gfn_t next_last_level_gfn; |
ed5e484b BG |
84 | /* |
85 | * The next_last_level_gfn at the time when the thread last | |
86 | * yielded. Only yielding when the next_last_level_gfn != | |
87 | * yielded_gfn helps ensure forward progress. | |
88 | */ | |
89 | gfn_t yielded_gfn; | |
c9180b72 | 90 | /* Pointers to the page tables traversed to reach the current SPTE */ |
7cca2d0b | 91 | tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL]; |
c9180b72 | 92 | /* A pointer to the current SPTE */ |
7cca2d0b | 93 | tdp_ptep_t sptep; |
c9180b72 BG |
94 | /* The lowest GFN mapped by the current SPTE */ |
95 | gfn_t gfn; | |
96 | /* The level of the root page given to the iterator */ | |
97 | int root_level; | |
98 | /* The lowest level the iterator should traverse to */ | |
99 | int min_level; | |
100 | /* The iterator's current level within the paging structure */ | |
101 | int level; | |
08889894 SC |
102 | /* The address space ID, i.e. SMM vs. regular. */ |
103 | int as_id; | |
c9180b72 BG |
104 | /* A snapshot of the value at sptep */ |
105 | u64 old_spte; | |
106 | /* | |
107 | * Whether the iterator has a valid state. This will be false if the | |
108 | * iterator walks off the end of the paging structure. | |
109 | */ | |
110 | bool valid; | |
3a0f64de SC |
111 | /* |
112 | * True if KVM dropped mmu_lock and yielded in the middle of a walk, in | |
113 | * which case tdp_iter_next() needs to restart the walk at the root | |
114 | * level instead of advancing to the next entry. | |
115 | */ | |
116 | bool yielded; | |
c9180b72 BG |
117 | }; |
118 | ||
119 | /* | |
120 | * Iterates over every SPTE mapping the GFN range [start, end) in a | |
121 | * preorder traversal. | |
122 | */ | |
77aa6075 DM |
123 | #define for_each_tdp_pte_min_level(iter, root, min_level, start, end) \ |
124 | for (tdp_iter_start(&iter, root, min_level, start); \ | |
c9180b72 BG |
125 | iter.valid && iter.gfn < end; \ |
126 | tdp_iter_next(&iter)) | |
127 | ||
77aa6075 DM |
128 | #define for_each_tdp_pte(iter, root, start, end) \ |
129 | for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) | |
a6a0b05d | 130 | |
7cca2d0b | 131 | tdp_ptep_t spte_to_child_pt(u64 pte, int level); |
c9180b72 | 132 | |
77aa6075 | 133 | void tdp_iter_start(struct tdp_iter *iter, struct kvm_mmu_page *root, |
74953d35 | 134 | int min_level, gfn_t next_last_level_gfn); |
c9180b72 | 135 | void tdp_iter_next(struct tdp_iter *iter); |
b601c3bc | 136 | void tdp_iter_restart(struct tdp_iter *iter); |
c9180b72 BG |
137 | |
138 | #endif /* __KVM_X86_MMU_TDP_ITER_H */ |