Commit | Line | Data |
---|---|---|
20c8ccb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
21ebbeda XG |
2 | /* |
3 | * Support KVM gust page tracking | |
4 | * | |
5 | * This feature allows us to track page access in guest. Currently, only | |
6 | * write access is tracked. | |
7 | * | |
8 | * Copyright(C) 2015 Intel Corporation. | |
9 | * | |
10 | * Author: | |
11 | * Xiao Guangrong <guangrong.xiao@linux.intel.com> | |
21ebbeda XG |
12 | */ |
13 | ||
14 | #include <linux/kvm_host.h> | |
b2d09103 IM |
15 | #include <linux/rculist.h> |
16 | ||
21ebbeda XG |
17 | #include <asm/kvm_host.h> |
18 | #include <asm/kvm_page_track.h> | |
19 | ||
20 | #include "mmu.h" | |
21 | ||
22 | void kvm_page_track_free_memslot(struct kvm_memory_slot *free, | |
23 | struct kvm_memory_slot *dont) | |
24 | { | |
25 | int i; | |
26 | ||
27 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) | |
28 | if (!dont || free->arch.gfn_track[i] != | |
29 | dont->arch.gfn_track[i]) { | |
30 | kvfree(free->arch.gfn_track[i]); | |
31 | free->arch.gfn_track[i] = NULL; | |
32 | } | |
33 | } | |
34 | ||
35 | int kvm_page_track_create_memslot(struct kvm_memory_slot *slot, | |
36 | unsigned long npages) | |
37 | { | |
38 | int i; | |
39 | ||
40 | for (i = 0; i < KVM_PAGE_TRACK_MAX; i++) { | |
778e1cdd KC |
41 | slot->arch.gfn_track[i] = |
42 | kvcalloc(npages, sizeof(*slot->arch.gfn_track[i]), | |
254272ce | 43 | GFP_KERNEL_ACCOUNT); |
21ebbeda XG |
44 | if (!slot->arch.gfn_track[i]) |
45 | goto track_free; | |
46 | } | |
47 | ||
48 | return 0; | |
49 | ||
50 | track_free: | |
51 | kvm_page_track_free_memslot(slot, NULL); | |
52 | return -ENOMEM; | |
53 | } | |
f29d4d78 XG |
54 | |
55 | static inline bool page_track_mode_is_valid(enum kvm_page_track_mode mode) | |
56 | { | |
57 | if (mode < 0 || mode >= KVM_PAGE_TRACK_MAX) | |
58 | return false; | |
59 | ||
60 | return true; | |
61 | } | |
62 | ||
63 | static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, | |
64 | enum kvm_page_track_mode mode, short count) | |
65 | { | |
66 | int index, val; | |
67 | ||
68 | index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); | |
69 | ||
70 | val = slot->arch.gfn_track[mode][index]; | |
71 | ||
72 | if (WARN_ON(val + count < 0 || val + count > USHRT_MAX)) | |
73 | return; | |
74 | ||
75 | slot->arch.gfn_track[mode][index] += count; | |
76 | } | |
77 | ||
78 | /* | |
79 | * add guest page to the tracking pool so that corresponding access on that | |
80 | * page will be intercepted. | |
81 | * | |
82 | * It should be called under the protection both of mmu-lock and kvm->srcu | |
83 | * or kvm->slots_lock. | |
84 | * | |
85 | * @kvm: the guest instance we are interested in. | |
86 | * @slot: the @gfn belongs to. | |
87 | * @gfn: the guest page. | |
88 | * @mode: tracking mode, currently only write track is supported. | |
89 | */ | |
90 | void kvm_slot_page_track_add_page(struct kvm *kvm, | |
91 | struct kvm_memory_slot *slot, gfn_t gfn, | |
92 | enum kvm_page_track_mode mode) | |
93 | { | |
94 | ||
95 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
96 | return; | |
97 | ||
98 | update_gfn_track(slot, gfn, mode, 1); | |
99 | ||
100 | /* | |
101 | * new track stops large page mapping for the | |
102 | * tracked page. | |
103 | */ | |
104 | kvm_mmu_gfn_disallow_lpage(slot, gfn); | |
105 | ||
106 | if (mode == KVM_PAGE_TRACK_WRITE) | |
107 | if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) | |
108 | kvm_flush_remote_tlbs(kvm); | |
109 | } | |
871b7ef2 | 110 | EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); |
f29d4d78 XG |
111 | |
112 | /* | |
113 | * remove the guest page from the tracking pool which stops the interception | |
114 | * of corresponding access on that page. It is the opposed operation of | |
115 | * kvm_slot_page_track_add_page(). | |
116 | * | |
117 | * It should be called under the protection both of mmu-lock and kvm->srcu | |
118 | * or kvm->slots_lock. | |
119 | * | |
120 | * @kvm: the guest instance we are interested in. | |
121 | * @slot: the @gfn belongs to. | |
122 | * @gfn: the guest page. | |
123 | * @mode: tracking mode, currently only write track is supported. | |
124 | */ | |
125 | void kvm_slot_page_track_remove_page(struct kvm *kvm, | |
126 | struct kvm_memory_slot *slot, gfn_t gfn, | |
127 | enum kvm_page_track_mode mode) | |
128 | { | |
129 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
130 | return; | |
131 | ||
132 | update_gfn_track(slot, gfn, mode, -1); | |
133 | ||
134 | /* | |
135 | * allow large page mapping for the tracked page | |
136 | * after the tracker is gone. | |
137 | */ | |
138 | kvm_mmu_gfn_allow_lpage(slot, gfn); | |
139 | } | |
871b7ef2 | 140 | EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page); |
3d0c27ad XG |
141 | |
142 | /* | |
143 | * check if the corresponding access on the specified guest page is tracked. | |
144 | */ | |
145 | bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, | |
146 | enum kvm_page_track_mode mode) | |
147 | { | |
a6adb106 PB |
148 | struct kvm_memory_slot *slot; |
149 | int index; | |
3d0c27ad XG |
150 | |
151 | if (WARN_ON(!page_track_mode_is_valid(mode))) | |
152 | return false; | |
153 | ||
a6adb106 PB |
154 | slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); |
155 | if (!slot) | |
156 | return false; | |
157 | ||
158 | index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); | |
6aa7de05 | 159 | return !!READ_ONCE(slot->arch.gfn_track[mode][index]); |
3d0c27ad | 160 | } |
0eb05bf2 | 161 | |
2beb6dad PB |
162 | void kvm_page_track_cleanup(struct kvm *kvm) |
163 | { | |
164 | struct kvm_page_track_notifier_head *head; | |
165 | ||
166 | head = &kvm->arch.track_notifier_head; | |
167 | cleanup_srcu_struct(&head->track_srcu); | |
168 | } | |
169 | ||
0eb05bf2 XG |
170 | void kvm_page_track_init(struct kvm *kvm) |
171 | { | |
172 | struct kvm_page_track_notifier_head *head; | |
173 | ||
174 | head = &kvm->arch.track_notifier_head; | |
175 | init_srcu_struct(&head->track_srcu); | |
176 | INIT_HLIST_HEAD(&head->track_notifier_list); | |
177 | } | |
178 | ||
179 | /* | |
180 | * register the notifier so that event interception for the tracked guest | |
181 | * pages can be received. | |
182 | */ | |
183 | void | |
184 | kvm_page_track_register_notifier(struct kvm *kvm, | |
185 | struct kvm_page_track_notifier_node *n) | |
186 | { | |
187 | struct kvm_page_track_notifier_head *head; | |
188 | ||
189 | head = &kvm->arch.track_notifier_head; | |
190 | ||
191 | spin_lock(&kvm->mmu_lock); | |
192 | hlist_add_head_rcu(&n->node, &head->track_notifier_list); | |
193 | spin_unlock(&kvm->mmu_lock); | |
194 | } | |
871b7ef2 | 195 | EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); |
0eb05bf2 XG |
196 | |
197 | /* | |
198 | * stop receiving the event interception. It is the opposed operation of | |
199 | * kvm_page_track_register_notifier(). | |
200 | */ | |
201 | void | |
202 | kvm_page_track_unregister_notifier(struct kvm *kvm, | |
203 | struct kvm_page_track_notifier_node *n) | |
204 | { | |
205 | struct kvm_page_track_notifier_head *head; | |
206 | ||
207 | head = &kvm->arch.track_notifier_head; | |
208 | ||
209 | spin_lock(&kvm->mmu_lock); | |
210 | hlist_del_rcu(&n->node); | |
211 | spin_unlock(&kvm->mmu_lock); | |
212 | synchronize_srcu(&head->track_srcu); | |
213 | } | |
871b7ef2 | 214 | EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); |
0eb05bf2 XG |
215 | |
216 | /* | |
217 | * Notify the node that write access is intercepted and write emulation is | |
218 | * finished at this time. | |
219 | * | |
220 | * The node should figure out if the written page is the one that node is | |
221 | * interested in by itself. | |
222 | */ | |
223 | void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, | |
224 | int bytes) | |
225 | { | |
226 | struct kvm_page_track_notifier_head *head; | |
227 | struct kvm_page_track_notifier_node *n; | |
228 | int idx; | |
229 | ||
230 | head = &vcpu->kvm->arch.track_notifier_head; | |
231 | ||
232 | if (hlist_empty(&head->track_notifier_list)) | |
233 | return; | |
234 | ||
235 | idx = srcu_read_lock(&head->track_srcu); | |
236 | hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) | |
237 | if (n->track_write) | |
d126363d | 238 | n->track_write(vcpu, gpa, new, bytes, n); |
0eb05bf2 XG |
239 | srcu_read_unlock(&head->track_srcu, idx); |
240 | } | |
ae7cd873 XC |
241 | |
242 | /* | |
243 | * Notify the node that memory slot is being removed or moved so that it can | |
244 | * drop write-protection for the pages in the memory slot. | |
245 | * | |
246 | * The node should figure out it has any write-protected pages in this slot | |
247 | * by itself. | |
248 | */ | |
249 | void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot) | |
250 | { | |
251 | struct kvm_page_track_notifier_head *head; | |
252 | struct kvm_page_track_notifier_node *n; | |
253 | int idx; | |
254 | ||
255 | head = &kvm->arch.track_notifier_head; | |
256 | ||
257 | if (hlist_empty(&head->track_notifier_list)) | |
258 | return; | |
259 | ||
260 | idx = srcu_read_lock(&head->track_srcu); | |
261 | hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) | |
262 | if (n->track_flush_slot) | |
d126363d | 263 | n->track_flush_slot(kvm, slot, n); |
ae7cd873 XC |
264 | srcu_read_unlock(&head->track_srcu, idx); |
265 | } |