Commit | Line | Data |
---|---|---|
2415e46e AP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2022 Ventana Micro Systems Inc. | |
4 | */ | |
5 | ||
13acfec2 AP |
6 | #include <linux/bitmap.h> |
7 | #include <linux/cpumask.h> | |
2415e46e AP |
8 | #include <linux/errno.h> |
9 | #include <linux/err.h> | |
10 | #include <linux/module.h> | |
13acfec2 | 11 | #include <linux/smp.h> |
2415e46e | 12 | #include <linux/kvm_host.h> |
13acfec2 | 13 | #include <asm/cacheflush.h> |
2415e46e | 14 | #include <asm/csr.h> |
5ff11248 | 15 | #include <asm/hwcap.h> |
bb233a11 | 16 | #include <asm/insn-def.h> |
2415e46e | 17 | |
e8ad17d2 | 18 | #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) |
5ff11248 | 19 | |
2415e46e AP |
20 | void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, |
21 | gpa_t gpa, gpa_t gpsz, | |
22 | unsigned long order) | |
23 | { | |
24 | gpa_t pos; | |
25 | ||
26 | if (PTRS_PER_PTE < (gpsz >> order)) { | |
27 | kvm_riscv_local_hfence_gvma_vmid_all(vmid); | |
28 | return; | |
29 | } | |
30 | ||
5ff11248 AP |
31 | if (has_svinval()) { |
32 | asm volatile (SFENCE_W_INVAL() ::: "memory"); | |
33 | for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) | |
34 | asm volatile (HINVAL_GVMA(%0, %1) | |
35 | : : "r" (pos >> 2), "r" (vmid) : "memory"); | |
36 | asm volatile (SFENCE_INVAL_IR() ::: "memory"); | |
37 | } else { | |
38 | for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) | |
39 | asm volatile (HFENCE_GVMA(%0, %1) | |
40 | : : "r" (pos >> 2), "r" (vmid) : "memory"); | |
41 | } | |
2415e46e AP |
42 | } |
43 | ||
44 | void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) | |
45 | { | |
bb233a11 | 46 | asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory"); |
2415e46e AP |
47 | } |
48 | ||
49 | void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, | |
50 | unsigned long order) | |
51 | { | |
52 | gpa_t pos; | |
53 | ||
54 | if (PTRS_PER_PTE < (gpsz >> order)) { | |
55 | kvm_riscv_local_hfence_gvma_all(); | |
56 | return; | |
57 | } | |
58 | ||
5ff11248 AP |
59 | if (has_svinval()) { |
60 | asm volatile (SFENCE_W_INVAL() ::: "memory"); | |
61 | for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) | |
62 | asm volatile(HINVAL_GVMA(%0, zero) | |
63 | : : "r" (pos >> 2) : "memory"); | |
64 | asm volatile (SFENCE_INVAL_IR() ::: "memory"); | |
65 | } else { | |
66 | for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) | |
67 | asm volatile(HFENCE_GVMA(%0, zero) | |
68 | : : "r" (pos >> 2) : "memory"); | |
69 | } | |
2415e46e AP |
70 | } |
71 | ||
72 | void kvm_riscv_local_hfence_gvma_all(void) | |
73 | { | |
bb233a11 | 74 | asm volatile(HFENCE_GVMA(zero, zero) : : : "memory"); |
2415e46e AP |
75 | } |
76 | ||
2415e46e AP |
77 | void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, |
78 | unsigned long asid, | |
79 | unsigned long gva, | |
80 | unsigned long gvsz, | |
81 | unsigned long order) | |
82 | { | |
83 | unsigned long pos, hgatp; | |
84 | ||
85 | if (PTRS_PER_PTE < (gvsz >> order)) { | |
86 | kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); | |
87 | return; | |
88 | } | |
89 | ||
90 | hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); | |
91 | ||
5ff11248 AP |
92 | if (has_svinval()) { |
93 | asm volatile (SFENCE_W_INVAL() ::: "memory"); | |
94 | for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) | |
95 | asm volatile(HINVAL_VVMA(%0, %1) | |
96 | : : "r" (pos), "r" (asid) : "memory"); | |
97 | asm volatile (SFENCE_INVAL_IR() ::: "memory"); | |
98 | } else { | |
99 | for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) | |
100 | asm volatile(HFENCE_VVMA(%0, %1) | |
101 | : : "r" (pos), "r" (asid) : "memory"); | |
102 | } | |
2415e46e AP |
103 | |
104 | csr_write(CSR_HGATP, hgatp); | |
105 | } | |
106 | ||
107 | void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, | |
108 | unsigned long asid) | |
109 | { | |
110 | unsigned long hgatp; | |
111 | ||
112 | hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); | |
113 | ||
bb233a11 | 114 | asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory"); |
2415e46e AP |
115 | |
116 | csr_write(CSR_HGATP, hgatp); | |
117 | } | |
118 | ||
119 | void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, | |
120 | unsigned long gva, unsigned long gvsz, | |
121 | unsigned long order) | |
122 | { | |
123 | unsigned long pos, hgatp; | |
124 | ||
125 | if (PTRS_PER_PTE < (gvsz >> order)) { | |
126 | kvm_riscv_local_hfence_vvma_all(vmid); | |
127 | return; | |
128 | } | |
129 | ||
130 | hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); | |
131 | ||
5ff11248 AP |
132 | if (has_svinval()) { |
133 | asm volatile (SFENCE_W_INVAL() ::: "memory"); | |
134 | for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) | |
135 | asm volatile(HINVAL_VVMA(%0, zero) | |
136 | : : "r" (pos) : "memory"); | |
137 | asm volatile (SFENCE_INVAL_IR() ::: "memory"); | |
138 | } else { | |
139 | for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) | |
140 | asm volatile(HFENCE_VVMA(%0, zero) | |
141 | : : "r" (pos) : "memory"); | |
142 | } | |
2415e46e AP |
143 | |
144 | csr_write(CSR_HGATP, hgatp); | |
145 | } | |
146 | ||
147 | void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) | |
148 | { | |
149 | unsigned long hgatp; | |
150 | ||
151 | hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); | |
152 | ||
bb233a11 | 153 | asm volatile(HFENCE_VVMA(zero, zero) : : : "memory"); |
2415e46e AP |
154 | |
155 | csr_write(CSR_HGATP, hgatp); | |
156 | } | |
13acfec2 | 157 | |
92e45050 AP |
158 | void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) |
159 | { | |
160 | unsigned long vmid; | |
161 | ||
162 | if (!kvm_riscv_gstage_vmid_bits() || | |
163 | vcpu->arch.last_exit_cpu == vcpu->cpu) | |
164 | return; | |
165 | ||
166 | /* | |
167 | * On RISC-V platforms with hardware VMID support, we share same | |
168 | * VMID for all VCPUs of a particular Guest/VM. This means we might | |
169 | * have stale G-stage TLB entries on the current Host CPU due to | |
170 | * some other VCPU of the same Guest which ran previously on the | |
171 | * current Host CPU. | |
172 | * | |
173 | * To cleanup stale TLB entries, we simply flush all G-stage TLB | |
174 | * entries by VMID whenever underlying Host CPU changes for a VCPU. | |
175 | */ | |
176 | ||
177 | vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); | |
178 | kvm_riscv_local_hfence_gvma_vmid_all(vmid); | |
179 | } | |
180 | ||
13acfec2 AP |
181 | void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) |
182 | { | |
c39cea6f | 183 | kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD); |
13acfec2 AP |
184 | local_flush_icache_all(); |
185 | } | |
186 | ||
187 | void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu) | |
188 | { | |
189 | struct kvm_vmid *vmid; | |
190 | ||
191 | vmid = &vcpu->kvm->arch.vmid; | |
192 | kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid)); | |
193 | } | |
194 | ||
195 | void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) | |
196 | { | |
197 | struct kvm_vmid *vmid; | |
198 | ||
199 | vmid = &vcpu->kvm->arch.vmid; | |
200 | kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid)); | |
201 | } | |
202 | ||
203 | static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu, | |
204 | struct kvm_riscv_hfence *out_data) | |
205 | { | |
206 | bool ret = false; | |
207 | struct kvm_vcpu_arch *varch = &vcpu->arch; | |
208 | ||
209 | spin_lock(&varch->hfence_lock); | |
210 | ||
211 | if (varch->hfence_queue[varch->hfence_head].type) { | |
212 | memcpy(out_data, &varch->hfence_queue[varch->hfence_head], | |
213 | sizeof(*out_data)); | |
214 | varch->hfence_queue[varch->hfence_head].type = 0; | |
215 | ||
216 | varch->hfence_head++; | |
217 | if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE) | |
218 | varch->hfence_head = 0; | |
219 | ||
220 | ret = true; | |
221 | } | |
222 | ||
223 | spin_unlock(&varch->hfence_lock); | |
224 | ||
225 | return ret; | |
226 | } | |
227 | ||
228 | static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, | |
229 | const struct kvm_riscv_hfence *data) | |
230 | { | |
231 | bool ret = false; | |
232 | struct kvm_vcpu_arch *varch = &vcpu->arch; | |
233 | ||
234 | spin_lock(&varch->hfence_lock); | |
235 | ||
236 | if (!varch->hfence_queue[varch->hfence_tail].type) { | |
237 | memcpy(&varch->hfence_queue[varch->hfence_tail], | |
238 | data, sizeof(*data)); | |
239 | ||
240 | varch->hfence_tail++; | |
241 | if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE) | |
242 | varch->hfence_tail = 0; | |
243 | ||
244 | ret = true; | |
245 | } | |
246 | ||
247 | spin_unlock(&varch->hfence_lock); | |
248 | ||
249 | return ret; | |
250 | } | |
251 | ||
252 | void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) | |
253 | { | |
254 | struct kvm_riscv_hfence d = { 0 }; | |
255 | struct kvm_vmid *v = &vcpu->kvm->arch.vmid; | |
256 | ||
257 | while (vcpu_hfence_dequeue(vcpu, &d)) { | |
258 | switch (d.type) { | |
259 | case KVM_RISCV_HFENCE_UNKNOWN: | |
260 | break; | |
261 | case KVM_RISCV_HFENCE_GVMA_VMID_GPA: | |
262 | kvm_riscv_local_hfence_gvma_vmid_gpa( | |
263 | READ_ONCE(v->vmid), | |
264 | d.addr, d.size, d.order); | |
265 | break; | |
266 | case KVM_RISCV_HFENCE_VVMA_ASID_GVA: | |
c39cea6f | 267 | kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); |
13acfec2 AP |
268 | kvm_riscv_local_hfence_vvma_asid_gva( |
269 | READ_ONCE(v->vmid), d.asid, | |
270 | d.addr, d.size, d.order); | |
271 | break; | |
272 | case KVM_RISCV_HFENCE_VVMA_ASID_ALL: | |
c39cea6f | 273 | kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_ASID_RCVD); |
13acfec2 AP |
274 | kvm_riscv_local_hfence_vvma_asid_all( |
275 | READ_ONCE(v->vmid), d.asid); | |
276 | break; | |
277 | case KVM_RISCV_HFENCE_VVMA_GVA: | |
c39cea6f | 278 | kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_RCVD); |
13acfec2 AP |
279 | kvm_riscv_local_hfence_vvma_gva( |
280 | READ_ONCE(v->vmid), | |
281 | d.addr, d.size, d.order); | |
282 | break; | |
283 | default: | |
284 | break; | |
285 | } | |
286 | } | |
287 | } | |
288 | ||
289 | static void make_xfence_request(struct kvm *kvm, | |
290 | unsigned long hbase, unsigned long hmask, | |
291 | unsigned int req, unsigned int fallback_req, | |
292 | const struct kvm_riscv_hfence *data) | |
293 | { | |
294 | unsigned long i; | |
295 | struct kvm_vcpu *vcpu; | |
296 | unsigned int actual_req = req; | |
297 | DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); | |
298 | ||
78329084 | 299 | bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); |
13acfec2 AP |
300 | kvm_for_each_vcpu(i, vcpu, kvm) { |
301 | if (hbase != -1UL) { | |
302 | if (vcpu->vcpu_id < hbase) | |
303 | continue; | |
304 | if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) | |
305 | continue; | |
306 | } | |
307 | ||
308 | bitmap_set(vcpu_mask, i, 1); | |
309 | ||
310 | if (!data || !data->type) | |
311 | continue; | |
312 | ||
313 | /* | |
314 | * Enqueue hfence data to VCPU hfence queue. If we don't | |
315 | * have space in the VCPU hfence queue then fallback to | |
316 | * a more conservative hfence request. | |
317 | */ | |
318 | if (!vcpu_hfence_enqueue(vcpu, data)) | |
319 | actual_req = fallback_req; | |
320 | } | |
321 | ||
322 | kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask); | |
323 | } | |
324 | ||
325 | void kvm_riscv_fence_i(struct kvm *kvm, | |
326 | unsigned long hbase, unsigned long hmask) | |
327 | { | |
328 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I, | |
329 | KVM_REQ_FENCE_I, NULL); | |
330 | } | |
331 | ||
332 | void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, | |
333 | unsigned long hbase, unsigned long hmask, | |
334 | gpa_t gpa, gpa_t gpsz, | |
335 | unsigned long order) | |
336 | { | |
337 | struct kvm_riscv_hfence data; | |
338 | ||
339 | data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; | |
340 | data.asid = 0; | |
341 | data.addr = gpa; | |
342 | data.size = gpsz; | |
343 | data.order = order; | |
344 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, | |
345 | KVM_REQ_HFENCE_GVMA_VMID_ALL, &data); | |
346 | } | |
347 | ||
348 | void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, | |
349 | unsigned long hbase, unsigned long hmask) | |
350 | { | |
351 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL, | |
352 | KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL); | |
353 | } | |
354 | ||
355 | void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, | |
356 | unsigned long hbase, unsigned long hmask, | |
357 | unsigned long gva, unsigned long gvsz, | |
358 | unsigned long order, unsigned long asid) | |
359 | { | |
360 | struct kvm_riscv_hfence data; | |
361 | ||
362 | data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; | |
363 | data.asid = asid; | |
364 | data.addr = gva; | |
365 | data.size = gvsz; | |
366 | data.order = order; | |
367 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, | |
368 | KVM_REQ_HFENCE_VVMA_ALL, &data); | |
369 | } | |
370 | ||
371 | void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, | |
372 | unsigned long hbase, unsigned long hmask, | |
373 | unsigned long asid) | |
374 | { | |
375 | struct kvm_riscv_hfence data; | |
376 | ||
377 | data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; | |
378 | data.asid = asid; | |
379 | data.addr = data.size = data.order = 0; | |
380 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, | |
381 | KVM_REQ_HFENCE_VVMA_ALL, &data); | |
382 | } | |
383 | ||
384 | void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, | |
385 | unsigned long hbase, unsigned long hmask, | |
386 | unsigned long gva, unsigned long gvsz, | |
387 | unsigned long order) | |
388 | { | |
389 | struct kvm_riscv_hfence data; | |
390 | ||
391 | data.type = KVM_RISCV_HFENCE_VVMA_GVA; | |
392 | data.asid = 0; | |
393 | data.addr = gva; | |
394 | data.size = gvsz; | |
395 | data.order = order; | |
396 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, | |
397 | KVM_REQ_HFENCE_VVMA_ALL, &data); | |
398 | } | |
399 | ||
400 | void kvm_riscv_hfence_vvma_all(struct kvm *kvm, | |
401 | unsigned long hbase, unsigned long hmask) | |
402 | { | |
403 | make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL, | |
404 | KVM_REQ_HFENCE_VVMA_ALL, NULL); | |
405 | } |