Commit | Line | Data |
---|---|---|
810a5212 TL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Hyper-V Isolation VM interface with paravisor and hypervisor | |
4 | * | |
5 | * Author: | |
6 | * Tianyu Lan <Tianyu.Lan@microsoft.com> | |
7 | */ | |
8 | ||
faff4406 | 9 | #include <linux/bitfield.h> |
810a5212 TL |
10 | #include <linux/hyperv.h> |
11 | #include <linux/types.h> | |
810a5212 | 12 | #include <linux/slab.h> |
faff4406 TL |
13 | #include <asm/svm.h> |
14 | #include <asm/sev.h> | |
810a5212 TL |
15 | #include <asm/io.h> |
16 | #include <asm/mshyperv.h> | |
faff4406 TL |
17 | #include <asm/hypervisor.h> |
18 | ||
19 | #ifdef CONFIG_AMD_MEM_ENCRYPT | |
20c89a55 TL |
20 | |
21 | #define GHCB_USAGE_HYPERV_CALL 1 | |
22 | ||
faff4406 TL |
23 | union hv_ghcb { |
24 | struct ghcb ghcb; | |
20c89a55 TL |
25 | struct { |
26 | u64 hypercalldata[509]; | |
27 | u64 outputgpa; | |
28 | union { | |
29 | union { | |
30 | struct { | |
31 | u32 callcode : 16; | |
32 | u32 isfast : 1; | |
33 | u32 reserved1 : 14; | |
34 | u32 isnested : 1; | |
35 | u32 countofelements : 12; | |
36 | u32 reserved2 : 4; | |
37 | u32 repstartindex : 12; | |
38 | u32 reserved3 : 4; | |
39 | }; | |
40 | u64 asuint64; | |
41 | } hypercallinput; | |
42 | union { | |
43 | struct { | |
44 | u16 callstatus; | |
45 | u16 reserved1; | |
46 | u32 elementsprocessed : 12; | |
47 | u32 reserved2 : 20; | |
48 | }; | |
49 | u64 asunit64; | |
50 | } hypercalloutput; | |
51 | }; | |
52 | u64 reserved2; | |
53 | } hypercall; | |
faff4406 TL |
54 | } __packed __aligned(HV_HYP_PAGE_SIZE); |
55 | ||
49d6a3c0 TL |
56 | static u16 hv_ghcb_version __ro_after_init; |
57 | ||
20c89a55 TL |
58 | u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size) |
59 | { | |
60 | union hv_ghcb *hv_ghcb; | |
61 | void **ghcb_base; | |
62 | unsigned long flags; | |
63 | u64 status; | |
64 | ||
65 | if (!hv_ghcb_pg) | |
66 | return -EFAULT; | |
67 | ||
68 | WARN_ON(in_nmi()); | |
69 | ||
70 | local_irq_save(flags); | |
71 | ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg); | |
72 | hv_ghcb = (union hv_ghcb *)*ghcb_base; | |
73 | if (!hv_ghcb) { | |
74 | local_irq_restore(flags); | |
75 | return -EFAULT; | |
76 | } | |
77 | ||
78 | hv_ghcb->ghcb.protocol_version = GHCB_PROTOCOL_MAX; | |
79 | hv_ghcb->ghcb.ghcb_usage = GHCB_USAGE_HYPERV_CALL; | |
80 | ||
81 | hv_ghcb->hypercall.outputgpa = (u64)output; | |
82 | hv_ghcb->hypercall.hypercallinput.asuint64 = 0; | |
83 | hv_ghcb->hypercall.hypercallinput.callcode = control; | |
84 | ||
85 | if (input_size) | |
86 | memcpy(hv_ghcb->hypercall.hypercalldata, input, input_size); | |
87 | ||
88 | VMGEXIT(); | |
89 | ||
90 | hv_ghcb->ghcb.ghcb_usage = 0xffffffff; | |
91 | memset(hv_ghcb->ghcb.save.valid_bitmap, 0, | |
92 | sizeof(hv_ghcb->ghcb.save.valid_bitmap)); | |
93 | ||
94 | status = hv_ghcb->hypercall.hypercalloutput.callstatus; | |
95 | ||
96 | local_irq_restore(flags); | |
97 | ||
98 | return status; | |
99 | } | |
100 | ||
49d6a3c0 TL |
101 | static inline u64 rd_ghcb_msr(void) |
102 | { | |
103 | return __rdmsr(MSR_AMD64_SEV_ES_GHCB); | |
104 | } | |
105 | ||
106 | static inline void wr_ghcb_msr(u64 val) | |
107 | { | |
108 | native_wrmsrl(MSR_AMD64_SEV_ES_GHCB, val); | |
109 | } | |
110 | ||
111 | static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code, | |
112 | u64 exit_info_1, u64 exit_info_2) | |
113 | { | |
114 | /* Fill in protocol and format specifiers */ | |
115 | ghcb->protocol_version = hv_ghcb_version; | |
116 | ghcb->ghcb_usage = GHCB_DEFAULT_USAGE; | |
117 | ||
118 | ghcb_set_sw_exit_code(ghcb, exit_code); | |
119 | ghcb_set_sw_exit_info_1(ghcb, exit_info_1); | |
120 | ghcb_set_sw_exit_info_2(ghcb, exit_info_2); | |
121 | ||
122 | VMGEXIT(); | |
123 | ||
124 | if (ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0)) | |
125 | return ES_VMM_ERROR; | |
126 | else | |
127 | return ES_OK; | |
128 | } | |
129 | ||
130 | void hv_ghcb_terminate(unsigned int set, unsigned int reason) | |
131 | { | |
132 | u64 val = GHCB_MSR_TERM_REQ; | |
133 | ||
134 | /* Tell the hypervisor what went wrong. */ | |
135 | val |= GHCB_SEV_TERM_REASON(set, reason); | |
136 | ||
137 | /* Request Guest Termination from Hypvervisor */ | |
138 | wr_ghcb_msr(val); | |
139 | VMGEXIT(); | |
140 | ||
141 | while (true) | |
142 | asm volatile("hlt\n" : : : "memory"); | |
143 | } | |
144 | ||
145 | bool hv_ghcb_negotiate_protocol(void) | |
146 | { | |
147 | u64 ghcb_gpa; | |
148 | u64 val; | |
149 | ||
150 | /* Save ghcb page gpa. */ | |
151 | ghcb_gpa = rd_ghcb_msr(); | |
152 | ||
153 | /* Do the GHCB protocol version negotiation */ | |
154 | wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ); | |
155 | VMGEXIT(); | |
156 | val = rd_ghcb_msr(); | |
157 | ||
158 | if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP) | |
159 | return false; | |
160 | ||
161 | if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN || | |
162 | GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX) | |
163 | return false; | |
164 | ||
165 | hv_ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), | |
166 | GHCB_PROTOCOL_MAX); | |
167 | ||
168 | /* Write ghcb page back after negotiating protocol. */ | |
169 | wr_ghcb_msr(ghcb_gpa); | |
170 | VMGEXIT(); | |
171 | ||
172 | return true; | |
173 | } | |
174 | ||
faff4406 TL |
175 | void hv_ghcb_msr_write(u64 msr, u64 value) |
176 | { | |
177 | union hv_ghcb *hv_ghcb; | |
178 | void **ghcb_base; | |
179 | unsigned long flags; | |
faff4406 TL |
180 | |
181 | if (!hv_ghcb_pg) | |
182 | return; | |
183 | ||
184 | WARN_ON(in_nmi()); | |
185 | ||
186 | local_irq_save(flags); | |
187 | ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg); | |
188 | hv_ghcb = (union hv_ghcb *)*ghcb_base; | |
189 | if (!hv_ghcb) { | |
190 | local_irq_restore(flags); | |
191 | return; | |
192 | } | |
193 | ||
194 | ghcb_set_rcx(&hv_ghcb->ghcb, msr); | |
195 | ghcb_set_rax(&hv_ghcb->ghcb, lower_32_bits(value)); | |
196 | ghcb_set_rdx(&hv_ghcb->ghcb, upper_32_bits(value)); | |
197 | ||
49d6a3c0 | 198 | if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 1, 0)) |
faff4406 TL |
199 | pr_warn("Fail to write msr via ghcb %llx.\n", msr); |
200 | ||
201 | local_irq_restore(flags); | |
202 | } | |
203 | EXPORT_SYMBOL_GPL(hv_ghcb_msr_write); | |
204 | ||
205 | void hv_ghcb_msr_read(u64 msr, u64 *value) | |
206 | { | |
207 | union hv_ghcb *hv_ghcb; | |
208 | void **ghcb_base; | |
209 | unsigned long flags; | |
faff4406 TL |
210 | |
211 | /* Check size of union hv_ghcb here. */ | |
212 | BUILD_BUG_ON(sizeof(union hv_ghcb) != HV_HYP_PAGE_SIZE); | |
213 | ||
214 | if (!hv_ghcb_pg) | |
215 | return; | |
216 | ||
217 | WARN_ON(in_nmi()); | |
218 | ||
219 | local_irq_save(flags); | |
220 | ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg); | |
221 | hv_ghcb = (union hv_ghcb *)*ghcb_base; | |
222 | if (!hv_ghcb) { | |
223 | local_irq_restore(flags); | |
224 | return; | |
225 | } | |
226 | ||
227 | ghcb_set_rcx(&hv_ghcb->ghcb, msr); | |
49d6a3c0 | 228 | if (hv_ghcb_hv_call(&hv_ghcb->ghcb, SVM_EXIT_MSR, 0, 0)) |
faff4406 TL |
229 | pr_warn("Fail to read msr via ghcb %llx.\n", msr); |
230 | else | |
231 | *value = (u64)lower_32_bits(hv_ghcb->ghcb.save.rax) | |
232 | | ((u64)lower_32_bits(hv_ghcb->ghcb.save.rdx) << 32); | |
233 | local_irq_restore(flags); | |
234 | } | |
235 | EXPORT_SYMBOL_GPL(hv_ghcb_msr_read); | |
236 | #endif | |
237 | ||
238 | enum hv_isolation_type hv_get_isolation_type(void) | |
239 | { | |
240 | if (!(ms_hyperv.priv_high & HV_ISOLATION)) | |
241 | return HV_ISOLATION_TYPE_NONE; | |
242 | return FIELD_GET(HV_ISOLATION_TYPE, ms_hyperv.isolation_config_b); | |
243 | } | |
244 | EXPORT_SYMBOL_GPL(hv_get_isolation_type); | |
245 | ||
246 | /* | |
247 | * hv_is_isolation_supported - Check system runs in the Hyper-V | |
248 | * isolation VM. | |
249 | */ | |
250 | bool hv_is_isolation_supported(void) | |
251 | { | |
252 | if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) | |
253 | return false; | |
254 | ||
255 | if (!hypervisor_is_type(X86_HYPER_MS_HYPERV)) | |
256 | return false; | |
257 | ||
258 | return hv_get_isolation_type() != HV_ISOLATION_TYPE_NONE; | |
259 | } | |
260 | ||
261 | DEFINE_STATIC_KEY_FALSE(isolation_type_snp); | |
262 | ||
263 | /* | |
264 | * hv_isolation_type_snp - Check system runs in the AMD SEV-SNP based | |
265 | * isolation VM. | |
266 | */ | |
267 | bool hv_isolation_type_snp(void) | |
268 | { | |
269 | return static_branch_unlikely(&isolation_type_snp); | |
270 | } | |
810a5212 TL |
271 | |
272 | /* | |
273 | * hv_mark_gpa_visibility - Set pages visible to host via hvcall. | |
274 | * | |
275 | * In Isolation VM, all guest memory is encrypted from host and guest | |
276 | * needs to set memory visible to host via hvcall before sharing memory | |
277 | * with host. | |
278 | */ | |
279 | static int hv_mark_gpa_visibility(u16 count, const u64 pfn[], | |
280 | enum hv_mem_host_visibility visibility) | |
281 | { | |
282 | struct hv_gpa_range_for_visibility **input_pcpu, *input; | |
283 | u16 pages_processed; | |
284 | u64 hv_status; | |
285 | unsigned long flags; | |
286 | ||
287 | /* no-op if partition isolation is not enabled */ | |
288 | if (!hv_is_isolation_supported()) | |
289 | return 0; | |
290 | ||
291 | if (count > HV_MAX_MODIFY_GPA_REP_COUNT) { | |
292 | pr_err("Hyper-V: GPA count:%d exceeds supported:%lu\n", count, | |
293 | HV_MAX_MODIFY_GPA_REP_COUNT); | |
294 | return -EINVAL; | |
295 | } | |
296 | ||
297 | local_irq_save(flags); | |
298 | input_pcpu = (struct hv_gpa_range_for_visibility **) | |
299 | this_cpu_ptr(hyperv_pcpu_input_arg); | |
300 | input = *input_pcpu; | |
301 | if (unlikely(!input)) { | |
302 | local_irq_restore(flags); | |
303 | return -EINVAL; | |
304 | } | |
305 | ||
306 | input->partition_id = HV_PARTITION_ID_SELF; | |
307 | input->host_visibility = visibility; | |
308 | input->reserved0 = 0; | |
309 | input->reserved1 = 0; | |
310 | memcpy((void *)input->gpa_page_list, pfn, count * sizeof(*pfn)); | |
311 | hv_status = hv_do_rep_hypercall( | |
312 | HVCALL_MODIFY_SPARSE_GPA_PAGE_HOST_VISIBILITY, count, | |
313 | 0, input, &pages_processed); | |
314 | local_irq_restore(flags); | |
315 | ||
316 | if (hv_result_success(hv_status)) | |
317 | return 0; | |
318 | else | |
319 | return -EFAULT; | |
320 | } | |
321 | ||
322 | /* | |
323 | * hv_set_mem_host_visibility - Set specified memory visible to host. | |
324 | * | |
325 | * In Isolation VM, all guest memory is encrypted from host and guest | |
326 | * needs to set memory visible to host via hvcall before sharing memory | |
327 | * with host. This function works as wrap of hv_mark_gpa_visibility() | |
328 | * with memory base and size. | |
329 | */ | |
330 | int hv_set_mem_host_visibility(unsigned long kbuffer, int pagecount, bool visible) | |
331 | { | |
332 | enum hv_mem_host_visibility visibility = visible ? | |
333 | VMBUS_PAGE_VISIBLE_READ_WRITE : VMBUS_PAGE_NOT_VISIBLE; | |
334 | u64 *pfn_array; | |
335 | int ret = 0; | |
336 | int i, pfn; | |
337 | ||
338 | if (!hv_is_isolation_supported() || !hv_hypercall_pg) | |
339 | return 0; | |
340 | ||
341 | pfn_array = kmalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL); | |
342 | if (!pfn_array) | |
343 | return -ENOMEM; | |
344 | ||
345 | for (i = 0, pfn = 0; i < pagecount; i++) { | |
346 | pfn_array[pfn] = virt_to_hvpfn((void *)kbuffer + i * HV_HYP_PAGE_SIZE); | |
347 | pfn++; | |
348 | ||
349 | if (pfn == HV_MAX_MODIFY_GPA_REP_COUNT || i == pagecount - 1) { | |
350 | ret = hv_mark_gpa_visibility(pfn, pfn_array, | |
351 | visibility); | |
352 | if (ret) | |
353 | goto err_free_pfn_array; | |
354 | pfn = 0; | |
355 | } | |
356 | } | |
357 | ||
358 | err_free_pfn_array: | |
359 | kfree(pfn_array); | |
360 | return ret; | |
361 | } | |
846da38d TL |
362 | |
363 | /* | |
364 | * hv_map_memory - map memory to extra space in the AMD SEV-SNP Isolation VM. | |
365 | */ | |
366 | void *hv_map_memory(void *addr, unsigned long size) | |
367 | { | |
368 | unsigned long *pfns = kcalloc(size / PAGE_SIZE, | |
369 | sizeof(unsigned long), GFP_KERNEL); | |
370 | void *vaddr; | |
371 | int i; | |
372 | ||
373 | if (!pfns) | |
374 | return NULL; | |
375 | ||
376 | for (i = 0; i < size / PAGE_SIZE; i++) | |
377 | pfns[i] = vmalloc_to_pfn(addr + i * PAGE_SIZE) + | |
378 | (ms_hyperv.shared_gpa_boundary >> PAGE_SHIFT); | |
379 | ||
380 | vaddr = vmap_pfn(pfns, size / PAGE_SIZE, PAGE_KERNEL_IO); | |
381 | kfree(pfns); | |
382 | ||
383 | return vaddr; | |
384 | } | |
385 | ||
386 | void hv_unmap_memory(void *addr) | |
387 | { | |
388 | vunmap(addr); | |
389 | } |