Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
e08cae41 PA |
2 | #ifndef _ASM_X86_MSHYPER_H |
3 | #define _ASM_X86_MSHYPER_H | |
a2a47c6c | 4 | |
e08cae41 | 5 | #include <linux/types.h> |
26fcd952 | 6 | #include <linux/atomic.h> |
806c8927 | 7 | #include <linux/nmi.h> |
fc53662f | 8 | #include <asm/io.h> |
5a485803 | 9 | #include <asm/hyperv-tlfs.h> |
e70e5892 | 10 | #include <asm/nospec-branch.h> |
e08cae41 PA |
11 | |
12 | struct ms_hyperv_info { | |
13 | u32 features; | |
cc2dd402 | 14 | u32 misc_features; |
e08cae41 | 15 | u32 hints; |
5431390b | 16 | u32 nested_features; |
dd018597 VK |
17 | u32 max_vp_index; |
18 | u32 max_lp_index; | |
e08cae41 PA |
19 | }; |
20 | ||
21 | extern struct ms_hyperv_info ms_hyperv; | |
a2a47c6c | 22 | |
352c9624 S |
23 | |
24 | /* | |
415bd1cd | 25 | * Generate the guest ID. |
352c9624 S |
26 | */ |
27 | ||
28 | static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, | |
29 | __u64 d_info2) | |
30 | { | |
31 | __u64 guest_id = 0; | |
32 | ||
9b06e101 | 33 | guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); |
352c9624 S |
34 | guest_id |= (d_info1 << 48); |
35 | guest_id |= (kernel_version << 16); | |
36 | guest_id |= d_info2; | |
37 | ||
38 | return guest_id; | |
39 | } | |
40 | ||
e810e48c S |
41 | |
42 | /* Free the message slot and signal end-of-message if required */ | |
43 | static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) | |
44 | { | |
45 | /* | |
46 | * On crash we're reading some other CPU's message page and we need | |
47 | * to be careful: this other CPU may already had cleared the header | |
48 | * and the host may already had delivered some other message there. | |
49 | * In case we blindly write msg->header.message_type we're going | |
50 | * to lose it. We can still lose a message of the same type but | |
51 | * we count on the fact that there can only be one | |
52 | * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages | |
53 | * on crash. | |
54 | */ | |
55 | if (cmpxchg(&msg->header.message_type, old_msg_type, | |
56 | HVMSG_NONE) != old_msg_type) | |
57 | return; | |
58 | ||
59 | /* | |
60 | * Make sure the write to MessageType (ie set to | |
61 | * HVMSG_NONE) happens before we read the | |
62 | * MessagePending and EOMing. Otherwise, the EOMing | |
63 | * will not deliver any more messages since there is | |
64 | * no empty slot | |
65 | */ | |
66 | mb(); | |
67 | ||
68 | if (msg->header.message_flags.msg_pending) { | |
69 | /* | |
70 | * This will cause message queue rescan to | |
71 | * possibly deliver another msg from the | |
72 | * hypervisor | |
73 | */ | |
74 | wrmsrl(HV_X64_MSR_EOM, 0); | |
75 | } | |
76 | } | |
77 | ||
d5116b40 S |
78 | #define hv_init_timer(timer, tick) wrmsrl(timer, tick) |
79 | #define hv_init_timer_config(config, val) wrmsrl(config, val) | |
80 | ||
155e4a2f S |
81 | #define hv_get_simp(val) rdmsrl(HV_X64_MSR_SIMP, val) |
82 | #define hv_set_simp(val) wrmsrl(HV_X64_MSR_SIMP, val) | |
83 | ||
8e307bf8 S |
84 | #define hv_get_siefp(val) rdmsrl(HV_X64_MSR_SIEFP, val) |
85 | #define hv_set_siefp(val) wrmsrl(HV_X64_MSR_SIEFP, val) | |
86 | ||
06d1d98a S |
87 | #define hv_get_synic_state(val) rdmsrl(HV_X64_MSR_SCONTROL, val) |
88 | #define hv_set_synic_state(val) wrmsrl(HV_X64_MSR_SCONTROL, val) | |
89 | ||
7297ff0c S |
90 | #define hv_get_vp_index(index) rdmsrl(HV_X64_MSR_VP_INDEX, index) |
91 | ||
37e11d5c S |
92 | #define hv_get_synint_state(int_num, val) rdmsrl(int_num, val) |
93 | #define hv_set_synint_state(int_num, val) wrmsrl(int_num, val) | |
94 | ||
bc2b0331 | 95 | void hyperv_callback_vector(void); |
93286261 | 96 | void hyperv_reenlightenment_vector(void); |
cf910e83 SA |
97 | #ifdef CONFIG_TRACING |
98 | #define trace_hyperv_callback_vector hyperv_callback_vector | |
99 | #endif | |
bc2b0331 | 100 | void hyperv_vector_handler(struct pt_regs *regs); |
76d388cd TG |
101 | void hv_setup_vmbus_irq(void (*handler)(void)); |
102 | void hv_remove_vmbus_irq(void); | |
bc2b0331 | 103 | |
2517281d VK |
104 | void hv_setup_kexec_handler(void (*handler)(void)); |
105 | void hv_remove_kexec_handler(void); | |
b4370df2 VK |
106 | void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); |
107 | void hv_remove_crash_handler(void); | |
8730046c | 108 | |
248e742a MK |
109 | /* |
110 | * Routines for stimer0 Direct Mode handling. | |
111 | * On x86/x64, there are no percpu actions to take. | |
112 | */ | |
113 | void hv_stimer0_vector_handler(struct pt_regs *regs); | |
114 | void hv_stimer0_callback_vector(void); | |
115 | int hv_setup_stimer0_irq(int *irq, int *vector, void (*handler)(void)); | |
116 | void hv_remove_stimer0_irq(int irq); | |
117 | ||
118 | static inline void hv_enable_stimer0_percpu_irq(int irq) {} | |
119 | static inline void hv_disable_stimer0_percpu_irq(int irq) {} | |
120 | ||
121 | ||
8730046c | 122 | #if IS_ENABLED(CONFIG_HYPERV) |
dee863b5 | 123 | extern struct clocksource *hyperv_cs; |
fc53662f | 124 | extern void *hv_hypercall_pg; |
68bb7bfb | 125 | extern void __percpu **hyperv_pcpu_input_arg; |
fc53662f VK |
126 | |
127 | static inline u64 hv_do_hypercall(u64 control, void *input, void *output) | |
128 | { | |
129 | u64 input_address = input ? virt_to_phys(input) : 0; | |
130 | u64 output_address = output ? virt_to_phys(output) : 0; | |
131 | u64 hv_status; | |
fc53662f VK |
132 | |
133 | #ifdef CONFIG_X86_64 | |
134 | if (!hv_hypercall_pg) | |
135 | return U64_MAX; | |
136 | ||
137 | __asm__ __volatile__("mov %4, %%r8\n" | |
e70e5892 | 138 | CALL_NOSPEC |
f5caf621 | 139 | : "=a" (hv_status), ASM_CALL_CONSTRAINT, |
fc53662f | 140 | "+c" (control), "+d" (input_address) |
e70e5892 DW |
141 | : "r" (output_address), |
142 | THUNK_TARGET(hv_hypercall_pg) | |
fc53662f VK |
143 | : "cc", "memory", "r8", "r9", "r10", "r11"); |
144 | #else | |
145 | u32 input_address_hi = upper_32_bits(input_address); | |
146 | u32 input_address_lo = lower_32_bits(input_address); | |
147 | u32 output_address_hi = upper_32_bits(output_address); | |
148 | u32 output_address_lo = lower_32_bits(output_address); | |
149 | ||
150 | if (!hv_hypercall_pg) | |
151 | return U64_MAX; | |
152 | ||
e70e5892 | 153 | __asm__ __volatile__(CALL_NOSPEC |
fc53662f | 154 | : "=A" (hv_status), |
f5caf621 | 155 | "+c" (input_address_lo), ASM_CALL_CONSTRAINT |
fc53662f VK |
156 | : "A" (control), |
157 | "b" (input_address_hi), | |
158 | "D"(output_address_hi), "S"(output_address_lo), | |
e70e5892 | 159 | THUNK_TARGET(hv_hypercall_pg) |
fc53662f VK |
160 | : "cc", "memory"); |
161 | #endif /* !x86_64 */ | |
162 | return hv_status; | |
163 | } | |
dee863b5 | 164 | |
6a8edbd0 VK |
165 | /* Fast hypercall with 8 bytes of input and no output */ |
166 | static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) | |
167 | { | |
168 | u64 hv_status, control = (u64)code | HV_HYPERCALL_FAST_BIT; | |
6a8edbd0 VK |
169 | |
170 | #ifdef CONFIG_X86_64 | |
171 | { | |
e70e5892 | 172 | __asm__ __volatile__(CALL_NOSPEC |
f5caf621 | 173 | : "=a" (hv_status), ASM_CALL_CONSTRAINT, |
6a8edbd0 | 174 | "+c" (control), "+d" (input1) |
e70e5892 | 175 | : THUNK_TARGET(hv_hypercall_pg) |
6a8edbd0 VK |
176 | : "cc", "r8", "r9", "r10", "r11"); |
177 | } | |
178 | #else | |
179 | { | |
180 | u32 input1_hi = upper_32_bits(input1); | |
181 | u32 input1_lo = lower_32_bits(input1); | |
182 | ||
e70e5892 | 183 | __asm__ __volatile__ (CALL_NOSPEC |
6a8edbd0 VK |
184 | : "=A"(hv_status), |
185 | "+c"(input1_lo), | |
f5caf621 | 186 | ASM_CALL_CONSTRAINT |
6a8edbd0 VK |
187 | : "A" (control), |
188 | "b" (input1_hi), | |
e70e5892 | 189 | THUNK_TARGET(hv_hypercall_pg) |
6a8edbd0 VK |
190 | : "cc", "edi", "esi"); |
191 | } | |
192 | #endif | |
193 | return hv_status; | |
194 | } | |
195 | ||
806c8927 VK |
196 | /* |
197 | * Rep hypercalls. Callers of this functions are supposed to ensure that | |
198 | * rep_count and varhead_size comply with Hyper-V hypercall definition. | |
199 | */ | |
200 | static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, | |
201 | void *input, void *output) | |
202 | { | |
203 | u64 control = code; | |
204 | u64 status; | |
205 | u16 rep_comp; | |
206 | ||
207 | control |= (u64)varhead_size << HV_HYPERCALL_VARHEAD_OFFSET; | |
208 | control |= (u64)rep_count << HV_HYPERCALL_REP_COMP_OFFSET; | |
209 | ||
210 | do { | |
211 | status = hv_do_hypercall(control, input, output); | |
212 | if ((status & HV_HYPERCALL_RESULT_MASK) != HV_STATUS_SUCCESS) | |
213 | return status; | |
214 | ||
215 | /* Bits 32-43 of status have 'Reps completed' data. */ | |
216 | rep_comp = (status & HV_HYPERCALL_REP_COMP_MASK) >> | |
217 | HV_HYPERCALL_REP_COMP_OFFSET; | |
218 | ||
219 | control &= ~HV_HYPERCALL_REP_START_MASK; | |
220 | control |= (u64)rep_comp << HV_HYPERCALL_REP_START_OFFSET; | |
221 | ||
222 | touch_nmi_watchdog(); | |
223 | } while (rep_comp < rep_count); | |
224 | ||
225 | return status; | |
226 | } | |
227 | ||
7415aea6 VK |
228 | /* |
229 | * Hypervisor's notion of virtual processor ID is different from | |
230 | * Linux' notion of CPU ID. This information can only be retrieved | |
231 | * in the context of the calling CPU. Setup a map for easy access | |
232 | * to this information. | |
233 | */ | |
234 | extern u32 *hv_vp_index; | |
a3b74243 | 235 | extern u32 hv_max_vp_index; |
a46d15cc VK |
236 | extern struct hv_vp_assist_page **hv_vp_assist_page; |
237 | ||
238 | static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) | |
239 | { | |
240 | if (!hv_vp_assist_page) | |
241 | return NULL; | |
242 | ||
243 | return hv_vp_assist_page[cpu]; | |
244 | } | |
7415aea6 VK |
245 | |
246 | /** | |
247 | * hv_cpu_number_to_vp_number() - Map CPU to VP. | |
248 | * @cpu_number: CPU number in Linux terms | |
249 | * | |
250 | * This function returns the mapping between the Linux processor | |
251 | * number and the hypervisor's virtual processor number, useful | |
252 | * in making hypercalls and such that talk about specific | |
253 | * processors. | |
254 | * | |
255 | * Return: Virtual processor number in Hyper-V terms | |
256 | */ | |
257 | static inline int hv_cpu_number_to_vp_number(int cpu_number) | |
258 | { | |
259 | return hv_vp_index[cpu_number]; | |
260 | } | |
dee863b5 | 261 | |
366f03b0 S |
262 | static inline int cpumask_to_vpset(struct hv_vpset *vpset, |
263 | const struct cpumask *cpus) | |
264 | { | |
265 | int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; | |
266 | ||
267 | /* valid_bank_mask can represent up to 64 banks */ | |
268 | if (hv_max_vp_index / 64 >= 64) | |
269 | return 0; | |
270 | ||
271 | /* | |
272 | * Clear all banks up to the maximum possible bank as hv_flush_pcpu_ex | |
273 | * structs are not cleared between calls, we risk flushing unneeded | |
274 | * vCPUs otherwise. | |
275 | */ | |
276 | for (vcpu_bank = 0; vcpu_bank <= hv_max_vp_index / 64; vcpu_bank++) | |
277 | vpset->bank_contents[vcpu_bank] = 0; | |
278 | ||
279 | /* | |
280 | * Some banks may end up being empty but this is acceptable. | |
281 | */ | |
282 | for_each_cpu(cpu, cpus) { | |
283 | vcpu = hv_cpu_number_to_vp_number(cpu); | |
284 | vcpu_bank = vcpu / 64; | |
285 | vcpu_offset = vcpu % 64; | |
286 | __set_bit(vcpu_offset, (unsigned long *) | |
287 | &vpset->bank_contents[vcpu_bank]); | |
288 | if (vcpu_bank >= nr_bank) | |
289 | nr_bank = vcpu_bank + 1; | |
290 | } | |
291 | vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); | |
292 | return nr_bank; | |
293 | } | |
294 | ||
6b48cb5f | 295 | void __init hyperv_init(void); |
2ffd9e33 | 296 | void hyperv_setup_mmu_ops(void); |
7ed4325a | 297 | void hyperv_report_panic(struct pt_regs *regs, long err); |
4a5f3cde | 298 | bool hv_is_hyperv_initialized(void); |
d6f3609d | 299 | void hyperv_cleanup(void); |
93286261 VK |
300 | |
301 | void hyperv_reenlightenment_intr(struct pt_regs *regs); | |
302 | void set_hv_tscchange_cb(void (*cb)(void)); | |
303 | void clear_hv_tscchange_cb(void); | |
304 | void hyperv_stop_tsc_emulation(void); | |
6b48cb5f | 305 | void hv_apic_init(void); |
79cadff2 VK |
306 | #else /* CONFIG_HYPERV */ |
307 | static inline void hyperv_init(void) {} | |
4a5f3cde | 308 | static inline bool hv_is_hyperv_initialized(void) { return false; } |
79cadff2 | 309 | static inline void hyperv_cleanup(void) {} |
2ffd9e33 | 310 | static inline void hyperv_setup_mmu_ops(void) {} |
93286261 VK |
311 | static inline void set_hv_tscchange_cb(void (*cb)(void)) {} |
312 | static inline void clear_hv_tscchange_cb(void) {} | |
313 | static inline void hyperv_stop_tsc_emulation(void) {}; | |
6b48cb5f | 314 | static inline void hv_apic_init(void) {} |
a46d15cc VK |
315 | static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu) |
316 | { | |
317 | return NULL; | |
318 | } | |
79cadff2 VK |
319 | #endif /* CONFIG_HYPERV */ |
320 | ||
bd2a9ada VK |
321 | #ifdef CONFIG_HYPERV_TSCPAGE |
322 | struct ms_hyperv_tsc_page *hv_get_tsc_page(void); | |
e2768eaa VK |
323 | static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, |
324 | u64 *cur_tsc) | |
0733379b | 325 | { |
e2768eaa | 326 | u64 scale, offset; |
0733379b VK |
327 | u32 sequence; |
328 | ||
329 | /* | |
330 | * The protocol for reading Hyper-V TSC page is specified in Hypervisor | |
331 | * Top-Level Functional Specification ver. 3.0 and above. To get the | |
332 | * reference time we must do the following: | |
333 | * - READ ReferenceTscSequence | |
334 | * A special '0' value indicates the time source is unreliable and we | |
335 | * need to use something else. The currently published specification | |
336 | * versions (up to 4.0b) contain a mistake and wrongly claim '-1' | |
337 | * instead of '0' as the special value, see commit c35b82ef0294. | |
338 | * - ReferenceTime = | |
339 | * ((RDTSC() * ReferenceTscScale) >> 64) + ReferenceTscOffset | |
340 | * - READ ReferenceTscSequence again. In case its value has changed | |
341 | * since our first reading we need to discard ReferenceTime and repeat | |
342 | * the whole sequence as the hypervisor was updating the page in | |
343 | * between. | |
344 | */ | |
345 | do { | |
346 | sequence = READ_ONCE(tsc_pg->tsc_sequence); | |
347 | if (!sequence) | |
348 | return U64_MAX; | |
349 | /* | |
350 | * Make sure we read sequence before we read other values from | |
351 | * TSC page. | |
352 | */ | |
353 | smp_rmb(); | |
354 | ||
355 | scale = READ_ONCE(tsc_pg->tsc_scale); | |
356 | offset = READ_ONCE(tsc_pg->tsc_offset); | |
e2768eaa | 357 | *cur_tsc = rdtsc_ordered(); |
0733379b VK |
358 | |
359 | /* | |
360 | * Make sure we read sequence after we read all other values | |
361 | * from TSC page. | |
362 | */ | |
363 | smp_rmb(); | |
364 | ||
365 | } while (READ_ONCE(tsc_pg->tsc_sequence) != sequence); | |
366 | ||
e2768eaa VK |
367 | return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset; |
368 | } | |
369 | ||
370 | static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) | |
371 | { | |
372 | u64 cur_tsc; | |
373 | ||
374 | return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc); | |
0733379b VK |
375 | } |
376 | ||
bd2a9ada VK |
377 | #else |
378 | static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void) | |
379 | { | |
380 | return NULL; | |
381 | } | |
e2768eaa VK |
382 | |
383 | static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, | |
384 | u64 *cur_tsc) | |
385 | { | |
386 | BUG(); | |
387 | return U64_MAX; | |
388 | } | |
bd2a9ada | 389 | #endif |
a2a47c6c | 390 | #endif |