Commit | Line | Data |
---|---|---|
790c73f6 GOC |
1 | /* KVM paravirtual clock driver. A clocksource implementation |
2 | Copyright (C) 2008 Glauber de Oliveira Costa, Red Hat Inc. | |
3 | ||
4 | This program is free software; you can redistribute it and/or modify | |
5 | it under the terms of the GNU General Public License as published by | |
6 | the Free Software Foundation; either version 2 of the License, or | |
7 | (at your option) any later version. | |
8 | ||
9 | This program is distributed in the hope that it will be useful, | |
10 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | GNU General Public License for more details. | |
13 | ||
14 | You should have received a copy of the GNU General Public License | |
15 | along with this program; if not, write to the Free Software | |
16 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
17 | */ | |
18 | ||
19 | #include <linux/clocksource.h> | |
20 | #include <linux/kvm_para.h> | |
f6e16d5a | 21 | #include <asm/pvclock.h> |
790c73f6 GOC |
22 | #include <asm/msr.h> |
23 | #include <asm/apic.h> | |
24 | #include <linux/percpu.h> | |
3b5d56b9 | 25 | #include <linux/hardirq.h> |
0ad83caa | 26 | #include <linux/sched.h> |
e6017571 | 27 | #include <linux/sched/clock.h> |
368a540e | 28 | #include <linux/mm.h> |
736decac | 29 | |
819aeee0 | 30 | #include <asm/mem_encrypt.h> |
736decac | 31 | #include <asm/x86_init.h> |
1e977aa1 | 32 | #include <asm/reboot.h> |
f4066c2b | 33 | #include <asm/kvmclock.h> |
790c73f6 | 34 | |
404f6aac | 35 | static int kvmclock __ro_after_init = 1; |
838815a7 GC |
36 | static int msr_kvm_system_time = MSR_KVM_SYSTEM_TIME; |
37 | static int msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK; | |
a5a1d1c2 | 38 | static u64 kvm_sched_clock_offset; |
790c73f6 GOC |
39 | |
40 | static int parse_no_kvmclock(char *arg) | |
41 | { | |
42 | kvmclock = 0; | |
43 | return 0; | |
44 | } | |
45 | early_param("no-kvmclock", parse_no_kvmclock); | |
46 | ||
368a540e PT |
47 | /* Aligned to page sizes to match whats mapped via vsyscalls to userspace */ |
48 | #define HV_CLOCK_SIZE (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS) | |
368a540e PT |
49 | |
50 | static u8 hv_clock_mem[PAGE_ALIGN(HV_CLOCK_SIZE)] __aligned(PAGE_SIZE); | |
368a540e | 51 | |
790c73f6 | 52 | /* The hypervisor will put information about time periodically here */ |
3dc4f7cf | 53 | static struct pvclock_vsyscall_time_info *hv_clock; |
7ef363a3 | 54 | static struct pvclock_wall_clock wall_clock; |
790c73f6 | 55 | |
790c73f6 GOC |
56 | /* |
57 | * The wallclock is the time of day when we booted. Since then, some time may | |
58 | * have elapsed since the hypervisor wrote the data. So we try to account for | |
59 | * that with system time | |
60 | */ | |
e27c4929 | 61 | static void kvm_get_wallclock(struct timespec64 *now) |
790c73f6 | 62 | { |
f6e16d5a | 63 | struct pvclock_vcpu_time_info *vcpu_time; |
790c73f6 | 64 | int low, high; |
7069ed67 | 65 | int cpu; |
790c73f6 | 66 | |
7ef363a3 TG |
67 | low = (int)slow_virt_to_phys(&wall_clock); |
68 | high = ((u64)slow_virt_to_phys(&wall_clock) >> 32); | |
838815a7 GC |
69 | |
70 | native_write_msr(msr_kvm_wall_clock, low, high); | |
790c73f6 | 71 | |
c6338ce4 | 72 | cpu = get_cpu(); |
7069ed67 | 73 | |
3dc4f7cf | 74 | vcpu_time = &hv_clock[cpu].pvti; |
7ef363a3 | 75 | pvclock_read_wallclock(&wall_clock, vcpu_time, now); |
7069ed67 | 76 | |
c6338ce4 | 77 | put_cpu(); |
790c73f6 GOC |
78 | } |
79 | ||
e27c4929 | 80 | static int kvm_set_wallclock(const struct timespec64 *now) |
790c73f6 | 81 | { |
00875520 | 82 | return -ENODEV; |
790c73f6 GOC |
83 | } |
84 | ||
a5a1d1c2 | 85 | static u64 kvm_clock_read(void) |
790c73f6 | 86 | { |
f6e16d5a | 87 | struct pvclock_vcpu_time_info *src; |
a5a1d1c2 | 88 | u64 ret; |
7069ed67 | 89 | int cpu; |
790c73f6 | 90 | |
95ef1e52 | 91 | preempt_disable_notrace(); |
7069ed67 | 92 | cpu = smp_processor_id(); |
3dc4f7cf | 93 | src = &hv_clock[cpu].pvti; |
f6e16d5a | 94 | ret = pvclock_clocksource_read(src); |
95ef1e52 | 95 | preempt_enable_notrace(); |
f6e16d5a | 96 | return ret; |
790c73f6 | 97 | } |
f6e16d5a | 98 | |
a5a1d1c2 | 99 | static u64 kvm_clock_get_cycles(struct clocksource *cs) |
8e19608e MD |
100 | { |
101 | return kvm_clock_read(); | |
102 | } | |
103 | ||
a5a1d1c2 | 104 | static u64 kvm_sched_clock_read(void) |
72c930dc RK |
105 | { |
106 | return kvm_clock_read() - kvm_sched_clock_offset; | |
107 | } | |
108 | ||
109 | static inline void kvm_sched_clock_init(bool stable) | |
110 | { | |
111 | if (!stable) { | |
112 | pv_time_ops.sched_clock = kvm_clock_read; | |
acb04058 | 113 | clear_sched_clock_stable(); |
72c930dc RK |
114 | return; |
115 | } | |
116 | ||
117 | kvm_sched_clock_offset = kvm_clock_read(); | |
118 | pv_time_ops.sched_clock = kvm_sched_clock_read; | |
72c930dc RK |
119 | |
120 | printk(KERN_INFO "kvm-clock: using sched offset of %llu cycles\n", | |
121 | kvm_sched_clock_offset); | |
122 | ||
123 | BUILD_BUG_ON(sizeof(kvm_sched_clock_offset) > | |
124 | sizeof(((struct pvclock_vcpu_time_info *)NULL)->system_time)); | |
125 | } | |
126 | ||
0293615f GC |
127 | /* |
128 | * If we don't do that, there is the possibility that the guest | |
129 | * will calibrate under heavy load - thus, getting a lower lpj - | |
130 | * and execute the delays themselves without load. This is wrong, | |
131 | * because no delay loop can finish beforehand. | |
132 | * Any heuristics is subject to fail, because ultimately, a large | |
133 | * poll of guests can be running and trouble each other. So we preset | |
134 | * lpj here | |
135 | */ | |
136 | static unsigned long kvm_get_tsc_khz(void) | |
137 | { | |
e93353c9 | 138 | struct pvclock_vcpu_time_info *src; |
7069ed67 MT |
139 | int cpu; |
140 | unsigned long tsc_khz; | |
141 | ||
c6338ce4 | 142 | cpu = get_cpu(); |
3dc4f7cf | 143 | src = &hv_clock[cpu].pvti; |
7069ed67 | 144 | tsc_khz = pvclock_tsc_khz(src); |
c6338ce4 | 145 | put_cpu(); |
e10f7805 | 146 | setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ); |
7069ed67 | 147 | return tsc_khz; |
0293615f GC |
148 | } |
149 | ||
150 | static void kvm_get_preset_lpj(void) | |
151 | { | |
0293615f GC |
152 | unsigned long khz; |
153 | u64 lpj; | |
154 | ||
e93353c9 | 155 | khz = kvm_get_tsc_khz(); |
0293615f GC |
156 | |
157 | lpj = ((u64)khz * 1000); | |
158 | do_div(lpj, HZ); | |
159 | preset_lpj = lpj; | |
160 | } | |
161 | ||
3b5d56b9 EM |
162 | bool kvm_check_and_clear_guest_paused(void) |
163 | { | |
164 | bool ret = false; | |
165 | struct pvclock_vcpu_time_info *src; | |
7069ed67 MT |
166 | int cpu = smp_processor_id(); |
167 | ||
168 | if (!hv_clock) | |
169 | return ret; | |
3b5d56b9 | 170 | |
3dc4f7cf | 171 | src = &hv_clock[cpu].pvti; |
3b5d56b9 | 172 | if ((src->flags & PVCLOCK_GUEST_STOPPED) != 0) { |
7069ed67 | 173 | src->flags &= ~PVCLOCK_GUEST_STOPPED; |
d63285e9 | 174 | pvclock_touch_watchdogs(); |
3b5d56b9 EM |
175 | ret = true; |
176 | } | |
177 | ||
178 | return ret; | |
179 | } | |
3b5d56b9 | 180 | |
f4066c2b | 181 | struct clocksource kvm_clock = { |
790c73f6 | 182 | .name = "kvm-clock", |
8e19608e | 183 | .read = kvm_clock_get_cycles, |
790c73f6 GOC |
184 | .rating = 400, |
185 | .mask = CLOCKSOURCE_MASK(64), | |
790c73f6 GOC |
186 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
187 | }; | |
f4066c2b | 188 | EXPORT_SYMBOL_GPL(kvm_clock); |
790c73f6 | 189 | |
ca3f1017 | 190 | int kvm_register_clock(char *txt) |
790c73f6 GOC |
191 | { |
192 | int cpu = smp_processor_id(); | |
19b6a85b | 193 | int low, high, ret; |
fe1140cc JK |
194 | struct pvclock_vcpu_time_info *src; |
195 | ||
196 | if (!hv_clock) | |
197 | return 0; | |
19b6a85b | 198 | |
fe1140cc | 199 | src = &hv_clock[cpu].pvti; |
5dfd486c DH |
200 | low = (int)slow_virt_to_phys(src) | 1; |
201 | high = ((u64)slow_virt_to_phys(src) >> 32); | |
19b6a85b | 202 | ret = native_write_msr_safe(msr_kvm_system_time, low, high); |
f6e16d5a GH |
203 | printk(KERN_INFO "kvm-clock: cpu %d, msr %x:%x, %s\n", |
204 | cpu, high, low, txt); | |
838815a7 | 205 | |
19b6a85b | 206 | return ret; |
790c73f6 GOC |
207 | } |
208 | ||
b74f05d6 MT |
209 | static void kvm_save_sched_clock_state(void) |
210 | { | |
211 | } | |
212 | ||
213 | static void kvm_restore_sched_clock_state(void) | |
214 | { | |
215 | kvm_register_clock("primary cpu clock, resume"); | |
216 | } | |
217 | ||
b8ba5f10 | 218 | #ifdef CONFIG_X86_LOCAL_APIC |
148f9bb8 | 219 | static void kvm_setup_secondary_clock(void) |
790c73f6 GOC |
220 | { |
221 | /* | |
222 | * Now that the first cpu already had this clocksource initialized, | |
223 | * we shouldn't fail. | |
224 | */ | |
f6e16d5a | 225 | WARN_ON(kvm_register_clock("secondary cpu clock")); |
790c73f6 | 226 | } |
b8ba5f10 | 227 | #endif |
790c73f6 | 228 | |
1e977aa1 GC |
229 | /* |
230 | * After the clock is registered, the host will keep writing to the | |
231 | * registered memory location. If the guest happens to shutdown, this memory | |
232 | * won't be valid. In cases like kexec, in which you install a new kernel, this | |
233 | * means a random memory location will be kept being written. So before any | |
6a6256f9 | 234 | * kind of shutdown from our side, we unregister the clock by writing anything |
1e977aa1 GC |
235 | * that does not have the 'enable' bit set in the msr |
236 | */ | |
2965faa5 | 237 | #ifdef CONFIG_KEXEC_CORE |
1e977aa1 GC |
238 | static void kvm_crash_shutdown(struct pt_regs *regs) |
239 | { | |
838815a7 | 240 | native_write_msr(msr_kvm_system_time, 0, 0); |
d910f5c1 | 241 | kvm_disable_steal_time(); |
1e977aa1 GC |
242 | native_machine_crash_shutdown(regs); |
243 | } | |
244 | #endif | |
245 | ||
246 | static void kvm_shutdown(void) | |
247 | { | |
838815a7 | 248 | native_write_msr(msr_kvm_system_time, 0, 0); |
d910f5c1 | 249 | kvm_disable_steal_time(); |
1e977aa1 GC |
250 | native_machine_shutdown(); |
251 | } | |
252 | ||
790c73f6 GOC |
253 | void __init kvmclock_init(void) |
254 | { | |
0ad83caa | 255 | struct pvclock_vcpu_time_info *vcpu_time; |
368a540e | 256 | int cpu; |
0ad83caa | 257 | u8 flags; |
ed55705d | 258 | |
790c73f6 GOC |
259 | if (!kvm_para_available()) |
260 | return; | |
261 | ||
838815a7 GC |
262 | if (kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) { |
263 | msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW; | |
264 | msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW; | |
265 | } else if (!(kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE))) | |
266 | return; | |
267 | ||
368a540e | 268 | hv_clock = (struct pvclock_vsyscall_time_info *)hv_clock_mem; |
7069ed67 | 269 | |
0d75de4a | 270 | if (kvm_register_clock("primary cpu clock")) { |
7069ed67 | 271 | hv_clock = NULL; |
838815a7 | 272 | return; |
7069ed67 | 273 | } |
72c930dc | 274 | |
819aeee0 BS |
275 | printk(KERN_INFO "kvm-clock: Using msrs %x and %x", |
276 | msr_kvm_system_time, msr_kvm_wall_clock); | |
277 | ||
94ffba48 RK |
278 | pvclock_set_pvti_cpu0_va(hv_clock); |
279 | ||
72c930dc RK |
280 | if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT)) |
281 | pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT); | |
282 | ||
283 | cpu = get_cpu(); | |
284 | vcpu_time = &hv_clock[cpu].pvti; | |
285 | flags = pvclock_read_flags(vcpu_time); | |
286 | ||
287 | kvm_sched_clock_init(flags & PVCLOCK_TSC_STABLE_BIT); | |
288 | put_cpu(); | |
289 | ||
838815a7 | 290 | x86_platform.calibrate_tsc = kvm_get_tsc_khz; |
a4497a86 | 291 | x86_platform.calibrate_cpu = kvm_get_tsc_khz; |
838815a7 GC |
292 | x86_platform.get_wallclock = kvm_get_wallclock; |
293 | x86_platform.set_wallclock = kvm_set_wallclock; | |
b8ba5f10 | 294 | #ifdef CONFIG_X86_LOCAL_APIC |
df156f90 | 295 | x86_cpuinit.early_percpu_clock_init = |
838815a7 | 296 | kvm_setup_secondary_clock; |
b8ba5f10 | 297 | #endif |
b74f05d6 MT |
298 | x86_platform.save_sched_clock_state = kvm_save_sched_clock_state; |
299 | x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state; | |
838815a7 | 300 | machine_ops.shutdown = kvm_shutdown; |
2965faa5 | 301 | #ifdef CONFIG_KEXEC_CORE |
838815a7 | 302 | machine_ops.crash_shutdown = kvm_crash_shutdown; |
1e977aa1 | 303 | #endif |
838815a7 | 304 | kvm_get_preset_lpj(); |
b01cc1b0 | 305 | clocksource_register_hz(&kvm_clock, NSEC_PER_SEC); |
838815a7 | 306 | pv_info.name = "KVM"; |
790c73f6 | 307 | } |
3dc4f7cf MT |
308 | |
309 | int __init kvm_setup_vsyscall_timeinfo(void) | |
310 | { | |
311 | #ifdef CONFIG_X86_64 | |
312 | int cpu; | |
3dc4f7cf MT |
313 | u8 flags; |
314 | struct pvclock_vcpu_time_info *vcpu_time; | |
3dc4f7cf | 315 | |
fe1140cc JK |
316 | if (!hv_clock) |
317 | return 0; | |
318 | ||
c6338ce4 | 319 | cpu = get_cpu(); |
3dc4f7cf MT |
320 | |
321 | vcpu_time = &hv_clock[cpu].pvti; | |
322 | flags = pvclock_read_flags(vcpu_time); | |
323 | ||
c6338ce4 | 324 | put_cpu(); |
3dc4f7cf | 325 | |
94ffba48 RK |
326 | if (!(flags & PVCLOCK_TSC_STABLE_BIT)) |
327 | return 1; | |
328 | ||
3dc4f7cf MT |
329 | kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; |
330 | #endif | |
331 | return 0; | |
332 | } |