Commit | Line | Data |
---|---|---|
15c84731 JF |
1 | /* |
2 | * Xen time implementation. | |
3 | * | |
4 | * This is implemented in terms of a clocksource driver which uses | |
5 | * the hypervisor clock as a nanosecond timebase, and a clockevent | |
6 | * driver which uses the hypervisor's timer mechanism. | |
7 | * | |
8 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 | |
9 | */ | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/interrupt.h> | |
12 | #include <linux/clocksource.h> | |
13 | #include <linux/clockchips.h> | |
f91a8b44 | 14 | #include <linux/kernel_stat.h> |
f595ec96 | 15 | #include <linux/math64.h> |
5a0e3ad6 | 16 | #include <linux/gfp.h> |
c9d76a24 | 17 | #include <linux/slab.h> |
5584880e | 18 | #include <linux/pvclock_gtod.h> |
76096863 | 19 | #include <linux/timekeeper_internal.h> |
15c84731 | 20 | |
1c7b67f7 | 21 | #include <asm/pvclock.h> |
15c84731 JF |
22 | #include <asm/xen/hypervisor.h> |
23 | #include <asm/xen/hypercall.h> | |
24 | ||
25 | #include <xen/events.h> | |
409771d2 | 26 | #include <xen/features.h> |
15c84731 JF |
27 | #include <xen/interface/xen.h> |
28 | #include <xen/interface/vcpu.h> | |
29 | ||
30 | #include "xen-ops.h" | |
31 | ||
15c84731 JF |
32 | /* Xen may fire a timer up to this many ns early */ |
33 | #define TIMER_SLOP 100000 | |
f91a8b44 | 34 | #define NS_PER_TICK (1000000000LL / HZ) |
15c84731 | 35 | |
f91a8b44 | 36 | /* snapshots of runstate info */ |
c6e22f9e | 37 | static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); |
f91a8b44 | 38 | |
0b0c002c | 39 | /* unused ns of stolen time */ |
c6e22f9e | 40 | static DEFINE_PER_CPU(u64, xen_residual_stolen); |
f91a8b44 | 41 | |
f91a8b44 JF |
42 | static void do_stolen_accounting(void) |
43 | { | |
44 | struct vcpu_runstate_info state; | |
45 | struct vcpu_runstate_info *snap; | |
0b0c002c | 46 | s64 runnable, offline, stolen; |
f91a8b44 JF |
47 | cputime_t ticks; |
48 | ||
4ccefbe5 | 49 | xen_get_runstate_snapshot(&state); |
f91a8b44 JF |
50 | |
51 | WARN_ON(state.state != RUNSTATE_running); | |
52 | ||
89cbc767 | 53 | snap = this_cpu_ptr(&xen_runstate_snapshot); |
f91a8b44 JF |
54 | |
55 | /* work out how much time the VCPU has not been runn*ing* */ | |
f91a8b44 JF |
56 | runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable]; |
57 | offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline]; | |
58 | ||
59 | *snap = state; | |
60 | ||
61 | /* Add the appropriate number of ticks of stolen time, | |
79741dd3 | 62 | including any left-overs from last time. */ |
780f36d8 | 63 | stolen = runnable + offline + __this_cpu_read(xen_residual_stolen); |
f91a8b44 JF |
64 | |
65 | if (stolen < 0) | |
66 | stolen = 0; | |
67 | ||
f595ec96 | 68 | ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); |
780f36d8 | 69 | __this_cpu_write(xen_residual_stolen, stolen); |
79741dd3 | 70 | account_steal_ticks(ticks); |
f91a8b44 JF |
71 | } |
72 | ||
e93ef949 | 73 | /* Get the TSC speed from Xen */ |
409771d2 | 74 | static unsigned long xen_tsc_khz(void) |
15c84731 | 75 | { |
3807f345 | 76 | struct pvclock_vcpu_time_info *info = |
15c84731 JF |
77 | &HYPERVISOR_shared_info->vcpu_info[0].time; |
78 | ||
3807f345 | 79 | return pvclock_tsc_khz(info); |
15c84731 JF |
80 | } |
81 | ||
ee7686bc | 82 | cycle_t xen_clocksource_read(void) |
15c84731 | 83 | { |
1c7b67f7 | 84 | struct pvclock_vcpu_time_info *src; |
15c84731 | 85 | cycle_t ret; |
15c84731 | 86 | |
f1c39625 | 87 | preempt_disable_notrace(); |
3251f20b | 88 | src = &__this_cpu_read(xen_vcpu)->time; |
1c7b67f7 | 89 | ret = pvclock_clocksource_read(src); |
f1c39625 | 90 | preempt_enable_notrace(); |
15c84731 JF |
91 | return ret; |
92 | } | |
93 | ||
8e19608e MD |
94 | static cycle_t xen_clocksource_get_cycles(struct clocksource *cs) |
95 | { | |
96 | return xen_clocksource_read(); | |
97 | } | |
98 | ||
15c84731 JF |
99 | static void xen_read_wallclock(struct timespec *ts) |
100 | { | |
1c7b67f7 GH |
101 | struct shared_info *s = HYPERVISOR_shared_info; |
102 | struct pvclock_wall_clock *wall_clock = &(s->wc); | |
103 | struct pvclock_vcpu_time_info *vcpu_time; | |
15c84731 | 104 | |
1c7b67f7 GH |
105 | vcpu_time = &get_cpu_var(xen_vcpu)->time; |
106 | pvclock_read_wallclock(wall_clock, vcpu_time, ts); | |
107 | put_cpu_var(xen_vcpu); | |
15c84731 JF |
108 | } |
109 | ||
3565184e | 110 | static void xen_get_wallclock(struct timespec *now) |
15c84731 | 111 | { |
3565184e | 112 | xen_read_wallclock(now); |
15c84731 | 113 | } |
15c84731 | 114 | |
3565184e | 115 | static int xen_set_wallclock(const struct timespec *now) |
15c84731 | 116 | { |
47433b8c | 117 | return -1; |
15c84731 JF |
118 | } |
119 | ||
47433b8c DV |
120 | static int xen_pvclock_gtod_notify(struct notifier_block *nb, |
121 | unsigned long was_set, void *priv) | |
15c84731 | 122 | { |
47433b8c | 123 | /* Protected by the calling core code serialization */ |
187b26a9 | 124 | static struct timespec64 next_sync; |
5584880e | 125 | |
fdb9eb9f | 126 | struct xen_platform_op op; |
76096863 SS |
127 | struct timespec64 now; |
128 | struct timekeeper *tk = priv; | |
129 | static bool settime64_supported = true; | |
130 | int ret; | |
fdb9eb9f | 131 | |
76096863 SS |
132 | now.tv_sec = tk->xtime_sec; |
133 | now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); | |
5584880e | 134 | |
47433b8c DV |
135 | /* |
136 | * We only take the expensive HV call when the clock was set | |
137 | * or when the 11 minutes RTC synchronization time elapsed. | |
138 | */ | |
187b26a9 | 139 | if (!was_set && timespec64_compare(&now, &next_sync) < 0) |
47433b8c | 140 | return NOTIFY_OK; |
fdb9eb9f | 141 | |
76096863 SS |
142 | again: |
143 | if (settime64_supported) { | |
144 | op.cmd = XENPF_settime64; | |
145 | op.u.settime64.mbz = 0; | |
146 | op.u.settime64.secs = now.tv_sec; | |
147 | op.u.settime64.nsecs = now.tv_nsec; | |
148 | op.u.settime64.system_time = xen_clocksource_read(); | |
149 | } else { | |
150 | op.cmd = XENPF_settime32; | |
151 | op.u.settime32.secs = now.tv_sec; | |
152 | op.u.settime32.nsecs = now.tv_nsec; | |
153 | op.u.settime32.system_time = xen_clocksource_read(); | |
154 | } | |
155 | ||
156 | ret = HYPERVISOR_platform_op(&op); | |
157 | ||
158 | if (ret == -ENOSYS && settime64_supported) { | |
159 | settime64_supported = false; | |
160 | goto again; | |
161 | } | |
162 | if (ret < 0) | |
163 | return NOTIFY_BAD; | |
fdb9eb9f | 164 | |
47433b8c DV |
165 | /* |
166 | * Move the next drift compensation time 11 minutes | |
167 | * ahead. That's emulating the sync_cmos_clock() update for | |
168 | * the hardware RTC. | |
169 | */ | |
170 | next_sync = now; | |
171 | next_sync.tv_sec += 11 * 60; | |
172 | ||
5584880e | 173 | return NOTIFY_OK; |
15c84731 JF |
174 | } |
175 | ||
5584880e DV |
176 | static struct notifier_block xen_pvclock_gtod_notifier = { |
177 | .notifier_call = xen_pvclock_gtod_notify, | |
178 | }; | |
179 | ||
15c84731 JF |
180 | static struct clocksource xen_clocksource __read_mostly = { |
181 | .name = "xen", | |
182 | .rating = 400, | |
8e19608e | 183 | .read = xen_clocksource_get_cycles, |
15c84731 | 184 | .mask = ~0, |
15c84731 JF |
185 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
186 | }; | |
187 | ||
188 | /* | |
189 | Xen clockevent implementation | |
190 | ||
191 | Xen has two clockevent implementations: | |
192 | ||
193 | The old timer_op one works with all released versions of Xen prior | |
194 | to version 3.0.4. This version of the hypervisor provides a | |
195 | single-shot timer with nanosecond resolution. However, sharing the | |
196 | same event channel is a 100Hz tick which is delivered while the | |
197 | vcpu is running. We don't care about or use this tick, but it will | |
198 | cause the core time code to think the timer fired too soon, and | |
199 | will end up resetting it each time. It could be filtered, but | |
200 | doing so has complications when the ktime clocksource is not yet | |
201 | the xen clocksource (ie, at boot time). | |
202 | ||
203 | The new vcpu_op-based timer interface allows the tick timer period | |
204 | to be changed or turned off. The tick timer is not useful as a | |
205 | periodic timer because events are only delivered to running vcpus. | |
206 | The one-shot timer can report when a timeout is in the past, so | |
207 | set_next_event is capable of returning -ETIME when appropriate. | |
208 | This interface is used when available. | |
209 | */ | |
210 | ||
211 | ||
212 | /* | |
213 | Get a hypervisor absolute time. In theory we could maintain an | |
214 | offset between the kernel's time and the hypervisor's time, and | |
215 | apply that to a kernel's absolute timeout. Unfortunately the | |
216 | hypervisor and kernel times can drift even if the kernel is using | |
217 | the Xen clocksource, because ntp can warp the kernel's clocksource. | |
218 | */ | |
219 | static s64 get_abs_timeout(unsigned long delta) | |
220 | { | |
221 | return xen_clocksource_read() + delta; | |
222 | } | |
223 | ||
955381dd | 224 | static int xen_timerop_shutdown(struct clock_event_device *evt) |
15c84731 | 225 | { |
955381dd VK |
226 | /* cancel timeout */ |
227 | HYPERVISOR_set_timer_op(0); | |
228 | ||
229 | return 0; | |
15c84731 JF |
230 | } |
231 | ||
232 | static int xen_timerop_set_next_event(unsigned long delta, | |
233 | struct clock_event_device *evt) | |
234 | { | |
955381dd | 235 | WARN_ON(!clockevent_state_oneshot(evt)); |
15c84731 JF |
236 | |
237 | if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0) | |
238 | BUG(); | |
239 | ||
240 | /* We may have missed the deadline, but there's no real way of | |
241 | knowing for sure. If the event was in the past, then we'll | |
242 | get an immediate interrupt. */ | |
243 | ||
244 | return 0; | |
245 | } | |
246 | ||
247 | static const struct clock_event_device xen_timerop_clockevent = { | |
955381dd VK |
248 | .name = "xen", |
249 | .features = CLOCK_EVT_FEAT_ONESHOT, | |
15c84731 | 250 | |
955381dd VK |
251 | .max_delta_ns = 0xffffffff, |
252 | .min_delta_ns = TIMER_SLOP, | |
15c84731 | 253 | |
955381dd VK |
254 | .mult = 1, |
255 | .shift = 0, | |
256 | .rating = 500, | |
15c84731 | 257 | |
955381dd VK |
258 | .set_state_shutdown = xen_timerop_shutdown, |
259 | .set_next_event = xen_timerop_set_next_event, | |
15c84731 JF |
260 | }; |
261 | ||
955381dd VK |
262 | static int xen_vcpuop_shutdown(struct clock_event_device *evt) |
263 | { | |
264 | int cpu = smp_processor_id(); | |
15c84731 | 265 | |
955381dd VK |
266 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) || |
267 | HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) | |
268 | BUG(); | |
15c84731 | 269 | |
955381dd VK |
270 | return 0; |
271 | } | |
272 | ||
273 | static int xen_vcpuop_set_oneshot(struct clock_event_device *evt) | |
15c84731 JF |
274 | { |
275 | int cpu = smp_processor_id(); | |
276 | ||
955381dd VK |
277 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) |
278 | BUG(); | |
279 | ||
280 | return 0; | |
15c84731 JF |
281 | } |
282 | ||
283 | static int xen_vcpuop_set_next_event(unsigned long delta, | |
284 | struct clock_event_device *evt) | |
285 | { | |
286 | int cpu = smp_processor_id(); | |
287 | struct vcpu_set_singleshot_timer single; | |
288 | int ret; | |
289 | ||
955381dd | 290 | WARN_ON(!clockevent_state_oneshot(evt)); |
15c84731 JF |
291 | |
292 | single.timeout_abs_ns = get_abs_timeout(delta); | |
c06b6d70 SS |
293 | /* Get an event anyway, even if the timeout is already expired */ |
294 | single.flags = 0; | |
15c84731 JF |
295 | |
296 | ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single); | |
c06b6d70 | 297 | BUG_ON(ret != 0); |
15c84731 JF |
298 | |
299 | return ret; | |
300 | } | |
301 | ||
302 | static const struct clock_event_device xen_vcpuop_clockevent = { | |
303 | .name = "xen", | |
304 | .features = CLOCK_EVT_FEAT_ONESHOT, | |
305 | ||
306 | .max_delta_ns = 0xffffffff, | |
307 | .min_delta_ns = TIMER_SLOP, | |
308 | ||
309 | .mult = 1, | |
310 | .shift = 0, | |
311 | .rating = 500, | |
312 | ||
955381dd VK |
313 | .set_state_shutdown = xen_vcpuop_shutdown, |
314 | .set_state_oneshot = xen_vcpuop_set_oneshot, | |
15c84731 JF |
315 | .set_next_event = xen_vcpuop_set_next_event, |
316 | }; | |
317 | ||
318 | static const struct clock_event_device *xen_clockevent = | |
319 | &xen_timerop_clockevent; | |
31620a19 KRW |
320 | |
321 | struct xen_clock_event_device { | |
322 | struct clock_event_device evt; | |
7be0772d | 323 | char name[16]; |
31620a19 KRW |
324 | }; |
325 | static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 }; | |
15c84731 JF |
326 | |
327 | static irqreturn_t xen_timer_interrupt(int irq, void *dev_id) | |
328 | { | |
89cbc767 | 329 | struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt); |
15c84731 JF |
330 | irqreturn_t ret; |
331 | ||
332 | ret = IRQ_NONE; | |
333 | if (evt->event_handler) { | |
334 | evt->event_handler(evt); | |
335 | ret = IRQ_HANDLED; | |
336 | } | |
337 | ||
f91a8b44 JF |
338 | do_stolen_accounting(); |
339 | ||
15c84731 JF |
340 | return ret; |
341 | } | |
342 | ||
09e99da7 KRW |
343 | void xen_teardown_timer(int cpu) |
344 | { | |
345 | struct clock_event_device *evt; | |
346 | BUG_ON(cpu == 0); | |
347 | evt = &per_cpu(xen_clock_events, cpu).evt; | |
348 | ||
349 | if (evt->irq >= 0) { | |
350 | unbind_from_irqhandler(evt->irq, NULL); | |
351 | evt->irq = -1; | |
09e99da7 KRW |
352 | } |
353 | } | |
354 | ||
f87e4cac | 355 | void xen_setup_timer(int cpu) |
15c84731 | 356 | { |
7be0772d VK |
357 | struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu); |
358 | struct clock_event_device *evt = &xevt->evt; | |
15c84731 JF |
359 | int irq; |
360 | ||
ef35a4e6 | 361 | WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu); |
09e99da7 KRW |
362 | if (evt->irq >= 0) |
363 | xen_teardown_timer(cpu); | |
ef35a4e6 | 364 | |
15c84731 JF |
365 | printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu); |
366 | ||
7be0772d | 367 | snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu); |
15c84731 JF |
368 | |
369 | irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt, | |
9d71cee6 | 370 | IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER| |
8d5999df | 371 | IRQF_FORCE_RESUME|IRQF_EARLY_RESUME, |
7be0772d | 372 | xevt->name, NULL); |
8785c676 | 373 | (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX); |
15c84731 | 374 | |
15c84731 JF |
375 | memcpy(evt, xen_clockevent, sizeof(*evt)); |
376 | ||
320ab2b0 | 377 | evt->cpumask = cpumask_of(cpu); |
15c84731 | 378 | evt->irq = irq; |
f87e4cac JF |
379 | } |
380 | ||
d68d82af | 381 | |
f87e4cac JF |
382 | void xen_setup_cpu_clockevents(void) |
383 | { | |
89cbc767 | 384 | clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt)); |
15c84731 JF |
385 | } |
386 | ||
d07af1f0 JF |
387 | void xen_timer_resume(void) |
388 | { | |
389 | int cpu; | |
390 | ||
e7a3481c JF |
391 | pvclock_resume(); |
392 | ||
d07af1f0 JF |
393 | if (xen_clockevent != &xen_vcpuop_clockevent) |
394 | return; | |
395 | ||
396 | for_each_online_cpu(cpu) { | |
397 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) | |
398 | BUG(); | |
399 | } | |
400 | } | |
401 | ||
fb6ce5de | 402 | static const struct pv_time_ops xen_time_ops __initconst = { |
ca50a5f3 | 403 | .sched_clock = xen_clocksource_read, |
409771d2 SS |
404 | }; |
405 | ||
fb6ce5de | 406 | static void __init xen_time_init(void) |
15c84731 JF |
407 | { |
408 | int cpu = smp_processor_id(); | |
c4507257 | 409 | struct timespec tp; |
15c84731 | 410 | |
94dd85f6 PI |
411 | /* As Dom0 is never moved, no penalty on using TSC there */ |
412 | if (xen_initial_domain()) | |
413 | xen_clocksource.rating = 275; | |
414 | ||
b01cc1b0 | 415 | clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); |
15c84731 JF |
416 | |
417 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { | |
f91a8b44 | 418 | /* Successfully turned off 100Hz tick, so we have the |
15c84731 JF |
419 | vcpuop-based timer interface */ |
420 | printk(KERN_DEBUG "Xen: using vcpuop timer interface\n"); | |
421 | xen_clockevent = &xen_vcpuop_clockevent; | |
422 | } | |
423 | ||
424 | /* Set initial system time with full resolution */ | |
c4507257 JS |
425 | xen_read_wallclock(&tp); |
426 | do_settimeofday(&tp); | |
15c84731 | 427 | |
404ee5b1 | 428 | setup_force_cpu_cap(X86_FEATURE_TSC); |
15c84731 | 429 | |
be012920 | 430 | xen_setup_runstate_info(cpu); |
15c84731 | 431 | xen_setup_timer(cpu); |
f87e4cac | 432 | xen_setup_cpu_clockevents(); |
5584880e DV |
433 | |
434 | if (xen_initial_domain()) | |
435 | pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier); | |
15c84731 | 436 | } |
409771d2 | 437 | |
fb6ce5de | 438 | void __init xen_init_time_ops(void) |
409771d2 SS |
439 | { |
440 | pv_time_ops = xen_time_ops; | |
441 | ||
442 | x86_init.timers.timer_init = xen_time_init; | |
443 | x86_init.timers.setup_percpu_clockev = x86_init_noop; | |
444 | x86_cpuinit.setup_percpu_clockev = x86_init_noop; | |
445 | ||
446 | x86_platform.calibrate_tsc = xen_tsc_khz; | |
447 | x86_platform.get_wallclock = xen_get_wallclock; | |
47433b8c DV |
448 | /* Dom0 uses the native method to set the hardware RTC. */ |
449 | if (!xen_initial_domain()) | |
450 | x86_platform.set_wallclock = xen_set_wallclock; | |
409771d2 SS |
451 | } |
452 | ||
ca65f9fc | 453 | #ifdef CONFIG_XEN_PVHVM |
409771d2 SS |
454 | static void xen_hvm_setup_cpu_clockevents(void) |
455 | { | |
456 | int cpu = smp_processor_id(); | |
457 | xen_setup_runstate_info(cpu); | |
7918c92a KRW |
458 | /* |
459 | * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence | |
460 | * doing it xen_hvm_cpu_notify (which gets called by smp_init during | |
461 | * early bootup and also during CPU hotplug events). | |
462 | */ | |
409771d2 SS |
463 | xen_setup_cpu_clockevents(); |
464 | } | |
465 | ||
fb6ce5de | 466 | void __init xen_hvm_init_time_ops(void) |
409771d2 SS |
467 | { |
468 | /* vector callback is needed otherwise we cannot receive interrupts | |
31e7e931 SS |
469 | * on cpu > 0 and at this point we don't know how many cpus are |
470 | * available */ | |
471 | if (!xen_have_vector_callback) | |
409771d2 SS |
472 | return; |
473 | if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { | |
474 | printk(KERN_INFO "Xen doesn't support pvclock on HVM," | |
475 | "disable pv timer\n"); | |
476 | return; | |
477 | } | |
478 | ||
479 | pv_time_ops = xen_time_ops; | |
480 | x86_init.timers.setup_percpu_clockev = xen_time_init; | |
481 | x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; | |
482 | ||
483 | x86_platform.calibrate_tsc = xen_tsc_khz; | |
484 | x86_platform.get_wallclock = xen_get_wallclock; | |
485 | x86_platform.set_wallclock = xen_set_wallclock; | |
486 | } | |
ca65f9fc | 487 | #endif |