[linux-block.git] / arch / x86 / xen / time.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Xen time implementation.
 *
 * This is implemented in terms of a clocksource driver which uses
 * the hypervisor clock as a nanosecond timebase, and a clockevent
 * driver which uses the hypervisor's timer mechanism.
 *
 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
 */
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/clocksource.h>
#include <linux/clockchips.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/pvclock_gtod.h>
#include <linux/timekeeper_internal.h>

#include <asm/pvclock.h>
#include <asm/xen/hypervisor.h>
#include <asm/xen/hypercall.h>

#include <xen/events.h>
#include <xen/features.h>
#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>

#include "xen-ops.h"

/* Minimum amount of time until next clock event fires */
#define TIMER_SLOP	100000

static u64 xen_sched_clock_offset __read_mostly;

/* Get the TSC speed from Xen */
static unsigned long xen_tsc_khz(void)
{
	struct pvclock_vcpu_time_info *info =
		&HYPERVISOR_shared_info->vcpu_info[0].time;

	return pvclock_tsc_khz(info);
}

static u64 xen_clocksource_read(void)
{
        struct pvclock_vcpu_time_info *src;
	u64 ret;

	preempt_disable_notrace();
	src = &__this_cpu_read(xen_vcpu)->time;
	ret = pvclock_clocksource_read(src);
	preempt_enable_notrace();
	return ret;
}

static u64 xen_clocksource_get_cycles(struct clocksource *cs)
{
	return xen_clocksource_read();
}

static u64 xen_sched_clock(void)
{
	return xen_clocksource_read() - xen_sched_clock_offset;
}

static void xen_read_wallclock(struct timespec64 *ts)
{
	struct shared_info *s = HYPERVISOR_shared_info;
	struct pvclock_wall_clock *wall_clock = &(s->wc);
        struct pvclock_vcpu_time_info *vcpu_time;

	vcpu_time = &get_cpu_var(xen_vcpu)->time;
	pvclock_read_wallclock(wall_clock, vcpu_time, ts);
	put_cpu_var(xen_vcpu);
}

static void xen_get_wallclock(struct timespec64 *now)
{
	xen_read_wallclock(now);
}

static int xen_set_wallclock(const struct timespec64 *now)
{
	return -ENODEV;
}

static int xen_pvclock_gtod_notify(struct notifier_block *nb,
				   unsigned long was_set, void *priv)
{
	/* Protected by the calling core code serialization */
	static struct timespec64 next_sync;

	struct xen_platform_op op;
	struct timespec64 now;
	struct timekeeper *tk = priv;
	static bool settime64_supported = true;
	int ret;

	now.tv_sec = tk->xtime_sec;
	now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);

	/*
	 * We only take the expensive HV call when the clock was set
	 * or when the 11 minutes RTC synchronization time elapsed.
	 */
	if (!was_set && timespec64_compare(&now, &next_sync) < 0)
		return NOTIFY_OK;

again:
	if (settime64_supported) {
		op.cmd = XENPF_settime64;
		op.u.settime64.mbz = 0;
		op.u.settime64.secs = now.tv_sec;
		op.u.settime64.nsecs = now.tv_nsec;
		op.u.settime64.system_time = xen_clocksource_read();
	} else {
		op.cmd = XENPF_settime32;
		op.u.settime32.secs = now.tv_sec;
		op.u.settime32.nsecs = now.tv_nsec;
		op.u.settime32.system_time = xen_clocksource_read();
	}

	ret = HYPERVISOR_platform_op(&op);

	if (ret == -ENOSYS && settime64_supported) {
		settime64_supported = false;
		goto again;
	}
	if (ret < 0)
		return NOTIFY_BAD;

	/*
	 * Move the next drift compensation time 11 minutes
	 * ahead. That's emulating the sync_cmos_clock() update for
	 * the hardware RTC.
	 */
	next_sync = now;
	next_sync.tv_sec += 11 * 60;

	return NOTIFY_OK;
}

static struct notifier_block xen_pvclock_gtod_notifier = {
	.notifier_call = xen_pvclock_gtod_notify,
};

static struct clocksource xen_clocksource __read_mostly = {
	.name = "xen",
	.rating = 400,
	.read = xen_clocksource_get_cycles,
	.mask = ~0,
	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
};

/*
   Xen clockevent implementation

   Xen has two clockevent implementations:

   The old timer_op one works with all released versions of Xen prior
   to version 3.0.4.  This version of the hypervisor provides a
   single-shot timer with nanosecond resolution.  However, sharing the
   same event channel is a 100Hz tick which is delivered while the
   vcpu is running.  We don't care about or use this tick, but it will
   cause the core time code to think the timer fired too soon, and
   will end up resetting it each time.  It could be filtered, but
   doing so has complications when the ktime clocksource is not yet
   the xen clocksource (ie, at boot time).

   The new vcpu_op-based timer interface allows the tick timer period
   to be changed or turned off.  The tick timer is not useful as a
   periodic timer because events are only delivered to running vcpus.
   The one-shot timer can report when a timeout is in the past, so
   set_next_event is capable of returning -ETIME when appropriate.
   This interface is used when available.
*/


/*
  Get a hypervisor absolute time.  In theory we could maintain an
  offset between the kernel's time and the hypervisor's time, and
  apply that to a kernel's absolute timeout.  Unfortunately the
  hypervisor and kernel times can drift even if the kernel is using
  the Xen clocksource, because ntp can warp the kernel's clocksource.
*/
static s64 get_abs_timeout(unsigned long delta)
{
	return xen_clocksource_read() + delta;
}

static int xen_timerop_shutdown(struct clock_event_device *evt)
{
	/* cancel timeout */
	HYPERVISOR_set_timer_op(0);

	return 0;
}

static int xen_timerop_set_next_event(unsigned long delta,
				      struct clock_event_device *evt)
{
	WARN_ON(!clockevent_state_oneshot(evt));

	if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
		BUG();

	/* We may have missed the deadline, but there's no real way of
	   knowing for sure.  If the event was in the past, then we'll
	   get an immediate interrupt. */

	return 0;
}

static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
	.name			= "xen",
	.features		= CLOCK_EVT_FEAT_ONESHOT,

	.max_delta_ns		= 0xffffffff,
	.max_delta_ticks	= 0xffffffff,
	.min_delta_ns		= TIMER_SLOP,
	.min_delta_ticks	= TIMER_SLOP,

	.mult			= 1,
	.shift			= 0,
	.rating			= 500,

	.set_state_shutdown	= xen_timerop_shutdown,
	.set_next_event		= xen_timerop_set_next_event,
};

static int xen_vcpuop_shutdown(struct clock_event_device *evt)
{
	int cpu = smp_processor_id();

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
			       NULL) ||
	    HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
			       NULL))
		BUG();

	return 0;
}

static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
{
	int cpu = smp_processor_id();

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
			       NULL))
		BUG();

	return 0;
}

static int xen_vcpuop_set_next_event(unsigned long delta,
				     struct clock_event_device *evt)
{
	int cpu = smp_processor_id();
	struct vcpu_set_singleshot_timer single;
	int ret;

	WARN_ON(!clockevent_state_oneshot(evt));

	single.timeout_abs_ns = get_abs_timeout(delta);
	/* Get an event anyway, even if the timeout is already expired */
	single.flags = 0;

	ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
				 &single);
	BUG_ON(ret != 0);

	return ret;
}

static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
	.name = "xen",
	.features = CLOCK_EVT_FEAT_ONESHOT,

	.max_delta_ns = 0xffffffff,
	.max_delta_ticks = 0xffffffff,
	.min_delta_ns = TIMER_SLOP,
	.min_delta_ticks = TIMER_SLOP,

	.mult = 1,
	.shift = 0,
	.rating = 500,

	.set_state_shutdown = xen_vcpuop_shutdown,
	.set_state_oneshot = xen_vcpuop_set_oneshot,
	.set_next_event = xen_vcpuop_set_next_event,
};

static const struct clock_event_device *xen_clockevent =
	&xen_timerop_clockevent;

struct xen_clock_event_device {
	struct clock_event_device evt;
	char name[16];
};
static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };

static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
{
	struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
	irqreturn_t ret;

	ret = IRQ_NONE;
	if (evt->event_handler) {
		evt->event_handler(evt);
		ret = IRQ_HANDLED;
	}

	return ret;
}

void xen_teardown_timer(int cpu)
{
	struct clock_event_device *evt;
	evt = &per_cpu(xen_clock_events, cpu).evt;

	if (evt->irq >= 0) {
		unbind_from_irqhandler(evt->irq, NULL);
		evt->irq = -1;
	}
}

void xen_setup_timer(int cpu)
{
	struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
	struct clock_event_device *evt = &xevt->evt;
	int irq;

	WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
	if (evt->irq >= 0)
		xen_teardown_timer(cpu);

	printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);

	snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);

	irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
				      IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
				      IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
				      xevt->name, NULL);
	(void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);

	memcpy(evt, xen_clockevent, sizeof(*evt));

	evt->cpumask = cpumask_of(cpu);
	evt->irq = irq;
}


void xen_setup_cpu_clockevents(void)
{
	clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
}

void xen_timer_resume(void)
{
	int cpu;

	if (xen_clockevent != &xen_vcpuop_clockevent)
		return;

	for_each_online_cpu(cpu) {
		if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
				       xen_vcpu_nr(cpu), NULL))
			BUG();
	}
}

static const struct pv_time_ops xen_time_ops __initconst = {
	.sched_clock = xen_sched_clock,
	.steal_clock = xen_steal_clock,
};

static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
static u64 xen_clock_value_saved;

void xen_save_time_memory_area(void)
{
	struct vcpu_register_time_memory_area t;
	int ret;

	xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;

	if (!xen_clock)
		return;

	t.addr.v = NULL;

	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
	if (ret != 0)
		pr_notice("Cannot save secondary vcpu_time_info (err %d)",
			  ret);
	else
		clear_page(xen_clock);
}

void xen_restore_time_memory_area(void)
{
	struct vcpu_register_time_memory_area t;
	int ret;

	if (!xen_clock)
		goto out;

	t.addr.v = &xen_clock->pvti;

	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);

	/*
	 * We don't disable VCLOCK_PVCLOCK entirely if it fails to register the
	 * secondary time info with Xen or if we migrated to a host without the
	 * necessary flags. On both of these cases what happens is either
	 * process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT
	 * bit set. Userspace checks the latter and if 0, it discards the data
	 * in pvti and fallbacks to a system call for a reliable timestamp.
	 */
	if (ret != 0)
		pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
			  ret);

out:
	/* Need pvclock_resume() before using xen_clocksource_read(). */
	pvclock_resume();
	xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
}

static void xen_setup_vsyscall_time_info(void)
{
	struct vcpu_register_time_memory_area t;
	struct pvclock_vsyscall_time_info *ti;
	int ret;

	ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
	if (!ti)
		return;

	t.addr.v = &ti->pvti;

	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
	if (ret) {
		pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret);
		free_page((unsigned long)ti);
		return;
	}

	/*
	 * If primary time info had this bit set, secondary should too since
	 * it's the same data on both just different memory regions. But we
	 * still check it in case hypervisor is buggy.
	 */
	if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
		t.addr.v = NULL;
		ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
					 0, &t);
		if (!ret)
			free_page((unsigned long)ti);

		pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n");
		return;
	}

	xen_clock = ti;
	pvclock_set_pvti_cpu0_va(xen_clock);

	xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK;
}

static void __init xen_time_init(void)
{
	struct pvclock_vcpu_time_info *pvti;
	int cpu = smp_processor_id();
	struct timespec64 tp;

	/* As Dom0 is never moved, no penalty on using TSC there */
	if (xen_initial_domain())
		xen_clocksource.rating = 275;

	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
			       NULL) == 0) {
		/* Successfully turned off 100Hz tick, so we have the
		   vcpuop-based timer interface */
		printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
		xen_clockevent = &xen_vcpuop_clockevent;
	}

	/* Set initial system time with full resolution */
	xen_read_wallclock(&tp);
	do_settimeofday64(&tp);

	setup_force_cpu_cap(X86_FEATURE_TSC);

	/*
	 * We check ahead on the primary time info if this
	 * bit is supported hence speeding up Xen clocksource.
	 */
	pvti = &__this_cpu_read(xen_vcpu)->time;
	if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
		pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
		xen_setup_vsyscall_time_info();
	}

	xen_setup_runstate_info(cpu);
	xen_setup_timer(cpu);
	xen_setup_cpu_clockevents();

	xen_time_setup_guest();

	if (xen_initial_domain())
		pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
}

void __init xen_init_time_ops(void)
{
	xen_sched_clock_offset = xen_clocksource_read();
	pv_ops.time = xen_time_ops;

	x86_init.timers.timer_init = xen_time_init;
	x86_init.timers.setup_percpu_clockev = x86_init_noop;
	x86_cpuinit.setup_percpu_clockev = x86_init_noop;

	x86_platform.calibrate_tsc = xen_tsc_khz;
	x86_platform.get_wallclock = xen_get_wallclock;
	/* Dom0 uses the native method to set the hardware RTC. */
	if (!xen_initial_domain())
		x86_platform.set_wallclock = xen_set_wallclock;
}

#ifdef CONFIG_XEN_PVHVM
static void xen_hvm_setup_cpu_clockevents(void)
{
	int cpu = smp_processor_id();
	xen_setup_runstate_info(cpu);
	/*
	 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
	 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
	 * early bootup and also during CPU hotplug events).
	 */
	xen_setup_cpu_clockevents();
}

void __init xen_hvm_init_time_ops(void)
{
	/*
	 * vector callback is needed otherwise we cannot receive interrupts
	 * on cpu > 0 and at this point we don't know how many cpus are
	 * available.
	 */
	if (!xen_have_vector_callback)
		return;

	if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
		pr_info("Xen doesn't support pvclock on HVM, disable pv timer");
		return;
	}

	xen_sched_clock_offset = xen_clocksource_read();
	pv_ops.time = xen_time_ops;
	x86_init.timers.setup_percpu_clockev = xen_time_init;
	x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;

	x86_platform.calibrate_tsc = xen_tsc_khz;
	x86_platform.get_wallclock = xen_get_wallclock;
	x86_platform.set_wallclock = xen_set_wallclock;
}
#endif

/* Kernel parameter to specify Xen timer slop */
static int __init parse_xen_timer_slop(char *ptr)
{
	unsigned long slop = memparse(ptr, NULL);

	xen_timerop_clockevent.min_delta_ns = slop;
	xen_timerop_clockevent.min_delta_ticks = slop;
	xen_vcpuop_clockevent.min_delta_ns = slop;
	xen_vcpuop_clockevent.min_delta_ticks = slop;

	return 0;
}
early_param("xen_timer_slop", parse_xen_timer_slop);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
15c84731 JF	2	/*
	3	* Xen time implementation.
	4	*
	5	* This is implemented in terms of a clocksource driver which uses
	6	* the hypervisor clock as a nanosecond timebase, and a clockevent
	7	* driver which uses the hypervisor's timer mechanism.
	8	*
	9	* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
	10	*/
	11	#include <linux/kernel.h>
	12	#include <linux/interrupt.h>
	13	#include <linux/clocksource.h>
	14	#include <linux/clockchips.h>
5a0e3ad6	15	#include <linux/gfp.h>
c9d76a24	16	#include <linux/slab.h>
5584880e	17	#include <linux/pvclock_gtod.h>
76096863	18	#include <linux/timekeeper_internal.h>
15c84731	19
1c7b67f7	20	#include <asm/pvclock.h>
15c84731 JF	21	#include <asm/xen/hypervisor.h>
	22	#include <asm/xen/hypercall.h>
	23
	24	#include <xen/events.h>
409771d2	25	#include <xen/features.h>
15c84731 JF	26	#include <xen/interface/xen.h>
	27	#include <xen/interface/vcpu.h>
	28
	29	#include "xen-ops.h"
	30
2ec16bc0	31	/* Minimum amount of time until next clock event fires */
15c84731	32	#define TIMER_SLOP 100000
f91a8b44	33
38669ba2 PT	34	static u64 xen_sched_clock_offset __read_mostly;
38669ba2 PT	35
e93ef949	36	/* Get the TSC speed from Xen */
409771d2	37	static unsigned long xen_tsc_khz(void)
15c84731	38	{
3807f345	39	struct pvclock_vcpu_time_info *info =
15c84731 JF	40	&HYPERVISOR_shared_info->vcpu_info[0].time;
15c84731 JF	41
3807f345	42	return pvclock_tsc_khz(info);
15c84731 JF	43	}
15c84731 JF	44
7b25b9cb	45	static u64 xen_clocksource_read(void)
15c84731	46	{
1c7b67f7	47	struct pvclock_vcpu_time_info *src;
a5a1d1c2	48	u64 ret;
15c84731	49
f1c39625	50	preempt_disable_notrace();
3251f20b	51	src = &__this_cpu_read(xen_vcpu)->time;
1c7b67f7	52	ret = pvclock_clocksource_read(src);
f1c39625	53	preempt_enable_notrace();
15c84731 JF	54	return ret;
	55	}
	56
a5a1d1c2	57	static u64 xen_clocksource_get_cycles(struct clocksource *cs)
8e19608e MD	58	{
	59	return xen_clocksource_read();
	60	}
	61
38669ba2 PT	62	static u64 xen_sched_clock(void)
	63	{
	64	return xen_clocksource_read() - xen_sched_clock_offset;
	65	}
	66
e27c4929	67	static void xen_read_wallclock(struct timespec64 *ts)
15c84731	68	{
1c7b67f7 GH	69	struct shared_info *s = HYPERVISOR_shared_info;
	70	struct pvclock_wall_clock *wall_clock = &(s->wc);
	71	struct pvclock_vcpu_time_info *vcpu_time;
15c84731	72
1c7b67f7 GH	73	vcpu_time = &get_cpu_var(xen_vcpu)->time;
	74	pvclock_read_wallclock(wall_clock, vcpu_time, ts);
	75	put_cpu_var(xen_vcpu);
15c84731 JF	76	}
15c84731 JF	77
e27c4929	78	static void xen_get_wallclock(struct timespec64 *now)
15c84731	79	{
3565184e	80	xen_read_wallclock(now);
15c84731	81	}
15c84731	82
e27c4929	83	static int xen_set_wallclock(const struct timespec64 *now)
15c84731	84	{
b5494ad8	85	return -ENODEV;
15c84731 JF	86	}
15c84731 JF	87
47433b8c DV	88	static int xen_pvclock_gtod_notify(struct notifier_block *nb,
47433b8c DV	89	unsigned long was_set, void *priv)
15c84731	90	{
47433b8c	91	/* Protected by the calling core code serialization */
187b26a9	92	static struct timespec64 next_sync;
5584880e	93
fdb9eb9f	94	struct xen_platform_op op;
76096863 SS	95	struct timespec64 now;
	96	struct timekeeper *tk = priv;
	97	static bool settime64_supported = true;
	98	int ret;
fdb9eb9f	99
76096863 SS	100	now.tv_sec = tk->xtime_sec;
76096863 SS	101	now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
5584880e	102
47433b8c DV	103	/*
	104	* We only take the expensive HV call when the clock was set
	105	* or when the 11 minutes RTC synchronization time elapsed.
	106	*/
187b26a9	107	if (!was_set && timespec64_compare(&now, &next_sync) < 0)
47433b8c	108	return NOTIFY_OK;
fdb9eb9f	109
76096863 SS	110	again:
	111	if (settime64_supported) {
	112	op.cmd = XENPF_settime64;
	113	op.u.settime64.mbz = 0;
	114	op.u.settime64.secs = now.tv_sec;
	115	op.u.settime64.nsecs = now.tv_nsec;
	116	op.u.settime64.system_time = xen_clocksource_read();
	117	} else {
	118	op.cmd = XENPF_settime32;
	119	op.u.settime32.secs = now.tv_sec;
	120	op.u.settime32.nsecs = now.tv_nsec;
	121	op.u.settime32.system_time = xen_clocksource_read();
	122	}
	123
	124	ret = HYPERVISOR_platform_op(&op);
	125
	126	if (ret == -ENOSYS && settime64_supported) {
	127	settime64_supported = false;
	128	goto again;
	129	}
	130	if (ret < 0)
	131	return NOTIFY_BAD;
fdb9eb9f	132
47433b8c DV	133	/*
	134	* Move the next drift compensation time 11 minutes
	135	* ahead. That's emulating the sync_cmos_clock() update for
	136	* the hardware RTC.
	137	*/
	138	next_sync = now;
	139	next_sync.tv_sec += 11 * 60;
	140
5584880e	141	return NOTIFY_OK;
15c84731 JF	142	}
15c84731 JF	143
5584880e DV	144	static struct notifier_block xen_pvclock_gtod_notifier = {
	145	.notifier_call = xen_pvclock_gtod_notify,
	146	};
	147
15c84731 JF	148	static struct clocksource xen_clocksource __read_mostly = {
	149	.name = "xen",
	150	.rating = 400,
8e19608e	151	.read = xen_clocksource_get_cycles,
15c84731	152	.mask = ~0,
15c84731 JF	153	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
	154	};
	155
	156	/*
	157	Xen clockevent implementation
	158
	159	Xen has two clockevent implementations:
	160
	161	The old timer_op one works with all released versions of Xen prior
	162	to version 3.0.4. This version of the hypervisor provides a
	163	single-shot timer with nanosecond resolution. However, sharing the
	164	same event channel is a 100Hz tick which is delivered while the
	165	vcpu is running. We don't care about or use this tick, but it will
	166	cause the core time code to think the timer fired too soon, and
	167	will end up resetting it each time. It could be filtered, but
	168	doing so has complications when the ktime clocksource is not yet
	169	the xen clocksource (ie, at boot time).
	170
	171	The new vcpu_op-based timer interface allows the tick timer period
	172	to be changed or turned off. The tick timer is not useful as a
	173	periodic timer because events are only delivered to running vcpus.
	174	The one-shot timer can report when a timeout is in the past, so
	175	set_next_event is capable of returning -ETIME when appropriate.
	176	This interface is used when available.
	177	*/
	178
	179
	180	/*
	181	Get a hypervisor absolute time. In theory we could maintain an
	182	offset between the kernel's time and the hypervisor's time, and
	183	apply that to a kernel's absolute timeout. Unfortunately the
	184	hypervisor and kernel times can drift even if the kernel is using
	185	the Xen clocksource, because ntp can warp the kernel's clocksource.
	186	*/
	187	static s64 get_abs_timeout(unsigned long delta)
	188	{
	189	return xen_clocksource_read() + delta;
	190	}
	191
955381dd	192	static int xen_timerop_shutdown(struct clock_event_device *evt)
15c84731	193	{
955381dd VK	194	/* cancel timeout */
	195	HYPERVISOR_set_timer_op(0);
	196
	197	return 0;
15c84731 JF	198	}
	199
	200	static int xen_timerop_set_next_event(unsigned long delta,
	201	struct clock_event_device *evt)
	202	{
955381dd	203	WARN_ON(!clockevent_state_oneshot(evt));
15c84731 JF	204
	205	if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
	206	BUG();
	207
	208	/* We may have missed the deadline, but there's no real way of
	209	knowing for sure. If the event was in the past, then we'll
	210	get an immediate interrupt. */
	211
	212	return 0;
	213	}
	214
2ec16bc0	215	static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
955381dd VK	216	.name = "xen",
955381dd VK	217	.features = CLOCK_EVT_FEAT_ONESHOT,
15c84731	218
955381dd	219	.max_delta_ns = 0xffffffff,
3d18d661	220	.max_delta_ticks = 0xffffffff,
955381dd	221	.min_delta_ns = TIMER_SLOP,
3d18d661	222	.min_delta_ticks = TIMER_SLOP,
15c84731	223
955381dd VK	224	.mult = 1,
	225	.shift = 0,
	226	.rating = 500,
15c84731	227
955381dd VK	228	.set_state_shutdown = xen_timerop_shutdown,
955381dd VK	229	.set_next_event = xen_timerop_set_next_event,
15c84731 JF	230	};
15c84731 JF	231
955381dd VK	232	static int xen_vcpuop_shutdown(struct clock_event_device *evt)
	233	{
	234	int cpu = smp_processor_id();
15c84731	235
ad5475f9 VK	236	if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
	237	NULL) \|\|
	238	HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
	239	NULL))
955381dd	240	BUG();
15c84731	241
955381dd VK	242	return 0;
	243	}
	244
	245	static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
15c84731 JF	246	{
	247	int cpu = smp_processor_id();
	248
ad5475f9 VK	249	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
ad5475f9 VK	250	NULL))
955381dd VK	251	BUG();
	252
	253	return 0;
15c84731 JF	254	}
	255
	256	static int xen_vcpuop_set_next_event(unsigned long delta,
	257	struct clock_event_device *evt)
	258	{
	259	int cpu = smp_processor_id();
	260	struct vcpu_set_singleshot_timer single;
	261	int ret;
	262
955381dd	263	WARN_ON(!clockevent_state_oneshot(evt));
15c84731 JF	264
15c84731 JF	265	single.timeout_abs_ns = get_abs_timeout(delta);
c06b6d70 SS	266	/* Get an event anyway, even if the timeout is already expired */
c06b6d70 SS	267	single.flags = 0;
15c84731	268
ad5475f9 VK	269	ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
ad5475f9 VK	270	&single);
c06b6d70	271	BUG_ON(ret != 0);
15c84731 JF	272
	273	return ret;
	274	}
	275
2ec16bc0	276	static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
15c84731 JF	277	.name = "xen",
	278	.features = CLOCK_EVT_FEAT_ONESHOT,
	279
	280	.max_delta_ns = 0xffffffff,
3d18d661	281	.max_delta_ticks = 0xffffffff,
15c84731	282	.min_delta_ns = TIMER_SLOP,
3d18d661	283	.min_delta_ticks = TIMER_SLOP,
15c84731 JF	284
	285	.mult = 1,
	286	.shift = 0,
	287	.rating = 500,
	288
955381dd VK	289	.set_state_shutdown = xen_vcpuop_shutdown,
955381dd VK	290	.set_state_oneshot = xen_vcpuop_set_oneshot,
15c84731 JF	291	.set_next_event = xen_vcpuop_set_next_event,
	292	};
	293
	294	static const struct clock_event_device *xen_clockevent =
	295	&xen_timerop_clockevent;
31620a19 KRW	296
	297	struct xen_clock_event_device {
	298	struct clock_event_device evt;
7be0772d	299	char name[16];
31620a19 KRW	300	};
31620a19 KRW	301	static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
15c84731 JF	302
	303	static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
	304	{
89cbc767	305	struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
15c84731 JF	306	irqreturn_t ret;
	307
	308	ret = IRQ_NONE;
	309	if (evt->event_handler) {
	310	evt->event_handler(evt);
	311	ret = IRQ_HANDLED;
	312	}
	313
	314	return ret;
	315	}
	316
09e99da7 KRW	317	void xen_teardown_timer(int cpu)
	318	{
	319	struct clock_event_device *evt;
09e99da7 KRW	320	evt = &per_cpu(xen_clock_events, cpu).evt;
	321
	322	if (evt->irq >= 0) {
	323	unbind_from_irqhandler(evt->irq, NULL);
	324	evt->irq = -1;
09e99da7 KRW	325	}
	326	}
	327
f87e4cac	328	void xen_setup_timer(int cpu)
15c84731	329	{
7be0772d VK	330	struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
7be0772d VK	331	struct clock_event_device *evt = &xevt->evt;
15c84731 JF	332	int irq;
15c84731 JF	333
ef35a4e6	334	WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
09e99da7 KRW	335	if (evt->irq >= 0)
09e99da7 KRW	336	xen_teardown_timer(cpu);
ef35a4e6	337
15c84731 JF	338	printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
15c84731 JF	339
7be0772d	340	snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
15c84731 JF	341
15c84731 JF	342	irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
9d71cee6	343	IRQF_PERCPU\|IRQF_NOBALANCING\|IRQF_TIMER\|
8d5999df	344	IRQF_FORCE_RESUME\|IRQF_EARLY_RESUME,
7be0772d	345	xevt->name, NULL);
8785c676	346	(void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
15c84731	347
15c84731 JF	348	memcpy(evt, xen_clockevent, sizeof(*evt));
15c84731 JF	349
320ab2b0	350	evt->cpumask = cpumask_of(cpu);
15c84731	351	evt->irq = irq;
f87e4cac JF	352	}
f87e4cac JF	353
d68d82af	354
f87e4cac JF	355	void xen_setup_cpu_clockevents(void)
f87e4cac JF	356	{
89cbc767	357	clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
15c84731 JF	358	}
15c84731 JF	359
d07af1f0 JF	360	void xen_timer_resume(void)
	361	{
	362	int cpu;
	363
	364	if (xen_clockevent != &xen_vcpuop_clockevent)
	365	return;
	366
	367	for_each_online_cpu(cpu) {
ad5475f9 VK	368	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
ad5475f9 VK	369	xen_vcpu_nr(cpu), NULL))
d07af1f0 JF	370	BUG();
	371	}
	372	}
	373
fb6ce5de	374	static const struct pv_time_ops xen_time_ops __initconst = {
38669ba2	375	.sched_clock = xen_sched_clock,
d34c30cc	376	.steal_clock = xen_steal_clock,
409771d2 SS	377	};
409771d2 SS	378
2229f70b	379	static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
867cefb4	380	static u64 xen_clock_value_saved;
2229f70b JM	381
	382	void xen_save_time_memory_area(void)
	383	{
	384	struct vcpu_register_time_memory_area t;
	385	int ret;
	386
867cefb4 JG	387	xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;
867cefb4 JG	388
2229f70b JM	389	if (!xen_clock)
	390	return;
	391
	392	t.addr.v = NULL;
	393
	394	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
	395	if (ret != 0)
	396	pr_notice("Cannot save secondary vcpu_time_info (err %d)",
	397	ret);
	398	else
	399	clear_page(xen_clock);
	400	}
	401
	402	void xen_restore_time_memory_area(void)
	403	{
	404	struct vcpu_register_time_memory_area t;
	405	int ret;
	406
	407	if (!xen_clock)
867cefb4	408	goto out;
2229f70b JM	409
	410	t.addr.v = &xen_clock->pvti;
	411
	412	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
	413
	414	/*
	415	* We don't disable VCLOCK_PVCLOCK entirely if it fails to register the
	416	* secondary time info with Xen or if we migrated to a host without the
	417	* necessary flags. On both of these cases what happens is either
	418	* process seeing a zeroed out pvti or seeing no PVCLOCK_TSC_STABLE_BIT
	419	* bit set. Userspace checks the latter and if 0, it discards the data
	420	* in pvti and fallbacks to a system call for a reliable timestamp.
	421	*/
	422	if (ret != 0)
	423	pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
	424	ret);
867cefb4 JG	425
	426	out:
	427	/* Need pvclock_resume() before using xen_clocksource_read(). */
	428	pvclock_resume();
	429	xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
2229f70b JM	430	}
	431
	432	static void xen_setup_vsyscall_time_info(void)
	433	{
	434	struct vcpu_register_time_memory_area t;
	435	struct pvclock_vsyscall_time_info *ti;
	436	int ret;
	437
	438	ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
	439	if (!ti)
	440	return;
	441
	442	t.addr.v = &ti->pvti;
	443
	444	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
	445	if (ret) {
	446	pr_notice("xen: VCLOCK_PVCLOCK not supported (err %d)\n", ret);
	447	free_page((unsigned long)ti);
	448	return;
	449	}
	450
	451	/*
	452	* If primary time info had this bit set, secondary should too since
	453	* it's the same data on both just different memory regions. But we
	454	* still check it in case hypervisor is buggy.
	455	*/
	456	if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
	457	t.addr.v = NULL;
	458	ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
	459	0, &t);
	460	if (!ret)
	461	free_page((unsigned long)ti);
	462
	463	pr_notice("xen: VCLOCK_PVCLOCK not supported (tsc unstable)\n");
	464	return;
	465	}
	466
	467	xen_clock = ti;
	468	pvclock_set_pvti_cpu0_va(xen_clock);
	469
	470	xen_clocksource.archdata.vclock_mode = VCLOCK_PVCLOCK;
	471	}
	472
fb6ce5de	473	static void __init xen_time_init(void)
15c84731	474	{
b8888080	475	struct pvclock_vcpu_time_info *pvti;
15c84731	476	int cpu = smp_processor_id();
e27c4929	477	struct timespec64 tp;
15c84731	478
94dd85f6 PI	479	/* As Dom0 is never moved, no penalty on using TSC there */
	480	if (xen_initial_domain())
	481	xen_clocksource.rating = 275;
	482
b01cc1b0	483	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
15c84731	484
ad5475f9 VK	485	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
ad5475f9 VK	486	NULL) == 0) {
f91a8b44	487	/* Successfully turned off 100Hz tick, so we have the
15c84731 JF	488	vcpuop-based timer interface */
	489	printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
	490	xen_clockevent = &xen_vcpuop_clockevent;
	491	}
	492
	493	/* Set initial system time with full resolution */
c4507257	494	xen_read_wallclock(&tp);
e27c4929	495	do_settimeofday64(&tp);
15c84731	496
404ee5b1	497	setup_force_cpu_cap(X86_FEATURE_TSC);
15c84731	498
b8888080 JM	499	/*
	500	* We check ahead on the primary time info if this
	501	* bit is supported hence speeding up Xen clocksource.
	502	*/
	503	pvti = &__this_cpu_read(xen_vcpu)->time;
2229f70b	504	if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
b8888080	505	pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
2229f70b JM	506	xen_setup_vsyscall_time_info();
2229f70b JM	507	}
b8888080	508
be012920	509	xen_setup_runstate_info(cpu);
15c84731	510	xen_setup_timer(cpu);
f87e4cac	511	xen_setup_cpu_clockevents();
5584880e	512
ecb23dc6 JG	513	xen_time_setup_guest();
ecb23dc6 JG	514
5584880e DV	515	if (xen_initial_domain())
5584880e DV	516	pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
15c84731	517	}
409771d2	518
7b25b9cb	519	void __init xen_init_time_ops(void)
409771d2	520	{
38669ba2	521	xen_sched_clock_offset = xen_clocksource_read();
5c83511b	522	pv_ops.time = xen_time_ops;
409771d2 SS	523
	524	x86_init.timers.timer_init = xen_time_init;
	525	x86_init.timers.setup_percpu_clockev = x86_init_noop;
	526	x86_cpuinit.setup_percpu_clockev = x86_init_noop;
	527
	528	x86_platform.calibrate_tsc = xen_tsc_khz;
	529	x86_platform.get_wallclock = xen_get_wallclock;
47433b8c DV	530	/* Dom0 uses the native method to set the hardware RTC. */
	531	if (!xen_initial_domain())
	532	x86_platform.set_wallclock = xen_set_wallclock;
409771d2 SS	533	}
409771d2 SS	534
ca65f9fc	535	#ifdef CONFIG_XEN_PVHVM
409771d2 SS	536	static void xen_hvm_setup_cpu_clockevents(void)
	537	{
	538	int cpu = smp_processor_id();
	539	xen_setup_runstate_info(cpu);
7918c92a KRW	540	/*
	541	* xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
	542	* doing it xen_hvm_cpu_notify (which gets called by smp_init during
	543	* early bootup and also during CPU hotplug events).
	544	*/
409771d2 SS	545	xen_setup_cpu_clockevents();
	546	}
	547
fb6ce5de	548	void __init xen_hvm_init_time_ops(void)
409771d2	549	{
84d582d2 BO	550	/*
	551	* vector callback is needed otherwise we cannot receive interrupts
	552	* on cpu > 0 and at this point we don't know how many cpus are
	553	* available.
	554	*/
	555	if (!xen_have_vector_callback)
	556	return;
	557
409771d2	558	if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
7b25b9cb	559	pr_info("Xen doesn't support pvclock on HVM, disable pv timer");
409771d2 SS	560	return;
	561	}
	562
38669ba2	563	xen_sched_clock_offset = xen_clocksource_read();
5c83511b	564	pv_ops.time = xen_time_ops;
409771d2 SS	565	x86_init.timers.setup_percpu_clockev = xen_time_init;
	566	x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
	567
	568	x86_platform.calibrate_tsc = xen_tsc_khz;
	569	x86_platform.get_wallclock = xen_get_wallclock;
	570	x86_platform.set_wallclock = xen_set_wallclock;
	571	}
ca65f9fc	572	#endif
2ec16bc0 RT	573
	574	/* Kernel parameter to specify Xen timer slop */
	575	static int __init parse_xen_timer_slop(char *ptr)
	576	{
	577	unsigned long slop = memparse(ptr, NULL);
	578
	579	xen_timerop_clockevent.min_delta_ns = slop;
	580	xen_timerop_clockevent.min_delta_ticks = slop;
	581	xen_vcpuop_clockevent.min_delta_ns = slop;
	582	xen_vcpuop_clockevent.min_delta_ticks = slop;
	583
	584	return 0;
	585	}
	586	early_param("xen_timer_slop", parse_xen_timer_slop);