[linux-block.git] / arch / arm / kernel / arch_timer.c

/*
 *  linux/arch/arm/kernel/arch_timer.c
 *
 *  Copyright (C) 2011 ARM Ltd.
 *  All Rights Reserved
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/jiffies.h>
#include <linux/clockchips.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/io.h>

#include <asm/delay.h>
#include <asm/localtimer.h>
#include <asm/arch_timer.h>
#include <asm/sched_clock.h>

static u32 arch_timer_rate;

enum ppi_nr {
	PHYS_SECURE_PPI,
	PHYS_NONSECURE_PPI,
	VIRT_PPI,
	HYP_PPI,
	MAX_TIMER_PPI
};

static int arch_timer_ppi[MAX_TIMER_PPI];

static struct clock_event_device __percpu **arch_timer_evt;
static struct delay_timer arch_delay_timer;

static bool arch_timer_use_virtual = true;

/*
 * Architected system timer support.
 */

#define ARCH_TIMER_CTRL_ENABLE		(1 << 0)
#define ARCH_TIMER_CTRL_IT_MASK		(1 << 1)
#define ARCH_TIMER_CTRL_IT_STAT		(1 << 2)

#define ARCH_TIMER_REG_CTRL		0
#define ARCH_TIMER_REG_FREQ		1
#define ARCH_TIMER_REG_TVAL		2

#define ARCH_TIMER_PHYS_ACCESS		0
#define ARCH_TIMER_VIRT_ACCESS		1

/*
 * These register accessors are marked inline so the compiler can
 * nicely work out which register we want, and chuck away the rest of
 * the code. At least it does so with a recent GCC (4.6.3).
 */
static inline void arch_timer_reg_write(const int access, const int reg, u32 val)
{
	if (access == ARCH_TIMER_PHYS_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" (val));
			break;
		case ARCH_TIMER_REG_TVAL:
			asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r" (val));
			break;
		}
	}

	if (access == ARCH_TIMER_VIRT_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" (val));
			break;
		case ARCH_TIMER_REG_TVAL:
			asm volatile("mcr p15, 0, %0, c14, c3, 0" : : "r" (val));
			break;
		}
	}

	isb();
}

static inline u32 arch_timer_reg_read(const int access, const int reg)
{
	u32 val = 0;

	if (access == ARCH_TIMER_PHYS_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mrc p15, 0, %0, c14, c2, 1" : "=r" (val));
			break;
		case ARCH_TIMER_REG_TVAL:
			asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (val));
			break;
		case ARCH_TIMER_REG_FREQ:
			asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (val));
			break;
		}
	}

	if (access == ARCH_TIMER_VIRT_ACCESS) {
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			asm volatile("mrc p15, 0, %0, c14, c3, 1" : "=r" (val));
			break;
		case ARCH_TIMER_REG_TVAL:
			asm volatile("mrc p15, 0, %0, c14, c3, 0" : "=r" (val));
			break;
		}
	}

	return val;
}

static inline u64 arch_counter_get_cntpct(void)
{
	u64 cval;
	asm volatile("mrrc p15, 0, %Q0, %R0, c14" : "=r" (cval));
	return cval;
}

static inline u64 arch_counter_get_cntvct(void)
{
	u64 cval;
	asm volatile("mrrc p15, 1, %Q0, %R0, c14" : "=r" (cval));
	return cval;
}

static irqreturn_t inline timer_handler(const int access,
					struct clock_event_device *evt)
{
	unsigned long ctrl;
	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
		evt->event_handler(evt);
		return IRQ_HANDLED;
	}

	return IRQ_NONE;
}

static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
{
	struct clock_event_device *evt = *(struct clock_event_device **)dev_id;

	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
{
	struct clock_event_device *evt = *(struct clock_event_device **)dev_id;

	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
}

static inline void timer_set_mode(const int access, int mode)
{
	unsigned long ctrl;
	switch (mode) {
	case CLOCK_EVT_MODE_UNUSED:
	case CLOCK_EVT_MODE_SHUTDOWN:
		ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
		ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
		break;
	default:
		break;
	}
}

static void arch_timer_set_mode_virt(enum clock_event_mode mode,
				     struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode);
}

static void arch_timer_set_mode_phys(enum clock_event_mode mode,
				     struct clock_event_device *clk)
{
	timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode);
}

static inline void set_next_event(const int access, unsigned long evt)
{
	unsigned long ctrl;
	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
	ctrl |= ARCH_TIMER_CTRL_ENABLE;
	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt);
	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
}

static int arch_timer_set_next_event_virt(unsigned long evt,
					  struct clock_event_device *unused)
{
	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt);
	return 0;
}

static int arch_timer_set_next_event_phys(unsigned long evt,
					  struct clock_event_device *unused)
{
	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt);
	return 0;
}

static int __cpuinit arch_timer_setup(struct clock_event_device *clk)
{
	clk->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP;
	clk->name = "arch_sys_timer";
	clk->rating = 450;
	if (arch_timer_use_virtual) {
		clk->irq = arch_timer_ppi[VIRT_PPI];
		clk->set_mode = arch_timer_set_mode_virt;
		clk->set_next_event = arch_timer_set_next_event_virt;
	} else {
		clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
		clk->set_mode = arch_timer_set_mode_phys;
		clk->set_next_event = arch_timer_set_next_event_phys;
	}

	clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, NULL);

	clockevents_config_and_register(clk, arch_timer_rate,
					0xf, 0x7fffffff);

	*__this_cpu_ptr(arch_timer_evt) = clk;

	if (arch_timer_use_virtual)
		enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
	else {
		enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
	}

	return 0;
}

static int arch_timer_available(void)
{
	u32 freq;

	if (arch_timer_rate == 0) {
		freq = arch_timer_reg_read(ARCH_TIMER_PHYS_ACCESS,
					   ARCH_TIMER_REG_FREQ);

		/* Check the timer frequency. */
		if (freq == 0) {
			pr_warn("Architected timer frequency not available\n");
			return -EINVAL;
		}

		arch_timer_rate = freq;
	}

	pr_info_once("Architected local timer running at %lu.%02luMHz (%s).\n",
		     (unsigned long)arch_timer_rate / 1000000,
		     (unsigned long)(arch_timer_rate / 10000) % 100,
		     arch_timer_use_virtual ? "virt" : "phys");
	return 0;
}

static u32 notrace arch_counter_get_cntpct32(void)
{
	cycle_t cnt = arch_counter_get_cntpct();

	/*
	 * The sched_clock infrastructure only knows about counters
	 * with at most 32bits. Forget about the upper 24 bits for the
	 * time being...
	 */
	return (u32)cnt;
}

static u32 notrace arch_counter_get_cntvct32(void)
{
	cycle_t cnt = arch_counter_get_cntvct();

	/*
	 * The sched_clock infrastructure only knows about counters
	 * with at most 32bits. Forget about the upper 24 bits for the
	 * time being...
	 */
	return (u32)cnt;
}

static cycle_t arch_counter_read(struct clocksource *cs)
{
	/*
	 * Always use the physical counter for the clocksource.
	 * CNTHCTL.PL1PCTEN must be set to 1.
	 */
	return arch_counter_get_cntpct();
}

static unsigned long arch_timer_read_current_timer(void)
{
	return arch_counter_get_cntpct();
}

static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
{
	/*
	 * Always use the physical counter for the clocksource.
	 * CNTHCTL.PL1PCTEN must be set to 1.
	 */
	return arch_counter_get_cntpct();
}

static struct clocksource clocksource_counter = {
	.name	= "arch_sys_counter",
	.rating	= 400,
	.read	= arch_counter_read,
	.mask	= CLOCKSOURCE_MASK(56),
	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
};

static struct cyclecounter cyclecounter = {
	.read	= arch_counter_read_cc,
	.mask	= CLOCKSOURCE_MASK(56),
};

static struct timecounter timecounter;

struct timecounter *arch_timer_get_timecounter(void)
{
	return &timecounter;
}

static void __cpuinit arch_timer_stop(struct clock_event_device *clk)
{
	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
		 clk->irq, smp_processor_id());

	if (arch_timer_use_virtual)
		disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
	else {
		disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
	}

	clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
}

static struct local_timer_ops arch_timer_ops __cpuinitdata = {
	.setup	= arch_timer_setup,
	.stop	= arch_timer_stop,
};

static struct clock_event_device arch_timer_global_evt;

static int __init arch_timer_register(void)
{
	int err;
	int ppi;

	err = arch_timer_available();
	if (err)
		goto out;

	arch_timer_evt = alloc_percpu(struct clock_event_device *);
	if (!arch_timer_evt) {
		err = -ENOMEM;
		goto out;
	}

	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	cyclecounter.mult = clocksource_counter.mult;
	cyclecounter.shift = clocksource_counter.shift;
	timecounter_init(&timecounter, &cyclecounter,
			 arch_counter_get_cntpct());

	if (arch_timer_use_virtual) {
		ppi = arch_timer_ppi[VIRT_PPI];
		err = request_percpu_irq(ppi, arch_timer_handler_virt,
					 "arch_timer", arch_timer_evt);
	} else {
		ppi = arch_timer_ppi[PHYS_SECURE_PPI];
		err = request_percpu_irq(ppi, arch_timer_handler_phys,
					 "arch_timer", arch_timer_evt);
		if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
			ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
			err = request_percpu_irq(ppi, arch_timer_handler_phys,
						 "arch_timer", arch_timer_evt);
			if (err)
				free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
						arch_timer_evt);
		}
	}

	if (err) {
		pr_err("arch_timer: can't register interrupt %d (%d)\n",
		       ppi, err);
		goto out_free;
	}

	err = local_timer_register(&arch_timer_ops);
	if (err) {
		/*
		 * We couldn't register as a local timer (could be
		 * because we're on a UP platform, or because some
		 * other local timer is already present...). Try as a
		 * global timer instead.
		 */
		arch_timer_global_evt.cpumask = cpumask_of(0);
		err = arch_timer_setup(&arch_timer_global_evt);
	}
	if (err)
		goto out_free_irq;

	/* Use the architected timer for the delay loop. */
	arch_delay_timer.read_current_timer = &arch_timer_read_current_timer;
	arch_delay_timer.freq = arch_timer_rate;
	register_current_timer_delay(&arch_delay_timer);
	return 0;

out_free_irq:
	if (arch_timer_use_virtual)
		free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
	else {
		free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
				arch_timer_evt);
		if (arch_timer_ppi[PHYS_NONSECURE_PPI])
			free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
					arch_timer_evt);
	}

out_free:
	free_percpu(arch_timer_evt);
out:
	return err;
}

static const struct of_device_id arch_timer_of_match[] __initconst = {
	{ .compatible	= "arm,armv7-timer",	},
	{},
};

int __init arch_timer_of_register(void)
{
	struct device_node *np;
	u32 freq;
	int i;

	np = of_find_matching_node(NULL, arch_timer_of_match);
	if (!np) {
		pr_err("arch_timer: can't find DT node\n");
		return -ENODEV;
	}

	/* Try to determine the frequency from the device tree or CNTFRQ */
	if (!of_property_read_u32(np, "clock-frequency", &freq))
		arch_timer_rate = freq;

	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);

	of_node_put(np);

	/*
	 * If no interrupt provided for virtual timer, we'll have to
	 * stick to the physical timer. It'd better be accessible...
	 */
	if (!arch_timer_ppi[VIRT_PPI]) {
		arch_timer_use_virtual = false;

		if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
		    !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
			pr_warn("arch_timer: No interrupt available, giving up\n");
			return -EINVAL;
		}
	}

	return arch_timer_register();
}

int __init arch_timer_sched_clock_init(void)
{
	u32 (*cnt32)(void);
	int err;

	err = arch_timer_available();
	if (err)
		return err;

	if (arch_timer_use_virtual)
		cnt32 = arch_counter_get_cntvct32;
	else
		cnt32 = arch_counter_get_cntpct32;

	setup_sched_clock(cnt32, 32, arch_timer_rate);
	return 0;
}
Commit	Line	Data
022c03a2 MZ	1	/*
	2	* linux/arch/arm/kernel/arch_timer.c
	3	*
	4	* Copyright (C) 2011 ARM Ltd.
	5	* All Rights Reserved
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*/
	11	#include <linux/init.h>
	12	#include <linux/kernel.h>
	13	#include <linux/delay.h>
	14	#include <linux/device.h>
	15	#include <linux/smp.h>
	16	#include <linux/cpu.h>
	17	#include <linux/jiffies.h>
	18	#include <linux/clockchips.h>
	19	#include <linux/interrupt.h>
0075242b	20	#include <linux/of_irq.h>
022c03a2 MZ	21	#include <linux/io.h>
022c03a2 MZ	22
56942fec	23	#include <asm/delay.h>
022c03a2 MZ	24	#include <asm/localtimer.h>
022c03a2 MZ	25	#include <asm/arch_timer.h>
3f61c80e	26	#include <asm/sched_clock.h>
022c03a2	27
ef01c1d1	28	static u32 arch_timer_rate;
f48b5f12 MZ	29
	30	enum ppi_nr {
	31	PHYS_SECURE_PPI,
	32	PHYS_NONSECURE_PPI,
	33	VIRT_PPI,
	34	HYP_PPI,
	35	MAX_TIMER_PPI
	36	};
	37
	38	static int arch_timer_ppi[MAX_TIMER_PPI];
022c03a2 MZ	39
022c03a2 MZ	40	static struct clock_event_device __percpu **arch_timer_evt;
56942fec	41	static struct delay_timer arch_delay_timer;
d0a533b1	42
f48b5f12 MZ	43	static bool arch_timer_use_virtual = true;
f48b5f12 MZ	44
022c03a2 MZ	45	/*
	46	* Architected system timer support.
	47	*/
	48
	49	#define ARCH_TIMER_CTRL_ENABLE (1 << 0)
	50	#define ARCH_TIMER_CTRL_IT_MASK (1 << 1)
	51	#define ARCH_TIMER_CTRL_IT_STAT (1 << 2)
	52
	53	#define ARCH_TIMER_REG_CTRL 0
	54	#define ARCH_TIMER_REG_FREQ 1
	55	#define ARCH_TIMER_REG_TVAL 2
	56
f48b5f12 MZ	57	#define ARCH_TIMER_PHYS_ACCESS 0
	58	#define ARCH_TIMER_VIRT_ACCESS 1
	59
	60	/*
	61	* These register accessors are marked inline so the compiler can
	62	* nicely work out which register we want, and chuck away the rest of
	63	* the code. At least it does so with a recent GCC (4.6.3).
	64	*/
	65	static inline void arch_timer_reg_write(const int access, const int reg, u32 val)
022c03a2	66	{
f48b5f12 MZ	67	if (access == ARCH_TIMER_PHYS_ACCESS) {
	68	switch (reg) {
	69	case ARCH_TIMER_REG_CTRL:
	70	asm volatile("mcr p15, 0, %0, c14, c2, 1" : : "r" (val));
	71	break;
	72	case ARCH_TIMER_REG_TVAL:
	73	asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r" (val));
	74	break;
	75	}
	76	}
	77
	78	if (access == ARCH_TIMER_VIRT_ACCESS) {
	79	switch (reg) {
	80	case ARCH_TIMER_REG_CTRL:
	81	asm volatile("mcr p15, 0, %0, c14, c3, 1" : : "r" (val));
	82	break;
	83	case ARCH_TIMER_REG_TVAL:
	84	asm volatile("mcr p15, 0, %0, c14, c3, 0" : : "r" (val));
	85	break;
	86	}
022c03a2 MZ	87	}
	88
	89	isb();
	90	}
	91
f48b5f12	92	static inline u32 arch_timer_reg_read(const int access, const int reg)
022c03a2	93	{
f48b5f12 MZ	94	u32 val = 0;
	95
	96	if (access == ARCH_TIMER_PHYS_ACCESS) {
	97	switch (reg) {
	98	case ARCH_TIMER_REG_CTRL:
	99	asm volatile("mrc p15, 0, %0, c14, c2, 1" : "=r" (val));
	100	break;
	101	case ARCH_TIMER_REG_TVAL:
	102	asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (val));
	103	break;
	104	case ARCH_TIMER_REG_FREQ:
	105	asm volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (val));
	106	break;
	107	}
	108	}
022c03a2	109
f48b5f12 MZ	110	if (access == ARCH_TIMER_VIRT_ACCESS) {
	111	switch (reg) {
	112	case ARCH_TIMER_REG_CTRL:
	113	asm volatile("mrc p15, 0, %0, c14, c3, 1" : "=r" (val));
	114	break;
	115	case ARCH_TIMER_REG_TVAL:
	116	asm volatile("mrc p15, 0, %0, c14, c3, 0" : "=r" (val));
	117	break;
	118	}
022c03a2 MZ	119	}
	120
	121	return val;
	122	}
	123
ef01c1d1	124	static inline u64 arch_counter_get_cntpct(void)
022c03a2	125	{
ef01c1d1 MR	126	u64 cval;
ef01c1d1 MR	127	asm volatile("mrrc p15, 0, %Q0, %R0, c14" : "=r" (cval));
f48b5f12 MZ	128	return cval;
f48b5f12 MZ	129	}
022c03a2	130
ef01c1d1	131	static inline u64 arch_counter_get_cntvct(void)
f48b5f12	132	{
ef01c1d1 MR	133	u64 cval;
	134	asm volatile("mrrc p15, 1, %Q0, %R0, c14" : "=r" (cval));
	135	return cval;
f48b5f12 MZ	136	}
	137
	138	static irqreturn_t inline timer_handler(const int access,
	139	struct clock_event_device *evt)
	140	{
	141	unsigned long ctrl;
	142	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
022c03a2 MZ	143	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
022c03a2 MZ	144	ctrl \|= ARCH_TIMER_CTRL_IT_MASK;
f48b5f12	145	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
022c03a2 MZ	146	evt->event_handler(evt);
	147	return IRQ_HANDLED;
	148	}
	149
	150	return IRQ_NONE;
	151	}
	152
f48b5f12	153	static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
022c03a2	154	{
f48b5f12	155	struct clock_event_device evt = (struct clock_event_device **)dev_id;
022c03a2	156
f48b5f12	157	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
022c03a2 MZ	158	}
022c03a2 MZ	159
f48b5f12	160	static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
022c03a2	161	{
f48b5f12 MZ	162	struct clock_event_device evt = (struct clock_event_device **)dev_id;
	163
	164	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
	165	}
	166
	167	static inline void timer_set_mode(const int access, int mode)
	168	{
	169	unsigned long ctrl;
022c03a2 MZ	170	switch (mode) {
	171	case CLOCK_EVT_MODE_UNUSED:
	172	case CLOCK_EVT_MODE_SHUTDOWN:
f48b5f12 MZ	173	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
	174	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
	175	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
022c03a2 MZ	176	break;
	177	default:
	178	break;
	179	}
	180	}
	181
f48b5f12 MZ	182	static void arch_timer_set_mode_virt(enum clock_event_mode mode,
f48b5f12 MZ	183	struct clock_event_device *clk)
022c03a2	184	{
f48b5f12 MZ	185	timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode);
	186	}
	187
	188	static void arch_timer_set_mode_phys(enum clock_event_mode mode,
	189	struct clock_event_device *clk)
	190	{
	191	timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode);
	192	}
022c03a2	193
f48b5f12 MZ	194	static inline void set_next_event(const int access, unsigned long evt)
	195	{
	196	unsigned long ctrl;
	197	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL);
022c03a2 MZ	198	ctrl \|= ARCH_TIMER_CTRL_ENABLE;
022c03a2 MZ	199	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
f48b5f12 MZ	200	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt);
	201	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl);
	202	}
022c03a2	203
f48b5f12 MZ	204	static int arch_timer_set_next_event_virt(unsigned long evt,
	205	struct clock_event_device *unused)
	206	{
	207	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt);
	208	return 0;
	209	}
022c03a2	210
f48b5f12 MZ	211	static int arch_timer_set_next_event_phys(unsigned long evt,
	212	struct clock_event_device *unused)
	213	{
	214	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt);
022c03a2 MZ	215	return 0;
	216	}
	217
	218	static int __cpuinit arch_timer_setup(struct clock_event_device *clk)
	219	{
27a5569d	220	clk->features = CLOCK_EVT_FEAT_ONESHOT \| CLOCK_EVT_FEAT_C3STOP;
022c03a2 MZ	221	clk->name = "arch_sys_timer";
022c03a2 MZ	222	clk->rating = 450;
f48b5f12 MZ	223	if (arch_timer_use_virtual) {
	224	clk->irq = arch_timer_ppi[VIRT_PPI];
	225	clk->set_mode = arch_timer_set_mode_virt;
	226	clk->set_next_event = arch_timer_set_next_event_virt;
	227	} else {
	228	clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
	229	clk->set_mode = arch_timer_set_mode_phys;
	230	clk->set_next_event = arch_timer_set_next_event_phys;
	231	}
	232
	233	clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, NULL);
022c03a2 MZ	234
	235	clockevents_config_and_register(clk, arch_timer_rate,
	236	0xf, 0x7fffffff);
	237
	238	*__this_cpu_ptr(arch_timer_evt) = clk;
	239
f48b5f12 MZ	240	if (arch_timer_use_virtual)
	241	enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
	242	else {
	243	enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
	244	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	245	enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
	246	}
022c03a2 MZ	247
	248	return 0;
	249	}
	250
022c03a2 MZ	251	static int arch_timer_available(void)
022c03a2 MZ	252	{
ef01c1d1	253	u32 freq;
022c03a2	254
022c03a2	255	if (arch_timer_rate == 0) {
f48b5f12 MZ	256	freq = arch_timer_reg_read(ARCH_TIMER_PHYS_ACCESS,
f48b5f12 MZ	257	ARCH_TIMER_REG_FREQ);
022c03a2 MZ	258
	259	/* Check the timer frequency. */
	260	if (freq == 0) {
	261	pr_warn("Architected timer frequency not available\n");
	262	return -EINVAL;
	263	}
	264
	265	arch_timer_rate = freq;
	266	}
	267
f48b5f12	268	pr_info_once("Architected local timer running at %lu.%02luMHz (%s).\n",
ef01c1d1 MR	269	(unsigned long)arch_timer_rate / 1000000,
ef01c1d1 MR	270	(unsigned long)(arch_timer_rate / 10000) % 100,
f48b5f12	271	arch_timer_use_virtual ? "virt" : "phys");
022c03a2 MZ	272	return 0;
	273	}
	274
f48b5f12	275	static u32 notrace arch_counter_get_cntpct32(void)
022c03a2	276	{
f48b5f12	277	cycle_t cnt = arch_counter_get_cntpct();
022c03a2	278
f48b5f12 MZ	279	/*
	280	* The sched_clock infrastructure only knows about counters
	281	* with at most 32bits. Forget about the upper 24 bits for the
	282	* time being...
	283	*/
	284	return (u32)cnt;
022c03a2 MZ	285	}
022c03a2 MZ	286
3f61c80e MZ	287	static u32 notrace arch_counter_get_cntvct32(void)
3f61c80e MZ	288	{
f48b5f12	289	cycle_t cnt = arch_counter_get_cntvct();
3f61c80e MZ	290
	291	/*
	292	* The sched_clock infrastructure only knows about counters
	293	* with at most 32bits. Forget about the upper 24 bits for the
	294	* time being...
	295	*/
f48b5f12	296	return (u32)cnt;
3f61c80e MZ	297	}
3f61c80e MZ	298
022c03a2 MZ	299	static cycle_t arch_counter_read(struct clocksource *cs)
022c03a2 MZ	300	{
f48b5f12 MZ	301	/*
	302	* Always use the physical counter for the clocksource.
	303	* CNTHCTL.PL1PCTEN must be set to 1.
	304	*/
022c03a2 MZ	305	return arch_counter_get_cntpct();
	306	}
	307
56942fec	308	static unsigned long arch_timer_read_current_timer(void)
923df96b	309	{
56942fec	310	return arch_counter_get_cntpct();
923df96b WD	311	}
923df96b WD	312
a1b2dde7 MZ	313	static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
	314	{
	315	/*
	316	* Always use the physical counter for the clocksource.
	317	* CNTHCTL.PL1PCTEN must be set to 1.
	318	*/
	319	return arch_counter_get_cntpct();
	320	}
	321
022c03a2 MZ	322	static struct clocksource clocksource_counter = {
	323	.name = "arch_sys_counter",
	324	.rating = 400,
	325	.read = arch_counter_read,
	326	.mask = CLOCKSOURCE_MASK(56),
	327	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
	328	};
	329
a1b2dde7 MZ	330	static struct cyclecounter cyclecounter = {
	331	.read = arch_counter_read_cc,
	332	.mask = CLOCKSOURCE_MASK(56),
	333	};
	334
	335	static struct timecounter timecounter;
	336
	337	struct timecounter *arch_timer_get_timecounter(void)
	338	{
	339	return &timecounter;
	340	}
	341
022c03a2 MZ	342	static void __cpuinit arch_timer_stop(struct clock_event_device *clk)
	343	{
	344	pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
	345	clk->irq, smp_processor_id());
f48b5f12 MZ	346
	347	if (arch_timer_use_virtual)
	348	disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
	349	else {
	350	disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
	351	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	352	disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
	353	}
	354
	355	clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
022c03a2 MZ	356	}
	357
	358	static struct local_timer_ops arch_timer_ops __cpuinitdata = {
	359	.setup = arch_timer_setup,
	360	.stop = arch_timer_stop,
	361	};
	362
273d16ad MZ	363	static struct clock_event_device arch_timer_global_evt;
273d16ad MZ	364
fb8a99f9	365	static int __init arch_timer_register(void)
022c03a2 MZ	366	{
022c03a2 MZ	367	int err;
f48b5f12	368	int ppi;
022c03a2	369
022c03a2 MZ	370	err = arch_timer_available();
022c03a2 MZ	371	if (err)
f48b5f12	372	goto out;
022c03a2 MZ	373
022c03a2 MZ	374	arch_timer_evt = alloc_percpu(struct clock_event_device *);
f48b5f12 MZ	375	if (!arch_timer_evt) {
	376	err = -ENOMEM;
	377	goto out;
	378	}
022c03a2 MZ	379
022c03a2 MZ	380	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
a1b2dde7 MZ	381	cyclecounter.mult = clocksource_counter.mult;
	382	cyclecounter.shift = clocksource_counter.shift;
	383	timecounter_init(&timecounter, &cyclecounter,
	384	arch_counter_get_cntpct());
022c03a2	385
f48b5f12 MZ	386	if (arch_timer_use_virtual) {
	387	ppi = arch_timer_ppi[VIRT_PPI];
	388	err = request_percpu_irq(ppi, arch_timer_handler_virt,
	389	"arch_timer", arch_timer_evt);
	390	} else {
	391	ppi = arch_timer_ppi[PHYS_SECURE_PPI];
	392	err = request_percpu_irq(ppi, arch_timer_handler_phys,
	393	"arch_timer", arch_timer_evt);
	394	if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
	395	ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
	396	err = request_percpu_irq(ppi, arch_timer_handler_phys,
	397	"arch_timer", arch_timer_evt);
	398	if (err)
	399	free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
	400	arch_timer_evt);
	401	}
	402	}
	403
022c03a2 MZ	404	if (err) {
022c03a2 MZ	405	pr_err("arch_timer: can't register interrupt %d (%d)\n",
f48b5f12	406	ppi, err);
022c03a2 MZ	407	goto out_free;
	408	}
	409
022c03a2	410	err = local_timer_register(&arch_timer_ops);
273d16ad MZ	411	if (err) {
	412	/*
	413	* We couldn't register as a local timer (could be
	414	* because we're on a UP platform, or because some
	415	* other local timer is already present...). Try as a
	416	* global timer instead.
	417	*/
	418	arch_timer_global_evt.cpumask = cpumask_of(0);
	419	err = arch_timer_setup(&arch_timer_global_evt);
	420	}
022c03a2 MZ	421	if (err)
	422	goto out_free_irq;
	423
56942fec JA	424	/* Use the architected timer for the delay loop. */
	425	arch_delay_timer.read_current_timer = &arch_timer_read_current_timer;
	426	arch_delay_timer.freq = arch_timer_rate;
	427	register_current_timer_delay(&arch_delay_timer);
022c03a2 MZ	428	return 0;
	429
	430	out_free_irq:
f48b5f12 MZ	431	if (arch_timer_use_virtual)
	432	free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
	433	else {
	434	free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
	435	arch_timer_evt);
	436	if (arch_timer_ppi[PHYS_NONSECURE_PPI])
	437	free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
	438	arch_timer_evt);
	439	}
022c03a2 MZ	440
	441	out_free:
	442	free_percpu(arch_timer_evt);
f48b5f12	443	out:
022c03a2 MZ	444	return err;
022c03a2 MZ	445	}
3f61c80e	446
0075242b MZ	447	static const struct of_device_id arch_timer_of_match[] __initconst = {
	448	{ .compatible = "arm,armv7-timer", },
	449	{},
	450	};
	451
	452	int __init arch_timer_of_register(void)
	453	{
	454	struct device_node *np;
	455	u32 freq;
f48b5f12	456	int i;
0075242b MZ	457
	458	np = of_find_matching_node(NULL, arch_timer_of_match);
	459	if (!np) {
	460	pr_err("arch_timer: can't find DT node\n");
	461	return -ENODEV;
	462	}
	463
	464	/* Try to determine the frequency from the device tree or CNTFRQ */
	465	if (!of_property_read_u32(np, "clock-frequency", &freq))
	466	arch_timer_rate = freq;
	467
f48b5f12 MZ	468	for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
	469	arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
	470
2b55d10c MR	471	of_node_put(np);
2b55d10c MR	472
f48b5f12 MZ	473	/*
	474	* If no interrupt provided for virtual timer, we'll have to
	475	* stick to the physical timer. It'd better be accessible...
	476	*/
	477	if (!arch_timer_ppi[VIRT_PPI]) {
	478	arch_timer_use_virtual = false;
	479
	480	if (!arch_timer_ppi[PHYS_SECURE_PPI] \|\|
	481	!arch_timer_ppi[PHYS_NONSECURE_PPI]) {
	482	pr_warn("arch_timer: No interrupt available, giving up\n");
	483	return -EINVAL;
	484	}
	485	}
0075242b	486
fb8a99f9	487	return arch_timer_register();
0075242b	488	}
0075242b	489
3f61c80e MZ	490	int __init arch_timer_sched_clock_init(void)
3f61c80e MZ	491	{
f48b5f12	492	u32 (*cnt32)(void);
3f61c80e MZ	493	int err;
	494
	495	err = arch_timer_available();
	496	if (err)
	497	return err;
	498
f48b5f12 MZ	499	if (arch_timer_use_virtual)
	500	cnt32 = arch_counter_get_cntvct32;
	501	else
	502	cnt32 = arch_counter_get_cntpct32;
	503
	504	setup_sched_clock(cnt32, 32, arch_timer_rate);
3f61c80e MZ	505	return 0;
3f61c80e MZ	506	}