[linux-2.6-block.git] / arch / sparc / kernel / perf_counter.c

/* Performance counter support for sparc64.
 *
 * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
 *
 * This code is based almost entirely upon the x86 perf counter
 * code, which is:
 *
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 */

#include <linux/perf_counter.h>
#include <linux/kprobes.h>
#include <linux/kernel.h>
#include <linux/kdebug.h>
#include <linux/mutex.h>

#include <asm/cpudata.h>
#include <asm/atomic.h>
#include <asm/nmi.h>
#include <asm/pcr.h>

/* Sparc64 chips have two performance counters, 32-bits each, with
 * overflow interrupts generated on transition from 0xffffffff to 0.
 * The counters are accessed in one go using a 64-bit register.
 *
 * Both counters are controlled using a single control register.  The
 * only way to stop all sampling is to clear all of the context (user,
 * supervisor, hypervisor) sampling enable bits.  But these bits apply
 * to both counters, thus the two counters can't be enabled/disabled
 * individually.
 *
 * The control register has two event fields, one for each of the two
 * counters.  It's thus nearly impossible to have one counter going
 * while keeping the other one stopped.  Therefore it is possible to
 * get overflow interrupts for counters not currently "in use" and
 * that condition must be checked in the overflow interrupt handler.
 *
 * So we use a hack, in that we program inactive counters with the
 * "sw_count0" and "sw_count1" events.  These count how many times
 * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
 * unusual way to encode a NOP and therefore will not trigger in
 * normal code.
 */

#define MAX_HWCOUNTERS			2
#define MAX_PERIOD			((1UL << 32) - 1)

#define PIC_UPPER_INDEX			0
#define PIC_LOWER_INDEX			1

struct cpu_hw_counters {
	struct perf_counter	*counters[MAX_HWCOUNTERS];
	unsigned long		used_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
	unsigned long		active_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
	int enabled;
};
DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = { .enabled = 1, };

struct perf_event_map {
	u16	encoding;
	u8	pic_mask;
#define PIC_NONE	0x00
#define PIC_UPPER	0x01
#define PIC_LOWER	0x02
};

struct sparc_pmu {
	const struct perf_event_map	*(*event_map)(int);
	int				max_events;
	int				upper_shift;
	int				lower_shift;
	int				event_mask;
	int				hv_bit;
	int				irq_bit;
	int				upper_nop;
	int				lower_nop;
};

static const struct perf_event_map ultra3i_perfmon_event_map[] = {
	[PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
};

static const struct perf_event_map *ultra3i_event_map(int event)
{
	return &ultra3i_perfmon_event_map[event];
}

static const struct sparc_pmu ultra3i_pmu = {
	.event_map	= ultra3i_event_map,
	.max_events	= ARRAY_SIZE(ultra3i_perfmon_event_map),
	.upper_shift	= 11,
	.lower_shift	= 4,
	.event_mask	= 0x3f,
	.upper_nop	= 0x1c,
	.lower_nop	= 0x14,
};

static const struct sparc_pmu *sparc_pmu __read_mostly;

static u64 event_encoding(u64 event, int idx)
{
	if (idx == PIC_UPPER_INDEX)
		event <<= sparc_pmu->upper_shift;
	else
		event <<= sparc_pmu->lower_shift;
	return event;
}

static u64 mask_for_index(int idx)
{
	return event_encoding(sparc_pmu->event_mask, idx);
}

static u64 nop_for_index(int idx)
{
	return event_encoding(idx == PIC_UPPER_INDEX ?
			      sparc_pmu->upper_nop :
			      sparc_pmu->lower_nop, idx);
}

static inline void sparc_pmu_enable_counter(struct hw_perf_counter *hwc,
					    int idx)
{
	u64 val, mask = mask_for_index(idx);

	val = pcr_ops->read();
	pcr_ops->write((val & ~mask) | hwc->config);
}

static inline void sparc_pmu_disable_counter(struct hw_perf_counter *hwc,
					     int idx)
{
	u64 mask = mask_for_index(idx);
	u64 nop = nop_for_index(idx);
	u64 val = pcr_ops->read();

	pcr_ops->write((val & ~mask) | nop);
}

void hw_perf_enable(void)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	u64 val;
	int i;

	if (cpuc->enabled)
		return;

	cpuc->enabled = 1;
	barrier();

	val = pcr_ops->read();

	for (i = 0; i < MAX_HWCOUNTERS; i++) {
		struct perf_counter *cp = cpuc->counters[i];
		struct hw_perf_counter *hwc;

		if (!cp)
			continue;
		hwc = &cp->hw;
		val |= hwc->config_base;
	}

	pcr_ops->write(val);
}

void hw_perf_disable(void)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	u64 val;

	if (!cpuc->enabled)
		return;

	cpuc->enabled = 0;

	val = pcr_ops->read();
	val &= ~(PCR_UTRACE | PCR_STRACE |
		 sparc_pmu->hv_bit | sparc_pmu->irq_bit);
	pcr_ops->write(val);
}

static u32 read_pmc(int idx)
{
	u64 val;

	read_pic(val);
	if (idx == PIC_UPPER_INDEX)
		val >>= 32;

	return val & 0xffffffff;
}

static void write_pmc(int idx, u64 val)
{
	u64 shift, mask, pic;

	shift = 0;
	if (idx == PIC_UPPER_INDEX)
		shift = 32;

	mask = ((u64) 0xffffffff) << shift;
	val <<= shift;

	read_pic(pic);
	pic &= ~mask;
	pic |= val;
	write_pic(pic);
}

static int sparc_perf_counter_set_period(struct perf_counter *counter,
					 struct hw_perf_counter *hwc, int idx)
{
	s64 left = atomic64_read(&hwc->period_left);
	s64 period = hwc->sample_period;
	int ret = 0;

	if (unlikely(left <= -period)) {
		left = period;
		atomic64_set(&hwc->period_left, left);
		hwc->last_period = period;
		ret = 1;
	}

	if (unlikely(left <= 0)) {
		left += period;
		atomic64_set(&hwc->period_left, left);
		hwc->last_period = period;
		ret = 1;
	}
	if (left > MAX_PERIOD)
		left = MAX_PERIOD;

	atomic64_set(&hwc->prev_count, (u64)-left);

	write_pmc(idx, (u64)(-left) & 0xffffffff);

	perf_counter_update_userpage(counter);

	return ret;
}

static int sparc_pmu_enable(struct perf_counter *counter)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
	int idx = hwc->idx;

	if (test_and_set_bit(idx, cpuc->used_mask))
		return -EAGAIN;

	sparc_pmu_disable_counter(hwc, idx);

	cpuc->counters[idx] = counter;
	set_bit(idx, cpuc->active_mask);

	sparc_perf_counter_set_period(counter, hwc, idx);
	sparc_pmu_enable_counter(hwc, idx);
	perf_counter_update_userpage(counter);
	return 0;
}

static u64 sparc_perf_counter_update(struct perf_counter *counter,
				     struct hw_perf_counter *hwc, int idx)
{
	int shift = 64 - 32;
	u64 prev_raw_count, new_raw_count;
	s64 delta;

again:
	prev_raw_count = atomic64_read(&hwc->prev_count);
	new_raw_count = read_pmc(idx);

	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
			     new_raw_count) != prev_raw_count)
		goto again;

	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	delta >>= shift;

	atomic64_add(delta, &counter->count);
	atomic64_sub(delta, &hwc->period_left);

	return new_raw_count;
}

static void sparc_pmu_disable(struct perf_counter *counter)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
	int idx = hwc->idx;

	clear_bit(idx, cpuc->active_mask);
	sparc_pmu_disable_counter(hwc, idx);

	barrier();

	sparc_perf_counter_update(counter, hwc, idx);
	cpuc->counters[idx] = NULL;
	clear_bit(idx, cpuc->used_mask);

	perf_counter_update_userpage(counter);
}

static void sparc_pmu_read(struct perf_counter *counter)
{
	struct hw_perf_counter *hwc = &counter->hw;
	sparc_perf_counter_update(counter, hwc, hwc->idx);
}

static void sparc_pmu_unthrottle(struct perf_counter *counter)
{
	struct hw_perf_counter *hwc = &counter->hw;
	sparc_pmu_enable_counter(hwc, hwc->idx);
}

static atomic_t active_counters = ATOMIC_INIT(0);
static DEFINE_MUTEX(pmc_grab_mutex);

void perf_counter_grab_pmc(void)
{
	if (atomic_inc_not_zero(&active_counters))
		return;

	mutex_lock(&pmc_grab_mutex);
	if (atomic_read(&active_counters) == 0) {
		if (atomic_read(&nmi_active) > 0) {
			on_each_cpu(stop_nmi_watchdog, NULL, 1);
			BUG_ON(atomic_read(&nmi_active) != 0);
		}
		atomic_inc(&active_counters);
	}
	mutex_unlock(&pmc_grab_mutex);
}

void perf_counter_release_pmc(void)
{
	if (atomic_dec_and_mutex_lock(&active_counters, &pmc_grab_mutex)) {
		if (atomic_read(&nmi_active) == 0)
			on_each_cpu(start_nmi_watchdog, NULL, 1);
		mutex_unlock(&pmc_grab_mutex);
	}
}

static void hw_perf_counter_destroy(struct perf_counter *counter)
{
	perf_counter_release_pmc();
}

static int __hw_perf_counter_init(struct perf_counter *counter)
{
	struct perf_counter_attr *attr = &counter->attr;
	struct hw_perf_counter *hwc = &counter->hw;
	const struct perf_event_map *pmap;
	u64 enc;

	if (atomic_read(&nmi_active) < 0)
		return -ENODEV;

	if (attr->type != PERF_TYPE_HARDWARE)
		return -EOPNOTSUPP;

	if (attr->config >= sparc_pmu->max_events)
		return -EINVAL;

	perf_counter_grab_pmc();
	counter->destroy = hw_perf_counter_destroy;

	/* We save the enable bits in the config_base.  So to
	 * turn off sampling just write 'config', and to enable
	 * things write 'config | config_base'.
	 */
	hwc->config_base = sparc_pmu->irq_bit;
	if (!attr->exclude_user)
		hwc->config_base |= PCR_UTRACE;
	if (!attr->exclude_kernel)
		hwc->config_base |= PCR_STRACE;
	if (!attr->exclude_hv)
		hwc->config_base |= sparc_pmu->hv_bit;

	if (!hwc->sample_period) {
		hwc->sample_period = MAX_PERIOD;
		hwc->last_period = hwc->sample_period;
		atomic64_set(&hwc->period_left, hwc->sample_period);
	}

	pmap = sparc_pmu->event_map(attr->config);

	enc = pmap->encoding;
	if (pmap->pic_mask & PIC_UPPER) {
		hwc->idx = PIC_UPPER_INDEX;
		enc <<= sparc_pmu->upper_shift;
	} else {
		hwc->idx = PIC_LOWER_INDEX;
		enc <<= sparc_pmu->lower_shift;
	}

	hwc->config |= enc;
	return 0;
}

static const struct pmu pmu = {
	.enable		= sparc_pmu_enable,
	.disable	= sparc_pmu_disable,
	.read		= sparc_pmu_read,
	.unthrottle	= sparc_pmu_unthrottle,
};

const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
{
	int err = __hw_perf_counter_init(counter);

	if (err)
		return ERR_PTR(err);
	return &pmu;
}

void perf_counter_print_debug(void)
{
	unsigned long flags;
	u64 pcr, pic;
	int cpu;

	if (!sparc_pmu)
		return;

	local_irq_save(flags);

	cpu = smp_processor_id();

	pcr = pcr_ops->read();
	read_pic(pic);

	pr_info("\n");
	pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
		cpu, pcr, pic);

	local_irq_restore(flags);
}

static int __kprobes perf_counter_nmi_handler(struct notifier_block *self,
					      unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
	struct perf_sample_data data;
	struct cpu_hw_counters *cpuc;
	struct pt_regs *regs;
	int idx;

	if (!atomic_read(&active_counters))
		return NOTIFY_DONE;

	switch (cmd) {
	case DIE_NMI:
		break;

	default:
		return NOTIFY_DONE;
	}

	regs = args->regs;

	data.regs = regs;
	data.addr = 0;

	cpuc = &__get_cpu_var(cpu_hw_counters);
	for (idx = 0; idx < MAX_HWCOUNTERS; idx++) {
		struct perf_counter *counter = cpuc->counters[idx];
		struct hw_perf_counter *hwc;
		u64 val;

		if (!test_bit(idx, cpuc->active_mask))
			continue;
		hwc = &counter->hw;
		val = sparc_perf_counter_update(counter, hwc, idx);
		if (val & (1ULL << 31))
			continue;

		data.period = counter->hw.last_period;
		if (!sparc_perf_counter_set_period(counter, hwc, idx))
			continue;

		if (perf_counter_overflow(counter, 1, &data))
			sparc_pmu_disable_counter(hwc, idx);
	}

	return NOTIFY_STOP;
}

static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
	.notifier_call		= perf_counter_nmi_handler,
};

static bool __init supported_pmu(void)
{
	if (!strcmp(sparc_pmu_type, "ultra3i")) {
		sparc_pmu = &ultra3i_pmu;
		return true;
	}
	return false;
}

void __init init_hw_perf_counters(void)
{
	pr_info("Performance counters: ");

	if (!supported_pmu()) {
		pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
		return;
	}

	pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);

	/* All sparc64 PMUs currently have 2 counters.  But this simple
	 * driver only supports one active counter at a time.
	 */
	perf_max_counters = 1;

	register_die_notifier(&perf_counter_nmi_notifier);
}
Commit	Line	Data
59abbd1e DM	1	/* Performance counter support for sparc64.
	2	*
	3	* Copyright (C) 2009 David S. Miller <davem@davemloft.net>
	4	*
	5	* This code is based almost entirely upon the x86 perf counter
	6	* code, which is:
	7	*
	8	* Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
	9	* Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
	10	* Copyright (C) 2009 Jaswinder Singh Rajput
	11	* Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
	12	* Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
	13	*/
	14
	15	#include <linux/perf_counter.h>
	16	#include <linux/kprobes.h>
	17	#include <linux/kernel.h>
	18	#include <linux/kdebug.h>
	19	#include <linux/mutex.h>
	20
	21	#include <asm/cpudata.h>
	22	#include <asm/atomic.h>
	23	#include <asm/nmi.h>
	24	#include <asm/pcr.h>
	25
	26	/* Sparc64 chips have two performance counters, 32-bits each, with
	27	* overflow interrupts generated on transition from 0xffffffff to 0.
	28	* The counters are accessed in one go using a 64-bit register.
	29	*
	30	* Both counters are controlled using a single control register. The
	31	* only way to stop all sampling is to clear all of the context (user,
	32	* supervisor, hypervisor) sampling enable bits. But these bits apply
	33	* to both counters, thus the two counters can't be enabled/disabled
	34	* individually.
	35	*
	36	* The control register has two event fields, one for each of the two
	37	* counters. It's thus nearly impossible to have one counter going
	38	* while keeping the other one stopped. Therefore it is possible to
	39	* get overflow interrupts for counters not currently "in use" and
	40	* that condition must be checked in the overflow interrupt handler.
	41	*
	42	* So we use a hack, in that we program inactive counters with the
	43	* "sw_count0" and "sw_count1" events. These count how many times
	44	* the instruction "sethi %hi(0xfc000), %g0" is executed. It's an
	45	* unusual way to encode a NOP and therefore will not trigger in
	46	* normal code.
	47	*/
	48
	49	#define MAX_HWCOUNTERS 2
	50	#define MAX_PERIOD ((1UL << 32) - 1)
	51
	52	#define PIC_UPPER_INDEX 0
	53	#define PIC_LOWER_INDEX 1
	54
59abbd1e DM	55	struct cpu_hw_counters {
	56	struct perf_counter *counters[MAX_HWCOUNTERS];
	57	unsigned long used_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
	58	unsigned long active_mask[BITS_TO_LONGS(MAX_HWCOUNTERS)];
	59	int enabled;
	60	};
	61	DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = { .enabled = 1, };
	62
	63	struct perf_event_map {
	64	u16 encoding;
	65	u8 pic_mask;
	66	#define PIC_NONE 0x00
	67	#define PIC_UPPER 0x01
	68	#define PIC_LOWER 0x02
	69	};
	70
	71	struct sparc_pmu {
	72	const struct perf_event_map (event_map)(int);
	73	int max_events;
	74	int upper_shift;
	75	int lower_shift;
	76	int event_mask;
91b9286d	77	int hv_bit;
496c07e3	78	int irq_bit;
660d1376 DM	79	int upper_nop;
660d1376 DM	80	int lower_nop;
59abbd1e DM	81	};
	82
	83	static const struct perf_event_map ultra3i_perfmon_event_map[] = {
	84	[PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER \| PIC_LOWER },
	85	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER \| PIC_LOWER },
	86	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
	87	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
	88	};
	89
	90	static const struct perf_event_map *ultra3i_event_map(int event)
	91	{
	92	return &ultra3i_perfmon_event_map[event];
	93	}
	94
	95	static const struct sparc_pmu ultra3i_pmu = {
	96	.event_map = ultra3i_event_map,
	97	.max_events = ARRAY_SIZE(ultra3i_perfmon_event_map),
	98	.upper_shift = 11,
	99	.lower_shift = 4,
	100	.event_mask = 0x3f,
660d1376 DM	101	.upper_nop = 0x1c,
660d1376 DM	102	.lower_nop = 0x14,
59abbd1e DM	103	};
	104
	105	static const struct sparc_pmu *sparc_pmu __read_mostly;
	106
	107	static u64 event_encoding(u64 event, int idx)
	108	{
	109	if (idx == PIC_UPPER_INDEX)
	110	event <<= sparc_pmu->upper_shift;
	111	else
	112	event <<= sparc_pmu->lower_shift;
	113	return event;
	114	}
	115
	116	static u64 mask_for_index(int idx)
	117	{
	118	return event_encoding(sparc_pmu->event_mask, idx);
	119	}
	120
	121	static u64 nop_for_index(int idx)
	122	{
	123	return event_encoding(idx == PIC_UPPER_INDEX ?
660d1376 DM	124	sparc_pmu->upper_nop :
660d1376 DM	125	sparc_pmu->lower_nop, idx);
59abbd1e DM	126	}
	127
	128	static inline void sparc_pmu_enable_counter(struct hw_perf_counter *hwc,
	129	int idx)
	130	{
	131	u64 val, mask = mask_for_index(idx);
	132
	133	val = pcr_ops->read();
	134	pcr_ops->write((val & ~mask) \| hwc->config);
	135	}
	136
	137	static inline void sparc_pmu_disable_counter(struct hw_perf_counter *hwc,
	138	int idx)
	139	{
	140	u64 mask = mask_for_index(idx);
	141	u64 nop = nop_for_index(idx);
	142	u64 val = pcr_ops->read();
	143
	144	pcr_ops->write((val & ~mask) \| nop);
	145	}
	146
	147	void hw_perf_enable(void)
	148	{
	149	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	150	u64 val;
	151	int i;
	152
	153	if (cpuc->enabled)
	154	return;
	155
	156	cpuc->enabled = 1;
	157	barrier();
	158
	159	val = pcr_ops->read();
	160
	161	for (i = 0; i < MAX_HWCOUNTERS; i++) {
	162	struct perf_counter *cp = cpuc->counters[i];
	163	struct hw_perf_counter *hwc;
	164
	165	if (!cp)
	166	continue;
	167	hwc = &cp->hw;
	168	val \|= hwc->config_base;
	169	}
	170
	171	pcr_ops->write(val);
	172	}
	173
	174	void hw_perf_disable(void)
	175	{
	176	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	177	u64 val;
	178
	179	if (!cpuc->enabled)
	180	return;
	181
	182	cpuc->enabled = 0;
	183
	184	val = pcr_ops->read();
496c07e3 DM	185	val &= ~(PCR_UTRACE \| PCR_STRACE \|
496c07e3 DM	186	sparc_pmu->hv_bit \| sparc_pmu->irq_bit);
59abbd1e DM	187	pcr_ops->write(val);
	188	}
	189
	190	static u32 read_pmc(int idx)
	191	{
	192	u64 val;
	193
	194	read_pic(val);
	195	if (idx == PIC_UPPER_INDEX)
	196	val >>= 32;
	197
	198	return val & 0xffffffff;
	199	}
	200
	201	static void write_pmc(int idx, u64 val)
	202	{
	203	u64 shift, mask, pic;
	204
	205	shift = 0;
	206	if (idx == PIC_UPPER_INDEX)
	207	shift = 32;
	208
	209	mask = ((u64) 0xffffffff) << shift;
	210	val <<= shift;
	211
	212	read_pic(pic);
	213	pic &= ~mask;
	214	pic \|= val;
	215	write_pic(pic);
	216	}
	217
	218	static int sparc_perf_counter_set_period(struct perf_counter *counter,
	219	struct hw_perf_counter *hwc, int idx)
	220	{
	221	s64 left = atomic64_read(&hwc->period_left);
	222	s64 period = hwc->sample_period;
	223	int ret = 0;
	224
	225	if (unlikely(left <= -period)) {
	226	left = period;
	227	atomic64_set(&hwc->period_left, left);
	228	hwc->last_period = period;
	229	ret = 1;
	230	}
	231
	232	if (unlikely(left <= 0)) {
	233	left += period;
	234	atomic64_set(&hwc->period_left, left);
	235	hwc->last_period = period;
	236	ret = 1;
	237	}
	238	if (left > MAX_PERIOD)
	239	left = MAX_PERIOD;
	240
	241	atomic64_set(&hwc->prev_count, (u64)-left);
	242
	243	write_pmc(idx, (u64)(-left) & 0xffffffff);
	244
	245	perf_counter_update_userpage(counter);
	246
	247	return ret;
	248	}
	249
	250	static int sparc_pmu_enable(struct perf_counter *counter)
251	{
252	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
253	struct hw_perf_counter *hwc = &counter->hw;
254	int idx = hwc->idx;
255
256	if (test_and_set_bit(idx, cpuc->used_mask))
257	return -EAGAIN;
258
259	sparc_pmu_disable_counter(hwc, idx);
260
261	cpuc->counters[idx] = counter;
262	set_bit(idx, cpuc->active_mask);
263
264	sparc_perf_counter_set_period(counter, hwc, idx);
265	sparc_pmu_enable_counter(hwc, idx);
266	perf_counter_update_userpage(counter);
267	return 0;
268	}
269
270	static u64 sparc_perf_counter_update(struct perf_counter *counter,
271	struct hw_perf_counter *hwc, int idx)
272	{
273	int shift = 64 - 32;
274	u64 prev_raw_count, new_raw_count;
275	s64 delta;
276
277	again:
278	prev_raw_count = atomic64_read(&hwc->prev_count);
279	new_raw_count = read_pmc(idx);
280
281	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
282	new_raw_count) != prev_raw_count)
283	goto again;
284
285	delta = (new_raw_count << shift) - (prev_raw_count << shift);
286	delta >>= shift;
287
288	atomic64_add(delta, &counter->count);
289	atomic64_sub(delta, &hwc->period_left);
290
291	return new_raw_count;
292	}
293
294	static void sparc_pmu_disable(struct perf_counter *counter)
295	{
296	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
297	struct hw_perf_counter *hwc = &counter->hw;
298	int idx = hwc->idx;
299
300	clear_bit(idx, cpuc->active_mask);
301	sparc_pmu_disable_counter(hwc, idx);
302
303	barrier();
304
305	sparc_perf_counter_update(counter, hwc, idx);
306	cpuc->counters[idx] = NULL;
307	clear_bit(idx, cpuc->used_mask);
308
309	perf_counter_update_userpage(counter);
310	}
311
312	static void sparc_pmu_read(struct perf_counter *counter)
313	{
314	struct hw_perf_counter *hwc = &counter->hw;
315	sparc_perf_counter_update(counter, hwc, hwc->idx);
316	}
317
318	static void sparc_pmu_unthrottle(struct perf_counter *counter)
319	{
320	struct hw_perf_counter *hwc = &counter->hw;
321	sparc_pmu_enable_counter(hwc, hwc->idx);
322	}
323
324	static atomic_t active_counters = ATOMIC_INIT(0);
325	static DEFINE_MUTEX(pmc_grab_mutex);
326
327	void perf_counter_grab_pmc(void)
328	{
329	if (atomic_inc_not_zero(&active_counters))
330	return;
331
332	mutex_lock(&pmc_grab_mutex);
333	if (atomic_read(&active_counters) == 0) {
334	if (atomic_read(&nmi_active) > 0) {
335	on_each_cpu(stop_nmi_watchdog, NULL, 1);
336	BUG_ON(atomic_read(&nmi_active) != 0);
337	}
338	atomic_inc(&active_counters);
339	}
340	mutex_unlock(&pmc_grab_mutex);
341	}
342
343	void perf_counter_release_pmc(void)
344	{
345	if (atomic_dec_and_mutex_lock(&active_counters, &pmc_grab_mutex)) {
346	if (atomic_read(&nmi_active) == 0)
347	on_each_cpu(start_nmi_watchdog, NULL, 1);
348	mutex_unlock(&pmc_grab_mutex);
349	}
350	}
351
352	static void hw_perf_counter_destroy(struct perf_counter *counter)
353	{
354	perf_counter_release_pmc();
355	}
356
357	static int __hw_perf_counter_init(struct perf_counter *counter)
358	{
359	struct perf_counter_attr *attr = &counter->attr;
360	struct hw_perf_counter *hwc = &counter->hw;
361	const struct perf_event_map *pmap;
362	u64 enc;
363
364	if (atomic_read(&nmi_active) < 0)
365	return -ENODEV;
366
367	if (attr->type != PERF_TYPE_HARDWARE)
368	return -EOPNOTSUPP;
369
370	if (attr->config >= sparc_pmu->max_events)
371	return -EINVAL;
372
373	perf_counter_grab_pmc();
374	counter->destroy = hw_perf_counter_destroy;
375
376	/* We save the enable bits in the config_base. So to
377	* turn off sampling just write 'config', and to enable
378	* things write 'config \| config_base'.
379	*/
496c07e3	380	hwc->config_base = sparc_pmu->irq_bit;
59abbd1e DM	381	if (!attr->exclude_user)
	382	hwc->config_base \|= PCR_UTRACE;
	383	if (!attr->exclude_kernel)
	384	hwc->config_base \|= PCR_STRACE;
91b9286d DM	385	if (!attr->exclude_hv)
91b9286d DM	386	hwc->config_base \|= sparc_pmu->hv_bit;
59abbd1e DM	387
	388	if (!hwc->sample_period) {
	389	hwc->sample_period = MAX_PERIOD;
	390	hwc->last_period = hwc->sample_period;
	391	atomic64_set(&hwc->period_left, hwc->sample_period);
	392	}
	393
	394	pmap = sparc_pmu->event_map(attr->config);
	395
	396	enc = pmap->encoding;
	397	if (pmap->pic_mask & PIC_UPPER) {
	398	hwc->idx = PIC_UPPER_INDEX;
	399	enc <<= sparc_pmu->upper_shift;
	400	} else {
	401	hwc->idx = PIC_LOWER_INDEX;
	402	enc <<= sparc_pmu->lower_shift;
	403	}
	404
	405	hwc->config \|= enc;
	406	return 0;
	407	}
	408
	409	static const struct pmu pmu = {
	410	.enable = sparc_pmu_enable,
	411	.disable = sparc_pmu_disable,
	412	.read = sparc_pmu_read,
	413	.unthrottle = sparc_pmu_unthrottle,
	414	};
	415
	416	const struct pmu hw_perf_counter_init(struct perf_counter counter)
	417	{
	418	int err = __hw_perf_counter_init(counter);
	419
	420	if (err)
	421	return ERR_PTR(err);
	422	return &pmu;
	423	}
	424
	425	void perf_counter_print_debug(void)
	426	{
	427	unsigned long flags;
	428	u64 pcr, pic;
	429	int cpu;
	430
	431	if (!sparc_pmu)
	432	return;
	433
	434	local_irq_save(flags);
	435
	436	cpu = smp_processor_id();
	437
	438	pcr = pcr_ops->read();
	439	read_pic(pic);
	440
	441	pr_info("\n");
	442	pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
	443	cpu, pcr, pic);
	444
	445	local_irq_restore(flags);
	446	}
	447
	448	static int __kprobes perf_counter_nmi_handler(struct notifier_block *self,
	449	unsigned long cmd, void *__args)
	450	{
451	struct die_args *args = __args;
452	struct perf_sample_data data;
453	struct cpu_hw_counters *cpuc;
454	struct pt_regs *regs;
455	int idx;
456
457	if (!atomic_read(&active_counters))
458	return NOTIFY_DONE;
459
460	switch (cmd) {
461	case DIE_NMI:
462	break;
463
464	default:
465	return NOTIFY_DONE;
466	}
467
468	regs = args->regs;
469
470	data.regs = regs;
471	data.addr = 0;
472
473	cpuc = &__get_cpu_var(cpu_hw_counters);
474	for (idx = 0; idx < MAX_HWCOUNTERS; idx++) {
475	struct perf_counter *counter = cpuc->counters[idx];
476	struct hw_perf_counter *hwc;
477	u64 val;
478
479	if (!test_bit(idx, cpuc->active_mask))
480	continue;
481	hwc = &counter->hw;
482	val = sparc_perf_counter_update(counter, hwc, idx);
483	if (val & (1ULL << 31))
484	continue;
485
486	data.period = counter->hw.last_period;
487	if (!sparc_perf_counter_set_period(counter, hwc, idx))
488	continue;
489
490	if (perf_counter_overflow(counter, 1, &data))
491	sparc_pmu_disable_counter(hwc, idx);
492	}
493
494	return NOTIFY_STOP;
495	}
496
497	static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
498	.notifier_call = perf_counter_nmi_handler,
499	};
500
501	static bool __init supported_pmu(void)
502	{
503	if (!strcmp(sparc_pmu_type, "ultra3i")) {
504	sparc_pmu = &ultra3i_pmu;
505	return true;
506	}
507	return false;
508	}
509
510	void __init init_hw_perf_counters(void)
511	{
512	pr_info("Performance counters: ");
513
514	if (!supported_pmu()) {
515	pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
516	return;
517	}
518
519	pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
520
521	/* All sparc64 PMUs currently have 2 counters. But this simple
522	* driver only supports one active counter at a time.
523	*/
524	perf_max_counters = 1;
525
526	register_die_notifier(&perf_counter_nmi_notifier);
527	}