[linux-2.6-block.git] / arch / sparc / kernel / perf_event.c

/* Performance event support for sparc64.
 *
 * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
 *
 * This code is based almost entirely upon the x86 perf event
 * code, which is:
 *
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2009 Jaswinder Singh Rajput
 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
 */

#include <linux/perf_event.h>
#include <linux/kprobes.h>
#include <linux/kernel.h>
#include <linux/kdebug.h>
#include <linux/mutex.h>

#include <asm/cpudata.h>
#include <asm/atomic.h>
#include <asm/nmi.h>
#include <asm/pcr.h>

/* Sparc64 chips have two performance counters, 32-bits each, with
 * overflow interrupts generated on transition from 0xffffffff to 0.
 * The counters are accessed in one go using a 64-bit register.
 *
 * Both counters are controlled using a single control register.  The
 * only way to stop all sampling is to clear all of the context (user,
 * supervisor, hypervisor) sampling enable bits.  But these bits apply
 * to both counters, thus the two counters can't be enabled/disabled
 * individually.
 *
 * The control register has two event fields, one for each of the two
 * counters.  It's thus nearly impossible to have one counter going
 * while keeping the other one stopped.  Therefore it is possible to
 * get overflow interrupts for counters not currently "in use" and
 * that condition must be checked in the overflow interrupt handler.
 *
 * So we use a hack, in that we program inactive counters with the
 * "sw_count0" and "sw_count1" events.  These count how many times
 * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
 * unusual way to encode a NOP and therefore will not trigger in
 * normal code.
 */

#define MAX_HWEVENTS			2
#define MAX_PERIOD			((1UL << 32) - 1)

#define PIC_UPPER_INDEX			0
#define PIC_LOWER_INDEX			1

struct cpu_hw_events {
	struct perf_event	*events[MAX_HWEVENTS];
	unsigned long		used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
	unsigned long		active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
	int enabled;
};
DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };

struct perf_event_map {
	u16	encoding;
	u8	pic_mask;
#define PIC_NONE	0x00
#define PIC_UPPER	0x01
#define PIC_LOWER	0x02
};

#define C(x) PERF_COUNT_HW_CACHE_##x

#define CACHE_OP_UNSUPPORTED	0xfffe
#define CACHE_OP_NONSENSE	0xffff

typedef struct perf_event_map cache_map_t
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX];

struct sparc_pmu {
	const struct perf_event_map	*(*event_map)(int);
	const cache_map_t		*cache_map;
	int				max_events;
	int				upper_shift;
	int				lower_shift;
	int				event_mask;
	int				hv_bit;
	int				irq_bit;
	int				upper_nop;
	int				lower_nop;
};

static const struct perf_event_map ultra3_perfmon_event_map[] = {
	[PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
};

static const struct perf_event_map *ultra3_event_map(int event_id)
{
	return &ultra3_perfmon_event_map[event_id];
}

static const cache_map_t ultra3_cache_map = {
[C(L1D)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
		[C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
		[C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
		[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
	},
},
[C(L1I)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
		[C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
		[ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
	},
	[ C(OP_PREFETCH) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
},
[C(LL)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
		[C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
	},
	[C(OP_WRITE)] = {
		[C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
		[C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
	},
	[C(OP_PREFETCH)] = {
		[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
		[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
	},
},
[C(DTLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
		[C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
	[ C(OP_PREFETCH) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
},
[C(ITLB)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
		[C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
	[ C(OP_PREFETCH) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
},
[C(BPU)] = {
	[C(OP_READ)] = {
		[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
		[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
	[ C(OP_PREFETCH) ] = {
		[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
		[ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
	},
},
};

static const struct sparc_pmu ultra3_pmu = {
	.event_map	= ultra3_event_map,
	.cache_map	= &ultra3_cache_map,
	.max_events	= ARRAY_SIZE(ultra3_perfmon_event_map),
	.upper_shift	= 11,
	.lower_shift	= 4,
	.event_mask	= 0x3f,
	.upper_nop	= 0x1c,
	.lower_nop	= 0x14,
};

static const struct perf_event_map niagara2_perfmon_event_map[] = {
	[PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
};

static const struct perf_event_map *niagara2_event_map(int event_id)
{
	return &niagara2_perfmon_event_map[event_id];
}

static const struct sparc_pmu niagara2_pmu = {
	.event_map	= niagara2_event_map,
	.max_events	= ARRAY_SIZE(niagara2_perfmon_event_map),
	.upper_shift	= 19,
	.lower_shift	= 6,
	.event_mask	= 0xfff,
	.hv_bit		= 0x8,
	.irq_bit	= 0x03,
	.upper_nop	= 0x220,
	.lower_nop	= 0x220,
};

static const struct sparc_pmu *sparc_pmu __read_mostly;

static u64 event_encoding(u64 event_id, int idx)
{
	if (idx == PIC_UPPER_INDEX)
		event_id <<= sparc_pmu->upper_shift;
	else
		event_id <<= sparc_pmu->lower_shift;
	return event_id;
}

static u64 mask_for_index(int idx)
{
	return event_encoding(sparc_pmu->event_mask, idx);
}

static u64 nop_for_index(int idx)
{
	return event_encoding(idx == PIC_UPPER_INDEX ?
			      sparc_pmu->upper_nop :
			      sparc_pmu->lower_nop, idx);
}

static inline void sparc_pmu_enable_event(struct hw_perf_event *hwc,
					    int idx)
{
	u64 val, mask = mask_for_index(idx);

	val = pcr_ops->read();
	pcr_ops->write((val & ~mask) | hwc->config);
}

static inline void sparc_pmu_disable_event(struct hw_perf_event *hwc,
					     int idx)
{
	u64 mask = mask_for_index(idx);
	u64 nop = nop_for_index(idx);
	u64 val = pcr_ops->read();

	pcr_ops->write((val & ~mask) | nop);
}

void hw_perf_enable(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	u64 val;
	int i;

	if (cpuc->enabled)
		return;

	cpuc->enabled = 1;
	barrier();

	val = pcr_ops->read();

	for (i = 0; i < MAX_HWEVENTS; i++) {
		struct perf_event *cp = cpuc->events[i];
		struct hw_perf_event *hwc;

		if (!cp)
			continue;
		hwc = &cp->hw;
		val |= hwc->config_base;
	}

	pcr_ops->write(val);
}

void hw_perf_disable(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	u64 val;

	if (!cpuc->enabled)
		return;

	cpuc->enabled = 0;

	val = pcr_ops->read();
	val &= ~(PCR_UTRACE | PCR_STRACE |
		 sparc_pmu->hv_bit | sparc_pmu->irq_bit);
	pcr_ops->write(val);
}

static u32 read_pmc(int idx)
{
	u64 val;

	read_pic(val);
	if (idx == PIC_UPPER_INDEX)
		val >>= 32;

	return val & 0xffffffff;
}

static void write_pmc(int idx, u64 val)
{
	u64 shift, mask, pic;

	shift = 0;
	if (idx == PIC_UPPER_INDEX)
		shift = 32;

	mask = ((u64) 0xffffffff) << shift;
	val <<= shift;

	read_pic(pic);
	pic &= ~mask;
	pic |= val;
	write_pic(pic);
}

static int sparc_perf_event_set_period(struct perf_event *event,
					 struct hw_perf_event *hwc, int idx)
{
	s64 left = atomic64_read(&hwc->period_left);
	s64 period = hwc->sample_period;
	int ret = 0;

	if (unlikely(left <= -period)) {
		left = period;
		atomic64_set(&hwc->period_left, left);
		hwc->last_period = period;
		ret = 1;
	}

	if (unlikely(left <= 0)) {
		left += period;
		atomic64_set(&hwc->period_left, left);
		hwc->last_period = period;
		ret = 1;
	}
	if (left > MAX_PERIOD)
		left = MAX_PERIOD;

	atomic64_set(&hwc->prev_count, (u64)-left);

	write_pmc(idx, (u64)(-left) & 0xffffffff);

	perf_event_update_userpage(event);

	return ret;
}

static int sparc_pmu_enable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct hw_perf_event *hwc = &event->hw;
	int idx = hwc->idx;

	if (test_and_set_bit(idx, cpuc->used_mask))
		return -EAGAIN;

	sparc_pmu_disable_event(hwc, idx);

	cpuc->events[idx] = event;
	set_bit(idx, cpuc->active_mask);

	sparc_perf_event_set_period(event, hwc, idx);
	sparc_pmu_enable_event(hwc, idx);
	perf_event_update_userpage(event);
	return 0;
}

static u64 sparc_perf_event_update(struct perf_event *event,
				     struct hw_perf_event *hwc, int idx)
{
	int shift = 64 - 32;
	u64 prev_raw_count, new_raw_count;
	s64 delta;

again:
	prev_raw_count = atomic64_read(&hwc->prev_count);
	new_raw_count = read_pmc(idx);

	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
			     new_raw_count) != prev_raw_count)
		goto again;

	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	delta >>= shift;

	atomic64_add(delta, &event->count);
	atomic64_sub(delta, &hwc->period_left);

	return new_raw_count;
}

static void sparc_pmu_disable(struct perf_event *event)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct hw_perf_event *hwc = &event->hw;
	int idx = hwc->idx;

	clear_bit(idx, cpuc->active_mask);
	sparc_pmu_disable_event(hwc, idx);

	barrier();

	sparc_perf_event_update(event, hwc, idx);
	cpuc->events[idx] = NULL;
	clear_bit(idx, cpuc->used_mask);

	perf_event_update_userpage(event);
}

static void sparc_pmu_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	sparc_perf_event_update(event, hwc, hwc->idx);
}

static void sparc_pmu_unthrottle(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	sparc_pmu_enable_event(hwc, hwc->idx);
}

static atomic_t active_events = ATOMIC_INIT(0);
static DEFINE_MUTEX(pmc_grab_mutex);

void perf_event_grab_pmc(void)
{
	if (atomic_inc_not_zero(&active_events))
		return;

	mutex_lock(&pmc_grab_mutex);
	if (atomic_read(&active_events) == 0) {
		if (atomic_read(&nmi_active) > 0) {
			on_each_cpu(stop_nmi_watchdog, NULL, 1);
			BUG_ON(atomic_read(&nmi_active) != 0);
		}
		atomic_inc(&active_events);
	}
	mutex_unlock(&pmc_grab_mutex);
}

void perf_event_release_pmc(void)
{
	if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
		if (atomic_read(&nmi_active) == 0)
			on_each_cpu(start_nmi_watchdog, NULL, 1);
		mutex_unlock(&pmc_grab_mutex);
	}
}

static const struct perf_event_map *sparc_map_cache_event(u64 config)
{
	unsigned int cache_type, cache_op, cache_result;
	const struct perf_event_map *pmap;

	if (!sparc_pmu->cache_map)
		return ERR_PTR(-ENOENT);

	cache_type = (config >>  0) & 0xff;
	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
		return ERR_PTR(-EINVAL);

	cache_op = (config >>  8) & 0xff;
	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
		return ERR_PTR(-EINVAL);

	cache_result = (config >> 16) & 0xff;
	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
		return ERR_PTR(-EINVAL);

	pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);

	if (pmap->encoding == CACHE_OP_UNSUPPORTED)
		return ERR_PTR(-ENOENT);

	if (pmap->encoding == CACHE_OP_NONSENSE)
		return ERR_PTR(-EINVAL);

	return pmap;
}

static void hw_perf_event_destroy(struct perf_event *event)
{
	perf_event_release_pmc();
}

static int __hw_perf_event_init(struct perf_event *event)
{
	struct perf_event_attr *attr = &event->attr;
	struct hw_perf_event *hwc = &event->hw;
	const struct perf_event_map *pmap;
	u64 enc;

	if (atomic_read(&nmi_active) < 0)
		return -ENODEV;

	if (attr->type == PERF_TYPE_HARDWARE) {
		if (attr->config >= sparc_pmu->max_events)
			return -EINVAL;
		pmap = sparc_pmu->event_map(attr->config);
	} else if (attr->type == PERF_TYPE_HW_CACHE) {
		pmap = sparc_map_cache_event(attr->config);
		if (IS_ERR(pmap))
			return PTR_ERR(pmap);
	} else
		return -EOPNOTSUPP;

	perf_event_grab_pmc();
	event->destroy = hw_perf_event_destroy;

	/* We save the enable bits in the config_base.  So to
	 * turn off sampling just write 'config', and to enable
	 * things write 'config | config_base'.
	 */
	hwc->config_base = sparc_pmu->irq_bit;
	if (!attr->exclude_user)
		hwc->config_base |= PCR_UTRACE;
	if (!attr->exclude_kernel)
		hwc->config_base |= PCR_STRACE;
	if (!attr->exclude_hv)
		hwc->config_base |= sparc_pmu->hv_bit;

	if (!hwc->sample_period) {
		hwc->sample_period = MAX_PERIOD;
		hwc->last_period = hwc->sample_period;
		atomic64_set(&hwc->period_left, hwc->sample_period);
	}

	enc = pmap->encoding;
	if (pmap->pic_mask & PIC_UPPER) {
		hwc->idx = PIC_UPPER_INDEX;
		enc <<= sparc_pmu->upper_shift;
	} else {
		hwc->idx = PIC_LOWER_INDEX;
		enc <<= sparc_pmu->lower_shift;
	}

	hwc->config |= enc;
	return 0;
}

static const struct pmu pmu = {
	.enable		= sparc_pmu_enable,
	.disable	= sparc_pmu_disable,
	.read		= sparc_pmu_read,
	.unthrottle	= sparc_pmu_unthrottle,
};

const struct pmu *hw_perf_event_init(struct perf_event *event)
{
	int err = __hw_perf_event_init(event);

	if (err)
		return ERR_PTR(err);
	return &pmu;
}

void perf_event_print_debug(void)
{
	unsigned long flags;
	u64 pcr, pic;
	int cpu;

	if (!sparc_pmu)
		return;

	local_irq_save(flags);

	cpu = smp_processor_id();

	pcr = pcr_ops->read();
	read_pic(pic);

	pr_info("\n");
	pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
		cpu, pcr, pic);

	local_irq_restore(flags);
}

static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
					      unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
	struct perf_sample_data data;
	struct cpu_hw_events *cpuc;
	struct pt_regs *regs;
	int idx;

	if (!atomic_read(&active_events))
		return NOTIFY_DONE;

	switch (cmd) {
	case DIE_NMI:
		break;

	default:
		return NOTIFY_DONE;
	}

	regs = args->regs;

	data.addr = 0;

	cpuc = &__get_cpu_var(cpu_hw_events);
	for (idx = 0; idx < MAX_HWEVENTS; idx++) {
		struct perf_event *event = cpuc->events[idx];
		struct hw_perf_event *hwc;
		u64 val;

		if (!test_bit(idx, cpuc->active_mask))
			continue;
		hwc = &event->hw;
		val = sparc_perf_event_update(event, hwc, idx);
		if (val & (1ULL << 31))
			continue;

		data.period = event->hw.last_period;
		if (!sparc_perf_event_set_period(event, hwc, idx))
			continue;

		if (perf_event_overflow(event, 1, &data, regs))
			sparc_pmu_disable_event(hwc, idx);
	}

	return NOTIFY_STOP;
}

static __read_mostly struct notifier_block perf_event_nmi_notifier = {
	.notifier_call		= perf_event_nmi_handler,
};

static bool __init supported_pmu(void)
{
	if (!strcmp(sparc_pmu_type, "ultra3") ||
	    !strcmp(sparc_pmu_type, "ultra3+") ||
	    !strcmp(sparc_pmu_type, "ultra3i") ||
	    !strcmp(sparc_pmu_type, "ultra4+")) {
		sparc_pmu = &ultra3_pmu;
		return true;
	}
	if (!strcmp(sparc_pmu_type, "niagara2")) {
		sparc_pmu = &niagara2_pmu;
		return true;
	}
	return false;
}

void __init init_hw_perf_events(void)
{
	pr_info("Performance events: ");

	if (!supported_pmu()) {
		pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
		return;
	}

	pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);

	/* All sparc64 PMUs currently have 2 events.  But this simple
	 * driver only supports one active event at a time.
	 */
	perf_max_events = 1;

	register_die_notifier(&perf_event_nmi_notifier);
}
Commit	Line	Data
cdd6c482	1	/* Performance event support for sparc64.
59abbd1e DM	2	*
	3	* Copyright (C) 2009 David S. Miller <davem@davemloft.net>
	4	*
cdd6c482	5	* This code is based almost entirely upon the x86 perf event
59abbd1e DM	6	* code, which is:
	7	*
	8	* Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
	9	* Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
	10	* Copyright (C) 2009 Jaswinder Singh Rajput
	11	* Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
	12	* Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
	13	*/
	14
cdd6c482	15	#include <linux/perf_event.h>
59abbd1e DM	16	#include <linux/kprobes.h>
	17	#include <linux/kernel.h>
	18	#include <linux/kdebug.h>
	19	#include <linux/mutex.h>
	20
	21	#include <asm/cpudata.h>
	22	#include <asm/atomic.h>
	23	#include <asm/nmi.h>
	24	#include <asm/pcr.h>
	25
	26	/* Sparc64 chips have two performance counters, 32-bits each, with
	27	* overflow interrupts generated on transition from 0xffffffff to 0.
	28	* The counters are accessed in one go using a 64-bit register.
	29	*
	30	* Both counters are controlled using a single control register. The
	31	* only way to stop all sampling is to clear all of the context (user,
	32	* supervisor, hypervisor) sampling enable bits. But these bits apply
	33	* to both counters, thus the two counters can't be enabled/disabled
	34	* individually.
	35	*
	36	* The control register has two event fields, one for each of the two
	37	* counters. It's thus nearly impossible to have one counter going
	38	* while keeping the other one stopped. Therefore it is possible to
	39	* get overflow interrupts for counters not currently "in use" and
	40	* that condition must be checked in the overflow interrupt handler.
	41	*
	42	* So we use a hack, in that we program inactive counters with the
	43	* "sw_count0" and "sw_count1" events. These count how many times
	44	* the instruction "sethi %hi(0xfc000), %g0" is executed. It's an
	45	* unusual way to encode a NOP and therefore will not trigger in
	46	* normal code.
	47	*/
	48
cdd6c482	49	#define MAX_HWEVENTS 2
59abbd1e DM	50	#define MAX_PERIOD ((1UL << 32) - 1)
	51
	52	#define PIC_UPPER_INDEX 0
	53	#define PIC_LOWER_INDEX 1
	54
cdd6c482 IM	55	struct cpu_hw_events {
	56	struct perf_event *events[MAX_HWEVENTS];
	57	unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
	58	unsigned long active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
59abbd1e DM	59	int enabled;
59abbd1e DM	60	};
cdd6c482	61	DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
59abbd1e DM	62
	63	struct perf_event_map {
	64	u16 encoding;
	65	u8 pic_mask;
	66	#define PIC_NONE 0x00
	67	#define PIC_UPPER 0x01
	68	#define PIC_LOWER 0x02
	69	};
	70
2ce4da2e DM	71	#define C(x) PERF_COUNT_HW_CACHE_##x
	72
	73	#define CACHE_OP_UNSUPPORTED 0xfffe
	74	#define CACHE_OP_NONSENSE 0xffff
	75
	76	typedef struct perf_event_map cache_map_t
	77	[PERF_COUNT_HW_CACHE_MAX]
	78	[PERF_COUNT_HW_CACHE_OP_MAX]
	79	[PERF_COUNT_HW_CACHE_RESULT_MAX];
	80
59abbd1e DM	81	struct sparc_pmu {
59abbd1e DM	82	const struct perf_event_map (event_map)(int);
2ce4da2e	83	const cache_map_t *cache_map;
59abbd1e DM	84	int max_events;
	85	int upper_shift;
	86	int lower_shift;
	87	int event_mask;
91b9286d	88	int hv_bit;
496c07e3	89	int irq_bit;
660d1376 DM	90	int upper_nop;
660d1376 DM	91	int lower_nop;
59abbd1e DM	92	};
59abbd1e DM	93
28e8f9be	94	static const struct perf_event_map ultra3_perfmon_event_map[] = {
59abbd1e DM	95	[PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER \| PIC_LOWER },
	96	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER \| PIC_LOWER },
	97	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
	98	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
	99	};
	100
28e8f9be	101	static const struct perf_event_map *ultra3_event_map(int event_id)
59abbd1e	102	{
28e8f9be	103	return &ultra3_perfmon_event_map[event_id];
59abbd1e DM	104	}
59abbd1e DM	105
28e8f9be	106	static const cache_map_t ultra3_cache_map = {
2ce4da2e DM	107	[C(L1D)] = {
	108	[C(OP_READ)] = {
	109	[C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
	110	[C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
	111	},
	112	[C(OP_WRITE)] = {
	113	[C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
	114	[C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
	115	},
	116	[C(OP_PREFETCH)] = {
	117	[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
	118	[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
	119	},
	120	},
	121	[C(L1I)] = {
	122	[C(OP_READ)] = {
	123	[C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
	124	[C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
	125	},
	126	[ C(OP_WRITE) ] = {
	127	[ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
	128	[ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
	129	},
	130	[ C(OP_PREFETCH) ] = {
	131	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
	132	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
	133	},
	134	},
	135	[C(LL)] = {
	136	[C(OP_READ)] = {
	137	[C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
	138	[C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
	139	},
	140	[C(OP_WRITE)] = {
	141	[C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
	142	[C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
	143	},
	144	[C(OP_PREFETCH)] = {
	145	[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
	146	[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
	147	},
	148	},
	149	[C(DTLB)] = {
	150	[C(OP_READ)] = {
	151	[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
	152	[C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
	153	},
	154	[ C(OP_WRITE) ] = {
	155	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
	156	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
	157	},
	158	[ C(OP_PREFETCH) ] = {
	159	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
	160	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
	161	},
	162	},
	163	[C(ITLB)] = {
	164	[C(OP_READ)] = {
	165	[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
	166	[C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
	167	},
	168	[ C(OP_WRITE) ] = {
	169	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
	170	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
171	},
172	[ C(OP_PREFETCH) ] = {
173	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
174	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
175	},
176	},
177	[C(BPU)] = {
178	[C(OP_READ)] = {
179	[C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
180	[C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
181	},
182	[ C(OP_WRITE) ] = {
183	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
184	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
185	},
186	[ C(OP_PREFETCH) ] = {
187	[ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
188	[ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
189	},
190	},
191	};
192
28e8f9be DM	193	static const struct sparc_pmu ultra3_pmu = {
	194	.event_map = ultra3_event_map,
	195	.cache_map = &ultra3_cache_map,
	196	.max_events = ARRAY_SIZE(ultra3_perfmon_event_map),
59abbd1e DM	197	.upper_shift = 11,
	198	.lower_shift = 4,
	199	.event_mask = 0x3f,
660d1376 DM	200	.upper_nop = 0x1c,
660d1376 DM	201	.lower_nop = 0x14,
59abbd1e DM	202	};
59abbd1e DM	203
b73d8847 DM	204	static const struct perf_event_map niagara2_perfmon_event_map[] = {
	205	[PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER \| PIC_LOWER },
	206	[PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER \| PIC_LOWER },
	207	[PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER \| PIC_LOWER },
	208	[PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER \| PIC_LOWER },
	209	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER \| PIC_LOWER },
	210	[PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER \| PIC_LOWER },
	211	};
	212
cdd6c482	213	static const struct perf_event_map *niagara2_event_map(int event_id)
b73d8847	214	{
cdd6c482	215	return &niagara2_perfmon_event_map[event_id];
b73d8847 DM	216	}
	217
	218	static const struct sparc_pmu niagara2_pmu = {
	219	.event_map = niagara2_event_map,
	220	.max_events = ARRAY_SIZE(niagara2_perfmon_event_map),
	221	.upper_shift = 19,
	222	.lower_shift = 6,
	223	.event_mask = 0xfff,
	224	.hv_bit = 0x8,
	225	.irq_bit = 0x03,
	226	.upper_nop = 0x220,
	227	.lower_nop = 0x220,
	228	};
	229
59abbd1e DM	230	static const struct sparc_pmu *sparc_pmu __read_mostly;
59abbd1e DM	231
cdd6c482	232	static u64 event_encoding(u64 event_id, int idx)
59abbd1e DM	233	{
59abbd1e DM	234	if (idx == PIC_UPPER_INDEX)
cdd6c482	235	event_id <<= sparc_pmu->upper_shift;
59abbd1e	236	else
cdd6c482 IM	237	event_id <<= sparc_pmu->lower_shift;
cdd6c482 IM	238	return event_id;
59abbd1e DM	239	}
	240
	241	static u64 mask_for_index(int idx)
	242	{
	243	return event_encoding(sparc_pmu->event_mask, idx);
	244	}
	245
	246	static u64 nop_for_index(int idx)
	247	{
	248	return event_encoding(idx == PIC_UPPER_INDEX ?
660d1376 DM	249	sparc_pmu->upper_nop :
660d1376 DM	250	sparc_pmu->lower_nop, idx);
59abbd1e DM	251	}
59abbd1e DM	252
cdd6c482	253	static inline void sparc_pmu_enable_event(struct hw_perf_event *hwc,
59abbd1e DM	254	int idx)
	255	{
	256	u64 val, mask = mask_for_index(idx);
	257
	258	val = pcr_ops->read();
	259	pcr_ops->write((val & ~mask) \| hwc->config);
	260	}
	261
cdd6c482	262	static inline void sparc_pmu_disable_event(struct hw_perf_event *hwc,
59abbd1e DM	263	int idx)
	264	{
	265	u64 mask = mask_for_index(idx);
	266	u64 nop = nop_for_index(idx);
	267	u64 val = pcr_ops->read();
	268
	269	pcr_ops->write((val & ~mask) \| nop);
	270	}
	271
	272	void hw_perf_enable(void)
	273	{
cdd6c482	274	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
59abbd1e DM	275	u64 val;
	276	int i;
	277
	278	if (cpuc->enabled)
	279	return;
	280
	281	cpuc->enabled = 1;
	282	barrier();
	283
	284	val = pcr_ops->read();
	285
cdd6c482 IM	286	for (i = 0; i < MAX_HWEVENTS; i++) {
	287	struct perf_event *cp = cpuc->events[i];
	288	struct hw_perf_event *hwc;
59abbd1e DM	289
	290	if (!cp)
	291	continue;
	292	hwc = &cp->hw;
	293	val \|= hwc->config_base;
	294	}
	295
	296	pcr_ops->write(val);
	297	}
	298
	299	void hw_perf_disable(void)
	300	{
cdd6c482	301	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
59abbd1e DM	302	u64 val;
	303
	304	if (!cpuc->enabled)
	305	return;
	306
	307	cpuc->enabled = 0;
	308
	309	val = pcr_ops->read();
496c07e3 DM	310	val &= ~(PCR_UTRACE \| PCR_STRACE \|
496c07e3 DM	311	sparc_pmu->hv_bit \| sparc_pmu->irq_bit);
59abbd1e DM	312	pcr_ops->write(val);
	313	}
	314
	315	static u32 read_pmc(int idx)
	316	{
	317	u64 val;
	318
	319	read_pic(val);
	320	if (idx == PIC_UPPER_INDEX)
	321	val >>= 32;
	322
	323	return val & 0xffffffff;
	324	}
	325
	326	static void write_pmc(int idx, u64 val)
	327	{
	328	u64 shift, mask, pic;
	329
	330	shift = 0;
	331	if (idx == PIC_UPPER_INDEX)
	332	shift = 32;
	333
	334	mask = ((u64) 0xffffffff) << shift;
	335	val <<= shift;
	336
	337	read_pic(pic);
	338	pic &= ~mask;
	339	pic \|= val;
	340	write_pic(pic);
	341	}
	342
cdd6c482 IM	343	static int sparc_perf_event_set_period(struct perf_event *event,
cdd6c482 IM	344	struct hw_perf_event *hwc, int idx)
59abbd1e DM	345	{
	346	s64 left = atomic64_read(&hwc->period_left);
	347	s64 period = hwc->sample_period;
	348	int ret = 0;
	349
	350	if (unlikely(left <= -period)) {
	351	left = period;
	352	atomic64_set(&hwc->period_left, left);
	353	hwc->last_period = period;
	354	ret = 1;
	355	}
	356
	357	if (unlikely(left <= 0)) {
	358	left += period;
	359	atomic64_set(&hwc->period_left, left);
	360	hwc->last_period = period;
	361	ret = 1;
	362	}
	363	if (left > MAX_PERIOD)
	364	left = MAX_PERIOD;
	365
	366	atomic64_set(&hwc->prev_count, (u64)-left);
	367
	368	write_pmc(idx, (u64)(-left) & 0xffffffff);
	369
cdd6c482	370	perf_event_update_userpage(event);
59abbd1e DM	371
	372	return ret;
	373	}
	374
cdd6c482	375	static int sparc_pmu_enable(struct perf_event *event)
59abbd1e	376	{
cdd6c482 IM	377	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
cdd6c482 IM	378	struct hw_perf_event *hwc = &event->hw;
59abbd1e DM	379	int idx = hwc->idx;
	380
	381	if (test_and_set_bit(idx, cpuc->used_mask))
	382	return -EAGAIN;
	383
cdd6c482	384	sparc_pmu_disable_event(hwc, idx);
59abbd1e	385
cdd6c482	386	cpuc->events[idx] = event;
59abbd1e DM	387	set_bit(idx, cpuc->active_mask);
59abbd1e DM	388
cdd6c482 IM	389	sparc_perf_event_set_period(event, hwc, idx);
	390	sparc_pmu_enable_event(hwc, idx);
	391	perf_event_update_userpage(event);
59abbd1e DM	392	return 0;
	393	}
	394
cdd6c482 IM	395	static u64 sparc_perf_event_update(struct perf_event *event,
cdd6c482 IM	396	struct hw_perf_event *hwc, int idx)
59abbd1e DM	397	{
	398	int shift = 64 - 32;
	399	u64 prev_raw_count, new_raw_count;
	400	s64 delta;
	401
	402	again:
	403	prev_raw_count = atomic64_read(&hwc->prev_count);
	404	new_raw_count = read_pmc(idx);
	405
	406	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
	407	new_raw_count) != prev_raw_count)
	408	goto again;
	409
	410	delta = (new_raw_count << shift) - (prev_raw_count << shift);
	411	delta >>= shift;
	412
cdd6c482	413	atomic64_add(delta, &event->count);
59abbd1e DM	414	atomic64_sub(delta, &hwc->period_left);
	415
	416	return new_raw_count;
	417	}
	418
cdd6c482	419	static void sparc_pmu_disable(struct perf_event *event)
59abbd1e	420	{
cdd6c482 IM	421	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
cdd6c482 IM	422	struct hw_perf_event *hwc = &event->hw;
59abbd1e DM	423	int idx = hwc->idx;
	424
	425	clear_bit(idx, cpuc->active_mask);
cdd6c482	426	sparc_pmu_disable_event(hwc, idx);
59abbd1e DM	427
	428	barrier();
	429
cdd6c482 IM	430	sparc_perf_event_update(event, hwc, idx);
cdd6c482 IM	431	cpuc->events[idx] = NULL;
59abbd1e DM	432	clear_bit(idx, cpuc->used_mask);
59abbd1e DM	433
cdd6c482	434	perf_event_update_userpage(event);
59abbd1e DM	435	}
59abbd1e DM	436
cdd6c482	437	static void sparc_pmu_read(struct perf_event *event)
59abbd1e	438	{
cdd6c482 IM	439	struct hw_perf_event *hwc = &event->hw;
cdd6c482 IM	440	sparc_perf_event_update(event, hwc, hwc->idx);
59abbd1e DM	441	}
59abbd1e DM	442
cdd6c482	443	static void sparc_pmu_unthrottle(struct perf_event *event)
59abbd1e	444	{
cdd6c482 IM	445	struct hw_perf_event *hwc = &event->hw;
cdd6c482 IM	446	sparc_pmu_enable_event(hwc, hwc->idx);
59abbd1e DM	447	}
59abbd1e DM	448
cdd6c482	449	static atomic_t active_events = ATOMIC_INIT(0);
59abbd1e DM	450	static DEFINE_MUTEX(pmc_grab_mutex);
59abbd1e DM	451
cdd6c482	452	void perf_event_grab_pmc(void)
59abbd1e	453	{
cdd6c482	454	if (atomic_inc_not_zero(&active_events))
59abbd1e DM	455	return;
	456
	457	mutex_lock(&pmc_grab_mutex);
cdd6c482	458	if (atomic_read(&active_events) == 0) {
59abbd1e DM	459	if (atomic_read(&nmi_active) > 0) {
	460	on_each_cpu(stop_nmi_watchdog, NULL, 1);
	461	BUG_ON(atomic_read(&nmi_active) != 0);
	462	}
cdd6c482	463	atomic_inc(&active_events);
59abbd1e DM	464	}
	465	mutex_unlock(&pmc_grab_mutex);
	466	}
	467
cdd6c482	468	void perf_event_release_pmc(void)
59abbd1e	469	{
cdd6c482	470	if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
59abbd1e DM	471	if (atomic_read(&nmi_active) == 0)
	472	on_each_cpu(start_nmi_watchdog, NULL, 1);
	473	mutex_unlock(&pmc_grab_mutex);
	474	}
	475	}
	476
2ce4da2e DM	477	static const struct perf_event_map *sparc_map_cache_event(u64 config)
	478	{
	479	unsigned int cache_type, cache_op, cache_result;
	480	const struct perf_event_map *pmap;
	481
	482	if (!sparc_pmu->cache_map)
	483	return ERR_PTR(-ENOENT);
	484
	485	cache_type = (config >> 0) & 0xff;
	486	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
	487	return ERR_PTR(-EINVAL);
	488
	489	cache_op = (config >> 8) & 0xff;
	490	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
	491	return ERR_PTR(-EINVAL);
	492
	493	cache_result = (config >> 16) & 0xff;
	494	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
	495	return ERR_PTR(-EINVAL);
	496
	497	pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
	498
	499	if (pmap->encoding == CACHE_OP_UNSUPPORTED)
	500	return ERR_PTR(-ENOENT);
	501
	502	if (pmap->encoding == CACHE_OP_NONSENSE)
	503	return ERR_PTR(-EINVAL);
	504
	505	return pmap;
	506	}
	507
cdd6c482	508	static void hw_perf_event_destroy(struct perf_event *event)
59abbd1e	509	{
cdd6c482	510	perf_event_release_pmc();
59abbd1e DM	511	}
59abbd1e DM	512
cdd6c482	513	static int __hw_perf_event_init(struct perf_event *event)
59abbd1e	514	{
cdd6c482 IM	515	struct perf_event_attr *attr = &event->attr;
cdd6c482 IM	516	struct hw_perf_event *hwc = &event->hw;
59abbd1e DM	517	const struct perf_event_map *pmap;
	518	u64 enc;
	519
	520	if (atomic_read(&nmi_active) < 0)
	521	return -ENODEV;
	522
2ce4da2e DM	523	if (attr->type == PERF_TYPE_HARDWARE) {
	524	if (attr->config >= sparc_pmu->max_events)
	525	return -EINVAL;
	526	pmap = sparc_pmu->event_map(attr->config);
	527	} else if (attr->type == PERF_TYPE_HW_CACHE) {
	528	pmap = sparc_map_cache_event(attr->config);
	529	if (IS_ERR(pmap))
	530	return PTR_ERR(pmap);
	531	} else
59abbd1e DM	532	return -EOPNOTSUPP;
59abbd1e DM	533
cdd6c482 IM	534	perf_event_grab_pmc();
cdd6c482 IM	535	event->destroy = hw_perf_event_destroy;
59abbd1e DM	536
	537	/* We save the enable bits in the config_base. So to
	538	* turn off sampling just write 'config', and to enable
	539	* things write 'config \| config_base'.
	540	*/
496c07e3	541	hwc->config_base = sparc_pmu->irq_bit;
59abbd1e DM	542	if (!attr->exclude_user)
	543	hwc->config_base \|= PCR_UTRACE;
	544	if (!attr->exclude_kernel)
	545	hwc->config_base \|= PCR_STRACE;
91b9286d DM	546	if (!attr->exclude_hv)
91b9286d DM	547	hwc->config_base \|= sparc_pmu->hv_bit;
59abbd1e DM	548
	549	if (!hwc->sample_period) {
	550	hwc->sample_period = MAX_PERIOD;
	551	hwc->last_period = hwc->sample_period;
	552	atomic64_set(&hwc->period_left, hwc->sample_period);
	553	}
	554
59abbd1e DM	555	enc = pmap->encoding;
	556	if (pmap->pic_mask & PIC_UPPER) {
	557	hwc->idx = PIC_UPPER_INDEX;
	558	enc <<= sparc_pmu->upper_shift;
	559	} else {
	560	hwc->idx = PIC_LOWER_INDEX;
	561	enc <<= sparc_pmu->lower_shift;
	562	}
	563
	564	hwc->config \|= enc;
	565	return 0;
	566	}
	567
	568	static const struct pmu pmu = {
	569	.enable = sparc_pmu_enable,
	570	.disable = sparc_pmu_disable,
	571	.read = sparc_pmu_read,
	572	.unthrottle = sparc_pmu_unthrottle,
	573	};
	574
cdd6c482	575	const struct pmu hw_perf_event_init(struct perf_event event)
59abbd1e	576	{
cdd6c482	577	int err = __hw_perf_event_init(event);
59abbd1e DM	578
	579	if (err)
	580	return ERR_PTR(err);
	581	return &pmu;
	582	}
	583
cdd6c482	584	void perf_event_print_debug(void)
59abbd1e DM	585	{
	586	unsigned long flags;
	587	u64 pcr, pic;
	588	int cpu;
	589
	590	if (!sparc_pmu)
	591	return;
	592
	593	local_irq_save(flags);
	594
	595	cpu = smp_processor_id();
	596
	597	pcr = pcr_ops->read();
	598	read_pic(pic);
	599
	600	pr_info("\n");
	601	pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
	602	cpu, pcr, pic);
	603
	604	local_irq_restore(flags);
	605	}
	606
cdd6c482	607	static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
59abbd1e DM	608	unsigned long cmd, void *__args)
	609	{
	610	struct die_args *args = __args;
	611	struct perf_sample_data data;
cdd6c482	612	struct cpu_hw_events *cpuc;
59abbd1e DM	613	struct pt_regs *regs;
	614	int idx;
	615
cdd6c482	616	if (!atomic_read(&active_events))
59abbd1e DM	617	return NOTIFY_DONE;
	618
	619	switch (cmd) {
	620	case DIE_NMI:
	621	break;
	622
	623	default:
	624	return NOTIFY_DONE;
	625	}
	626
	627	regs = args->regs;
	628
59abbd1e DM	629	data.addr = 0;
59abbd1e DM	630
cdd6c482 IM	631	cpuc = &__get_cpu_var(cpu_hw_events);
	632	for (idx = 0; idx < MAX_HWEVENTS; idx++) {
	633	struct perf_event *event = cpuc->events[idx];
	634	struct hw_perf_event *hwc;
59abbd1e DM	635	u64 val;
	636
	637	if (!test_bit(idx, cpuc->active_mask))
	638	continue;
cdd6c482 IM	639	hwc = &event->hw;
cdd6c482 IM	640	val = sparc_perf_event_update(event, hwc, idx);
59abbd1e DM	641	if (val & (1ULL << 31))
	642	continue;
	643
cdd6c482 IM	644	data.period = event->hw.last_period;
cdd6c482 IM	645	if (!sparc_perf_event_set_period(event, hwc, idx))
59abbd1e DM	646	continue;
59abbd1e DM	647
cdd6c482 IM	648	if (perf_event_overflow(event, 1, &data, regs))
cdd6c482 IM	649	sparc_pmu_disable_event(hwc, idx);
59abbd1e DM	650	}
	651
	652	return NOTIFY_STOP;
	653	}
	654
cdd6c482 IM	655	static __read_mostly struct notifier_block perf_event_nmi_notifier = {
cdd6c482 IM	656	.notifier_call = perf_event_nmi_handler,
59abbd1e DM	657	};
	658
	659	static bool __init supported_pmu(void)
	660	{
28e8f9be DM	661	if (!strcmp(sparc_pmu_type, "ultra3") \|\|
	662	!strcmp(sparc_pmu_type, "ultra3+") \|\|
	663	!strcmp(sparc_pmu_type, "ultra3i") \|\|
	664	!strcmp(sparc_pmu_type, "ultra4+")) {
	665	sparc_pmu = &ultra3_pmu;
59abbd1e DM	666	return true;
59abbd1e DM	667	}
b73d8847 DM	668	if (!strcmp(sparc_pmu_type, "niagara2")) {
	669	sparc_pmu = &niagara2_pmu;
	670	return true;
	671	}
59abbd1e DM	672	return false;
	673	}
	674
cdd6c482	675	void __init init_hw_perf_events(void)
59abbd1e	676	{
cdd6c482	677	pr_info("Performance events: ");
59abbd1e DM	678
	679	if (!supported_pmu()) {
	680	pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
	681	return;
	682	}
	683
	684	pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
	685
cdd6c482 IM	686	/* All sparc64 PMUs currently have 2 events. But this simple
cdd6c482 IM	687	* driver only supports one active event at a time.
59abbd1e	688	*/
cdd6c482	689	perf_max_events = 1;
59abbd1e	690
cdd6c482	691	register_die_notifier(&perf_event_nmi_notifier);
59abbd1e	692	}