[linux-2.6-block.git] / arch / powerpc / perf / 8xx-pmu.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Performance event support - PPC 8xx
 *
 * Copyright 2016 Christophe Leroy, CS Systemes d'Information
 */

#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/perf_event.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
#include <asm/pmc.h>
#include <asm/machdep.h>
#include <asm/firmware.h>
#include <asm/ptrace.h>
#include <asm/code-patching.h>

#define PERF_8xx_ID_CPU_CYCLES		1
#define PERF_8xx_ID_HW_INSTRUCTIONS	2
#define PERF_8xx_ID_ITLB_LOAD_MISS	3
#define PERF_8xx_ID_DTLB_LOAD_MISS	4

#define C(x)	PERF_COUNT_HW_CACHE_##x
#define DTLB_LOAD_MISS	(C(DTLB) | (C(OP_READ) << 8) | (C(RESULT_MISS) << 16))
#define ITLB_LOAD_MISS	(C(ITLB) | (C(OP_READ) << 8) | (C(RESULT_MISS) << 16))

extern unsigned long itlb_miss_counter, dtlb_miss_counter;
extern atomic_t instruction_counter;

static atomic_t insn_ctr_ref;
static atomic_t itlb_miss_ref;
static atomic_t dtlb_miss_ref;

static s64 get_insn_ctr(void)
{
	int ctr;
	unsigned long counta;

	do {
		ctr = atomic_read(&instruction_counter);
		counta = mfspr(SPRN_COUNTA);
	} while (ctr != atomic_read(&instruction_counter));

	return ((s64)ctr << 16) | (counta >> 16);
}

static int event_type(struct perf_event *event)
{
	switch (event->attr.type) {
	case PERF_TYPE_HARDWARE:
		if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES)
			return PERF_8xx_ID_CPU_CYCLES;
		if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS)
			return PERF_8xx_ID_HW_INSTRUCTIONS;
		break;
	case PERF_TYPE_HW_CACHE:
		if (event->attr.config == ITLB_LOAD_MISS)
			return PERF_8xx_ID_ITLB_LOAD_MISS;
		if (event->attr.config == DTLB_LOAD_MISS)
			return PERF_8xx_ID_DTLB_LOAD_MISS;
		break;
	case PERF_TYPE_RAW:
		break;
	default:
		return -ENOENT;
	}
	return -EOPNOTSUPP;
}

static int mpc8xx_pmu_event_init(struct perf_event *event)
{
	int type = event_type(event);

	if (type < 0)
		return type;
	return 0;
}

static int mpc8xx_pmu_add(struct perf_event *event, int flags)
{
	int type = event_type(event);
	s64 val = 0;

	if (type < 0)
		return type;

	switch (type) {
	case PERF_8xx_ID_CPU_CYCLES:
		val = get_tb();
		break;
	case PERF_8xx_ID_HW_INSTRUCTIONS:
		if (atomic_inc_return(&insn_ctr_ref) == 1)
			mtspr(SPRN_ICTRL, 0xc0080007);
		val = get_insn_ctr();
		break;
	case PERF_8xx_ID_ITLB_LOAD_MISS:
		if (atomic_inc_return(&itlb_miss_ref) == 1) {
			unsigned long target = patch_site_addr(&patch__itlbmiss_perf);

			patch_branch_site(&patch__itlbmiss_exit_1, target, 0);
#ifndef CONFIG_PIN_TLB_TEXT
			patch_branch_site(&patch__itlbmiss_exit_2, target, 0);
#endif
		}
		val = itlb_miss_counter;
		break;
	case PERF_8xx_ID_DTLB_LOAD_MISS:
		if (atomic_inc_return(&dtlb_miss_ref) == 1) {
			unsigned long target = patch_site_addr(&patch__dtlbmiss_perf);

			patch_branch_site(&patch__dtlbmiss_exit_1, target, 0);
			patch_branch_site(&patch__dtlbmiss_exit_2, target, 0);
			patch_branch_site(&patch__dtlbmiss_exit_3, target, 0);
		}
		val = dtlb_miss_counter;
		break;
	}
	local64_set(&event->hw.prev_count, val);
	return 0;
}

static void mpc8xx_pmu_read(struct perf_event *event)
{
	int type = event_type(event);
	s64 prev, val = 0, delta = 0;

	if (type < 0)
		return;

	do {
		prev = local64_read(&event->hw.prev_count);
		switch (type) {
		case PERF_8xx_ID_CPU_CYCLES:
			val = get_tb();
			delta = 16 * (val - prev);
			break;
		case PERF_8xx_ID_HW_INSTRUCTIONS:
			val = get_insn_ctr();
			delta = prev - val;
			if (delta < 0)
				delta += 0x1000000000000LL;
			break;
		case PERF_8xx_ID_ITLB_LOAD_MISS:
			val = itlb_miss_counter;
			delta = (s64)((s32)val - (s32)prev);
			break;
		case PERF_8xx_ID_DTLB_LOAD_MISS:
			val = dtlb_miss_counter;
			delta = (s64)((s32)val - (s32)prev);
			break;
		}
	} while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);

	local64_add(delta, &event->count);
}

static void mpc8xx_pmu_del(struct perf_event *event, int flags)
{
	/* mfspr r10, SPRN_SPRG_SCRATCH0 */
	unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) |
			    __PPC_SPR(SPRN_SPRG_SCRATCH0);

	mpc8xx_pmu_read(event);

	/* If it was the last user, stop counting to avoid useles overhead */
	switch (event_type(event)) {
	case PERF_8xx_ID_CPU_CYCLES:
		break;
	case PERF_8xx_ID_HW_INSTRUCTIONS:
		if (atomic_dec_return(&insn_ctr_ref) == 0)
			mtspr(SPRN_ICTRL, 7);
		break;
	case PERF_8xx_ID_ITLB_LOAD_MISS:
		if (atomic_dec_return(&itlb_miss_ref) == 0) {
			patch_instruction_site(&patch__itlbmiss_exit_1, insn);
#ifndef CONFIG_PIN_TLB_TEXT
			patch_instruction_site(&patch__itlbmiss_exit_2, insn);
#endif
		}
		break;
	case PERF_8xx_ID_DTLB_LOAD_MISS:
		if (atomic_dec_return(&dtlb_miss_ref) == 0) {
			patch_instruction_site(&patch__dtlbmiss_exit_1, insn);
			patch_instruction_site(&patch__dtlbmiss_exit_2, insn);
			patch_instruction_site(&patch__dtlbmiss_exit_3, insn);
		}
		break;
	}
}

static struct pmu mpc8xx_pmu = {
	.event_init	= mpc8xx_pmu_event_init,
	.add		= mpc8xx_pmu_add,
	.del		= mpc8xx_pmu_del,
	.read		= mpc8xx_pmu_read,
	.capabilities	= PERF_PMU_CAP_NO_INTERRUPT |
			  PERF_PMU_CAP_NO_NMI,
};

static int init_mpc8xx_pmu(void)
{
	mtspr(SPRN_ICTRL, 7);
	mtspr(SPRN_CMPA, 0);
	mtspr(SPRN_COUNTA, 0xffff);

	return perf_pmu_register(&mpc8xx_pmu, "cpu", PERF_TYPE_RAW);
}

early_initcall(init_mpc8xx_pmu);
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
75b82472 CL	2	/*
	3	* Performance event support - PPC 8xx
	4	*
	5	* Copyright 2016 Christophe Leroy, CS Systemes d'Information
75b82472 CL	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include <linux/sched.h>
	10	#include <linux/perf_event.h>
	11	#include <linux/percpu.h>
	12	#include <linux/hardirq.h>
	13	#include <asm/pmc.h>
	14	#include <asm/machdep.h>
	15	#include <asm/firmware.h>
	16	#include <asm/ptrace.h>
cd99ddbe	17	#include <asm/code-patching.h>
75b82472 CL	18
	19	#define PERF_8xx_ID_CPU_CYCLES 1
	20	#define PERF_8xx_ID_HW_INSTRUCTIONS 2
	21	#define PERF_8xx_ID_ITLB_LOAD_MISS 3
	22	#define PERF_8xx_ID_DTLB_LOAD_MISS 4
	23
	24	#define C(x) PERF_COUNT_HW_CACHE_##x
	25	#define DTLB_LOAD_MISS (C(DTLB) \| (C(OP_READ) << 8) \| (C(RESULT_MISS) << 16))
	26	#define ITLB_LOAD_MISS (C(ITLB) \| (C(OP_READ) << 8) \| (C(RESULT_MISS) << 16))
	27
	28	extern unsigned long itlb_miss_counter, dtlb_miss_counter;
	29	extern atomic_t instruction_counter;
	30
	31	static atomic_t insn_ctr_ref;
cd99ddbe CL	32	static atomic_t itlb_miss_ref;
cd99ddbe CL	33	static atomic_t dtlb_miss_ref;
75b82472 CL	34
	35	static s64 get_insn_ctr(void)
	36	{
	37	int ctr;
	38	unsigned long counta;
	39
	40	do {
	41	ctr = atomic_read(&instruction_counter);
	42	counta = mfspr(SPRN_COUNTA);
	43	} while (ctr != atomic_read(&instruction_counter));
	44
	45	return ((s64)ctr << 16) \| (counta >> 16);
	46	}
	47
	48	static int event_type(struct perf_event *event)
	49	{
	50	switch (event->attr.type) {
	51	case PERF_TYPE_HARDWARE:
	52	if (event->attr.config == PERF_COUNT_HW_CPU_CYCLES)
	53	return PERF_8xx_ID_CPU_CYCLES;
	54	if (event->attr.config == PERF_COUNT_HW_INSTRUCTIONS)
	55	return PERF_8xx_ID_HW_INSTRUCTIONS;
	56	break;
	57	case PERF_TYPE_HW_CACHE:
	58	if (event->attr.config == ITLB_LOAD_MISS)
	59	return PERF_8xx_ID_ITLB_LOAD_MISS;
	60	if (event->attr.config == DTLB_LOAD_MISS)
	61	return PERF_8xx_ID_DTLB_LOAD_MISS;
	62	break;
	63	case PERF_TYPE_RAW:
	64	break;
	65	default:
	66	return -ENOENT;
	67	}
	68	return -EOPNOTSUPP;
	69	}
	70
	71	static int mpc8xx_pmu_event_init(struct perf_event *event)
	72	{
	73	int type = event_type(event);
	74
	75	if (type < 0)
	76	return type;
	77	return 0;
	78	}
	79
	80	static int mpc8xx_pmu_add(struct perf_event *event, int flags)
	81	{
	82	int type = event_type(event);
	83	s64 val = 0;
	84
	85	if (type < 0)
	86	return type;
	87
	88	switch (type) {
	89	case PERF_8xx_ID_CPU_CYCLES:
	90	val = get_tb();
	91	break;
	92	case PERF_8xx_ID_HW_INSTRUCTIONS:
	93	if (atomic_inc_return(&insn_ctr_ref) == 1)
	94	mtspr(SPRN_ICTRL, 0xc0080007);
	95	val = get_insn_ctr();
	96	break;
	97	case PERF_8xx_ID_ITLB_LOAD_MISS:
cd99ddbe	98	if (atomic_inc_return(&itlb_miss_ref) == 1) {
709cf19c	99	unsigned long target = patch_site_addr(&patch__itlbmiss_perf);
cd99ddbe	100
709cf19c	101	patch_branch_site(&patch__itlbmiss_exit_1, target, 0);
cd99ddbe	102	#ifndef CONFIG_PIN_TLB_TEXT
709cf19c	103	patch_branch_site(&patch__itlbmiss_exit_2, target, 0);
cd99ddbe CL	104	#endif
cd99ddbe CL	105	}
75b82472 CL	106	val = itlb_miss_counter;
	107	break;
	108	case PERF_8xx_ID_DTLB_LOAD_MISS:
cd99ddbe	109	if (atomic_inc_return(&dtlb_miss_ref) == 1) {
709cf19c	110	unsigned long target = patch_site_addr(&patch__dtlbmiss_perf);
cd99ddbe	111
709cf19c CL	112	patch_branch_site(&patch__dtlbmiss_exit_1, target, 0);
	113	patch_branch_site(&patch__dtlbmiss_exit_2, target, 0);
	114	patch_branch_site(&patch__dtlbmiss_exit_3, target, 0);
cd99ddbe	115	}
75b82472 CL	116	val = dtlb_miss_counter;
	117	break;
	118	}
	119	local64_set(&event->hw.prev_count, val);
	120	return 0;
	121	}
	122
	123	static void mpc8xx_pmu_read(struct perf_event *event)
	124	{
	125	int type = event_type(event);
	126	s64 prev, val = 0, delta = 0;
	127
	128	if (type < 0)
	129	return;
	130
	131	do {
	132	prev = local64_read(&event->hw.prev_count);
	133	switch (type) {
	134	case PERF_8xx_ID_CPU_CYCLES:
	135	val = get_tb();
	136	delta = 16 * (val - prev);
	137	break;
	138	case PERF_8xx_ID_HW_INSTRUCTIONS:
	139	val = get_insn_ctr();
	140	delta = prev - val;
	141	if (delta < 0)
	142	delta += 0x1000000000000LL;
	143	break;
	144	case PERF_8xx_ID_ITLB_LOAD_MISS:
	145	val = itlb_miss_counter;
	146	delta = (s64)((s32)val - (s32)prev);
	147	break;
	148	case PERF_8xx_ID_DTLB_LOAD_MISS:
	149	val = dtlb_miss_counter;
	150	delta = (s64)((s32)val - (s32)prev);
	151	break;
	152	}
	153	} while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
	154
	155	local64_add(delta, &event->count);
	156	}
	157
	158	static void mpc8xx_pmu_del(struct perf_event *event, int flags)
	159	{
cd99ddbe CL	160	/* mfspr r10, SPRN_SPRG_SCRATCH0 */
	161	unsigned int insn = PPC_INST_MFSPR \| __PPC_RS(R10) \|
	162	__PPC_SPR(SPRN_SPRG_SCRATCH0);
	163
75b82472	164	mpc8xx_pmu_read(event);
75b82472 CL	165
75b82472 CL	166	/* If it was the last user, stop counting to avoid useles overhead */
cd99ddbe CL	167	switch (event_type(event)) {
	168	case PERF_8xx_ID_CPU_CYCLES:
	169	break;
	170	case PERF_8xx_ID_HW_INSTRUCTIONS:
	171	if (atomic_dec_return(&insn_ctr_ref) == 0)
	172	mtspr(SPRN_ICTRL, 7);
	173	break;
	174	case PERF_8xx_ID_ITLB_LOAD_MISS:
	175	if (atomic_dec_return(&itlb_miss_ref) == 0) {
709cf19c	176	patch_instruction_site(&patch__itlbmiss_exit_1, insn);
cd99ddbe	177	#ifndef CONFIG_PIN_TLB_TEXT
709cf19c	178	patch_instruction_site(&patch__itlbmiss_exit_2, insn);
cd99ddbe CL	179	#endif
	180	}
	181	break;
	182	case PERF_8xx_ID_DTLB_LOAD_MISS:
	183	if (atomic_dec_return(&dtlb_miss_ref) == 0) {
709cf19c CL	184	patch_instruction_site(&patch__dtlbmiss_exit_1, insn);
	185	patch_instruction_site(&patch__dtlbmiss_exit_2, insn);
	186	patch_instruction_site(&patch__dtlbmiss_exit_3, insn);
cd99ddbe CL	187	}
	188	break;
	189	}
75b82472 CL	190	}
	191
	192	static struct pmu mpc8xx_pmu = {
	193	.event_init = mpc8xx_pmu_event_init,
	194	.add = mpc8xx_pmu_add,
	195	.del = mpc8xx_pmu_del,
	196	.read = mpc8xx_pmu_read,
	197	.capabilities = PERF_PMU_CAP_NO_INTERRUPT \|
	198	PERF_PMU_CAP_NO_NMI,
	199	};
	200
	201	static int init_mpc8xx_pmu(void)
	202	{
	203	mtspr(SPRN_ICTRL, 7);
	204	mtspr(SPRN_CMPA, 0);
	205	mtspr(SPRN_COUNTA, 0xffff);
	206
	207	return perf_pmu_register(&mpc8xx_pmu, "cpu", PERF_TYPE_RAW);
	208	}
	209
	210	early_initcall(init_mpc8xx_pmu);