[linux-2.6-block.git] / arch / powerpc / perf / power9-pmu.c

/*
 * Performance counter support for POWER9 processors.
 *
 * Copyright 2009 Paul Mackerras, IBM Corporation.
 * Copyright 2013 Michael Ellerman, IBM Corporation.
 * Copyright 2016 Madhavan Srinivasan, IBM Corporation.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or later version.
 */

#define pr_fmt(fmt)	"power9-pmu: " fmt

#include "isa207-common.h"

/*
 * Raw event encoding for Power9:
 *
 *        60        56        52        48        44        40        36        32
 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
 *   | | [ ]                       [ ] [      thresh_cmp     ]   [  thresh_ctl   ]
 *   | |  |                         |                                     |
 *   | |  *- IFM (Linux)            |	               thresh start/stop -*
 *   | *- BHRB (Linux)              *sm
 *   *- EBB (Linux)
 *
 *        28        24        20        16        12         8         4         0
 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
 *   [   ] [  sample ]   [cache]   [ pmc ]   [unit ]   []    m   [    pmcxsel    ]
 *     |        |           |                          |     |
 *     |        |           |                          |     *- mark
 *     |        |           *- L1/L2/L3 cache_sel      |
 *     |        |                                      |
 *     |        *- sampling mode for marked events     *- combine
 *     |
 *     *- thresh_sel
 *
 * Below uses IBM bit numbering.
 *
 * MMCR1[x:y] = unit    (PMCxUNIT)
 * MMCR1[24]   = pmc1combine[0]
 * MMCR1[25]   = pmc1combine[1]
 * MMCR1[26]   = pmc2combine[0]
 * MMCR1[27]   = pmc2combine[1]
 * MMCR1[28]   = pmc3combine[0]
 * MMCR1[29]   = pmc3combine[1]
 * MMCR1[30]   = pmc4combine[0]
 * MMCR1[31]   = pmc4combine[1]
 *
 * if pmc == 3 and unit == 0 and pmcxsel[0:6] == 0b0101011
 *	MMCR1[20:27] = thresh_ctl
 * else if pmc == 4 and unit == 0xf and pmcxsel[0:6] == 0b0101001
 *	MMCR1[20:27] = thresh_ctl
 * else
 *	MMCRA[48:55] = thresh_ctl   (THRESH START/END)
 *
 * if thresh_sel:
 *	MMCRA[45:47] = thresh_sel
 *
 * if thresh_cmp:
 *	MMCRA[9:11] = thresh_cmp[0:2]
 *	MMCRA[12:18] = thresh_cmp[3:9]
 *
 * MMCR1[16] = cache_sel[2]
 * MMCR1[17] = cache_sel[3]
 *
 * if mark:
 *	MMCRA[63]    = 1		(SAMPLE_ENABLE)
 *	MMCRA[57:59] = sample[0:2]	(RAND_SAMP_ELIG)
 *	MMCRA[61:62] = sample[3:4]	(RAND_SAMP_MODE)
 *
 * if EBB and BHRB:
 *	MMCRA[32:33] = IFM
 *
 * MMCRA[SDAR_MODE]  = sm
 */

/*
 * Some power9 event codes.
 */
#define EVENT(_name, _code)	_name = _code,

enum {
#include "power9-events-list.h"
};

#undef EVENT

/* MMCRA IFM bits - POWER9 */
#define POWER9_MMCRA_IFM1		0x0000000040000000UL
#define POWER9_MMCRA_IFM2		0x0000000080000000UL
#define POWER9_MMCRA_IFM3		0x00000000C0000000UL

/* Nasty Power9 specific hack */
#define PVR_POWER9_CUMULUS		0x00002000

/* PowerISA v2.07 format attribute structure*/
extern struct attribute_group isa207_pmu_format_group;

int p9_dd21_bl_ev[] = {
	PM_MRK_ST_DONE_L2,
	PM_RADIX_PWC_L1_HIT,
	PM_FLOP_CMPL,
	PM_MRK_NTF_FIN,
	PM_RADIX_PWC_L2_HIT,
	PM_IFETCH_THROTTLE,
	PM_MRK_L2_TM_ST_ABORT_SISTER,
	PM_RADIX_PWC_L3_HIT,
	PM_RUN_CYC_SMT2_MODE,
	PM_TM_TX_PASS_RUN_INST,
	PM_DISP_HELD_SYNC_HOLD,
};

int p9_dd22_bl_ev[] = {
	PM_DTLB_MISS_16G,
	PM_DERAT_MISS_2M,
	PM_DTLB_MISS_2M,
	PM_MRK_DTLB_MISS_1G,
	PM_DTLB_MISS_4K,
	PM_DERAT_MISS_1G,
	PM_MRK_DERAT_MISS_2M,
	PM_MRK_DTLB_MISS_4K,
	PM_MRK_DTLB_MISS_16G,
	PM_DTLB_MISS_64K,
	PM_MRK_DERAT_MISS_1G,
	PM_MRK_DTLB_MISS_64K,
	PM_DISP_HELD_SYNC_HOLD,
	PM_DTLB_MISS_16M,
	PM_DTLB_MISS_1G,
	PM_MRK_DTLB_MISS_16M,
};

/* Table of alternatives, sorted by column 0 */
static const unsigned int power9_event_alternatives[][MAX_ALT] = {
	{ PM_INST_DISP,			PM_INST_DISP_ALT },
	{ PM_RUN_CYC_ALT,		PM_RUN_CYC },
	{ PM_RUN_INST_CMPL_ALT,		PM_RUN_INST_CMPL },
	{ PM_LD_MISS_L1,		PM_LD_MISS_L1_ALT },
	{ PM_BR_2PATH,			PM_BR_2PATH_ALT },
};

static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[])
{
	int num_alt = 0;

	num_alt = isa207_get_alternatives(event, alt,
					  ARRAY_SIZE(power9_event_alternatives), flags,
					  power9_event_alternatives);

	return num_alt;
}

GENERIC_EVENT_ATTR(cpu-cycles,			PM_CYC);
GENERIC_EVENT_ATTR(stalled-cycles-frontend,	PM_ICT_NOSLOT_CYC);
GENERIC_EVENT_ATTR(stalled-cycles-backend,	PM_CMPLU_STALL);
GENERIC_EVENT_ATTR(instructions,		PM_INST_CMPL);
GENERIC_EVENT_ATTR(branch-instructions,		PM_BR_CMPL);
GENERIC_EVENT_ATTR(branch-misses,		PM_BR_MPRED_CMPL);
GENERIC_EVENT_ATTR(cache-references,		PM_LD_REF_L1);
GENERIC_EVENT_ATTR(cache-misses,		PM_LD_MISS_L1_FIN);
GENERIC_EVENT_ATTR(mem-loads,			MEM_LOADS);
GENERIC_EVENT_ATTR(mem-stores,			MEM_STORES);

CACHE_EVENT_ATTR(L1-dcache-load-misses,		PM_LD_MISS_L1_FIN);
CACHE_EVENT_ATTR(L1-dcache-loads,		PM_LD_REF_L1);
CACHE_EVENT_ATTR(L1-dcache-prefetches,		PM_L1_PREF);
CACHE_EVENT_ATTR(L1-dcache-store-misses,	PM_ST_MISS_L1);
CACHE_EVENT_ATTR(L1-icache-load-misses,		PM_L1_ICACHE_MISS);
CACHE_EVENT_ATTR(L1-icache-loads,		PM_INST_FROM_L1);
CACHE_EVENT_ATTR(L1-icache-prefetches,		PM_IC_PREF_WRITE);
CACHE_EVENT_ATTR(LLC-load-misses,		PM_DATA_FROM_L3MISS);
CACHE_EVENT_ATTR(LLC-loads,			PM_DATA_FROM_L3);
CACHE_EVENT_ATTR(LLC-prefetches,		PM_L3_PREF_ALL);
CACHE_EVENT_ATTR(branch-load-misses,		PM_BR_MPRED_CMPL);
CACHE_EVENT_ATTR(branch-loads,			PM_BR_CMPL);
CACHE_EVENT_ATTR(dTLB-load-misses,		PM_DTLB_MISS);
CACHE_EVENT_ATTR(iTLB-load-misses,		PM_ITLB_MISS);

static struct attribute *power9_events_attr[] = {
	GENERIC_EVENT_PTR(PM_CYC),
	GENERIC_EVENT_PTR(PM_ICT_NOSLOT_CYC),
	GENERIC_EVENT_PTR(PM_CMPLU_STALL),
	GENERIC_EVENT_PTR(PM_INST_CMPL),
	GENERIC_EVENT_PTR(PM_BR_CMPL),
	GENERIC_EVENT_PTR(PM_BR_MPRED_CMPL),
	GENERIC_EVENT_PTR(PM_LD_REF_L1),
	GENERIC_EVENT_PTR(PM_LD_MISS_L1_FIN),
	GENERIC_EVENT_PTR(MEM_LOADS),
	GENERIC_EVENT_PTR(MEM_STORES),
	CACHE_EVENT_PTR(PM_LD_MISS_L1_FIN),
	CACHE_EVENT_PTR(PM_LD_REF_L1),
	CACHE_EVENT_PTR(PM_L1_PREF),
	CACHE_EVENT_PTR(PM_ST_MISS_L1),
	CACHE_EVENT_PTR(PM_L1_ICACHE_MISS),
	CACHE_EVENT_PTR(PM_INST_FROM_L1),
	CACHE_EVENT_PTR(PM_IC_PREF_WRITE),
	CACHE_EVENT_PTR(PM_DATA_FROM_L3MISS),
	CACHE_EVENT_PTR(PM_DATA_FROM_L3),
	CACHE_EVENT_PTR(PM_L3_PREF_ALL),
	CACHE_EVENT_PTR(PM_BR_MPRED_CMPL),
	CACHE_EVENT_PTR(PM_BR_CMPL),
	CACHE_EVENT_PTR(PM_DTLB_MISS),
	CACHE_EVENT_PTR(PM_ITLB_MISS),
	NULL
};

static struct attribute_group power9_pmu_events_group = {
	.name = "events",
	.attrs = power9_events_attr,
};

PMU_FORMAT_ATTR(event,		"config:0-51");
PMU_FORMAT_ATTR(pmcxsel,	"config:0-7");
PMU_FORMAT_ATTR(mark,		"config:8");
PMU_FORMAT_ATTR(combine,	"config:10-11");
PMU_FORMAT_ATTR(unit,		"config:12-15");
PMU_FORMAT_ATTR(pmc,		"config:16-19");
PMU_FORMAT_ATTR(cache_sel,	"config:20-23");
PMU_FORMAT_ATTR(sample_mode,	"config:24-28");
PMU_FORMAT_ATTR(thresh_sel,	"config:29-31");
PMU_FORMAT_ATTR(thresh_stop,	"config:32-35");
PMU_FORMAT_ATTR(thresh_start,	"config:36-39");
PMU_FORMAT_ATTR(thresh_cmp,	"config:40-49");
PMU_FORMAT_ATTR(sdar_mode,	"config:50-51");

static struct attribute *power9_pmu_format_attr[] = {
	&format_attr_event.attr,
	&format_attr_pmcxsel.attr,
	&format_attr_mark.attr,
	&format_attr_combine.attr,
	&format_attr_unit.attr,
	&format_attr_pmc.attr,
	&format_attr_cache_sel.attr,
	&format_attr_sample_mode.attr,
	&format_attr_thresh_sel.attr,
	&format_attr_thresh_stop.attr,
	&format_attr_thresh_start.attr,
	&format_attr_thresh_cmp.attr,
	&format_attr_sdar_mode.attr,
	NULL,
};

static struct attribute_group power9_pmu_format_group = {
	.name = "format",
	.attrs = power9_pmu_format_attr,
};

static const struct attribute_group *power9_pmu_attr_groups[] = {
	&power9_pmu_format_group,
	&power9_pmu_events_group,
	NULL,
};

static int power9_generic_events[] = {
	[PERF_COUNT_HW_CPU_CYCLES] =			PM_CYC,
	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =	PM_ICT_NOSLOT_CYC,
	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =	PM_CMPLU_STALL,
	[PERF_COUNT_HW_INSTRUCTIONS] =			PM_INST_CMPL,
	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] =		PM_BR_CMPL,
	[PERF_COUNT_HW_BRANCH_MISSES] =			PM_BR_MPRED_CMPL,
	[PERF_COUNT_HW_CACHE_REFERENCES] =		PM_LD_REF_L1,
	[PERF_COUNT_HW_CACHE_MISSES] =			PM_LD_MISS_L1_FIN,
};

static u64 power9_bhrb_filter_map(u64 branch_sample_type)
{
	u64 pmu_bhrb_filter = 0;

	/* BHRB and regular PMU events share the same privilege state
	 * filter configuration. BHRB is always recorded along with a
	 * regular PMU event. As the privilege state filter is handled
	 * in the basic PMC configuration of the accompanying regular
	 * PMU event, we ignore any separate BHRB specific request.
	 */

	/* No branch filter requested */
	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY)
		return pmu_bhrb_filter;

	/* Invalid branch filter options - HW does not support */
	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
		return -1;

	if (branch_sample_type & PERF_SAMPLE_BRANCH_IND_CALL)
		return -1;

	if (branch_sample_type & PERF_SAMPLE_BRANCH_CALL)
		return -1;

	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
		pmu_bhrb_filter |= POWER9_MMCRA_IFM1;
		return pmu_bhrb_filter;
	}

	/* Every thing else is unsupported */
	return -1;
}

static void power9_config_bhrb(u64 pmu_bhrb_filter)
{
	/* Enable BHRB filter in PMU */
	mtspr(SPRN_MMCRA, (mfspr(SPRN_MMCRA) | pmu_bhrb_filter));
}

#define C(x)	PERF_COUNT_HW_CACHE_##x

/*
 * Table of generalized cache-related events.
 * 0 means not supported, -1 means nonsensical, other values
 * are event codes.
 */
static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
	[ C(L1D) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = PM_LD_REF_L1,
			[ C(RESULT_MISS)   ] = PM_LD_MISS_L1_FIN,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = 0,
			[ C(RESULT_MISS)   ] = PM_ST_MISS_L1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = PM_L1_PREF,
			[ C(RESULT_MISS)   ] = 0,
		},
	},
	[ C(L1I) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = PM_INST_FROM_L1,
			[ C(RESULT_MISS)   ] = PM_L1_ICACHE_MISS,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = PM_L1_DEMAND_WRITE,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = PM_IC_PREF_WRITE,
			[ C(RESULT_MISS)   ] = 0,
		},
	},
	[ C(LL) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = PM_DATA_FROM_L3,
			[ C(RESULT_MISS)   ] = PM_DATA_FROM_L3MISS,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = 0,
			[ C(RESULT_MISS)   ] = 0,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = PM_L3_PREF_ALL,
			[ C(RESULT_MISS)   ] = 0,
		},
	},
	[ C(DTLB) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = 0,
			[ C(RESULT_MISS)   ] = PM_DTLB_MISS,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
	},
	[ C(ITLB) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = 0,
			[ C(RESULT_MISS)   ] = PM_ITLB_MISS,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
	},
	[ C(BPU) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = PM_BR_CMPL,
			[ C(RESULT_MISS)   ] = PM_BR_MPRED_CMPL,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
	},
	[ C(NODE) ] = {
		[ C(OP_READ) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_WRITE) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
		[ C(OP_PREFETCH) ] = {
			[ C(RESULT_ACCESS) ] = -1,
			[ C(RESULT_MISS)   ] = -1,
		},
	},
};

#undef C

static struct power_pmu power9_pmu = {
	.name			= "POWER9",
	.n_counter		= MAX_PMU_COUNTERS,
	.add_fields		= ISA207_ADD_FIELDS,
	.test_adder		= ISA207_TEST_ADDER,
	.group_constraint_mask	= CNST_CACHE_PMC4_MASK,
	.group_constraint_val	= CNST_CACHE_PMC4_VAL,
	.compute_mmcr		= isa207_compute_mmcr,
	.config_bhrb		= power9_config_bhrb,
	.bhrb_filter_map	= power9_bhrb_filter_map,
	.get_constraint		= isa207_get_constraint,
	.get_alternatives	= power9_get_alternatives,
	.get_mem_data_src	= isa207_get_mem_data_src,
	.get_mem_weight		= isa207_get_mem_weight,
	.disable_pmc		= isa207_disable_pmc,
	.flags			= PPMU_HAS_SIER | PPMU_ARCH_207S,
	.n_generic		= ARRAY_SIZE(power9_generic_events),
	.generic_events		= power9_generic_events,
	.cache_events		= &power9_cache_events,
	.attr_groups		= power9_pmu_attr_groups,
	.bhrb_nr		= 32,
};

static int __init init_power9_pmu(void)
{
	int rc = 0;
	unsigned int pvr = mfspr(SPRN_PVR);

	/* Comes from cpu_specs[] */
	if (!cur_cpu_spec->oprofile_cpu_type ||
	    strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power9"))
		return -ENODEV;

	/* Blacklist events */
	if (!(pvr & PVR_POWER9_CUMULUS)) {
		if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 1)) {
			power9_pmu.blacklist_ev = p9_dd21_bl_ev;
			power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd21_bl_ev);
		} else if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 2)) {
			power9_pmu.blacklist_ev = p9_dd22_bl_ev;
			power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd22_bl_ev);
		}
	}

	rc = register_power_pmu(&power9_pmu);
	if (rc)
		return rc;

	/* Tell userspace that EBB is supported */
	cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_EBB;

	return 0;
}
early_initcall(init_power9_pmu);
Commit	Line	Data
8c002dbd MS	1	/*
	2	* Performance counter support for POWER9 processors.
	3	*
	4	* Copyright 2009 Paul Mackerras, IBM Corporation.
	5	* Copyright 2013 Michael Ellerman, IBM Corporation.
	6	* Copyright 2016 Madhavan Srinivasan, IBM Corporation.
	7	*
	8	* This program is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU General Public License
	10	* as published by the Free Software Foundation; either version
	11	* 2 of the License, or later version.
	12	*/
	13
	14	#define pr_fmt(fmt) "power9-pmu: " fmt
	15
	16	#include "isa207-common.h"
	17
18201b20 MS	18	/*
	19	* Raw event encoding for Power9:
	20	*
	21	* 60 56 52 48 44 40 36 32
	22	* \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \|
	23	* \| \| [ ] [ ] [ thresh_cmp ] [ thresh_ctl ]
	24	* \| \| \| \| \|
78a16d9f	25	* \| \| - IFM (Linux) \| thresh start/stop -
18201b20 MS	26	* \| - BHRB (Linux) sm
	27	* *- EBB (Linux)
	28	*
	29	* 28 24 20 16 12 8 4 0
	30	* \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \| - - - - \|
	31	* [ ] [ sample ] [cache] [ pmc ] [unit ] [] m [ pmcxsel ]
	32	* \| \| \| \| \|
	33	* \| \| \| \| *- mark
	34	* \| \| *- L1/L2/L3 cache_sel \|
	35	* \| \| \|
	36	* \| - sampling mode for marked events - combine
	37	* \|
	38	* *- thresh_sel
	39	*
	40	* Below uses IBM bit numbering.
	41	*
	42	* MMCR1[x:y] = unit (PMCxUNIT)
	43	* MMCR1[24] = pmc1combine[0]
	44	* MMCR1[25] = pmc1combine[1]
	45	* MMCR1[26] = pmc2combine[0]
	46	* MMCR1[27] = pmc2combine[1]
	47	* MMCR1[28] = pmc3combine[0]
	48	* MMCR1[29] = pmc3combine[1]
	49	* MMCR1[30] = pmc4combine[0]
	50	* MMCR1[31] = pmc4combine[1]
	51	*
	52	* if pmc == 3 and unit == 0 and pmcxsel[0:6] == 0b0101011
78a16d9f	53	* MMCR1[20:27] = thresh_ctl
18201b20	54	* else if pmc == 4 and unit == 0xf and pmcxsel[0:6] == 0b0101001
78a16d9f	55	* MMCR1[20:27] = thresh_ctl
18201b20 MS	56	* else
	57	* MMCRA[48:55] = thresh_ctl (THRESH START/END)
	58	*
	59	* if thresh_sel:
	60	* MMCRA[45:47] = thresh_sel
	61	*
	62	* if thresh_cmp:
	63	* MMCRA[9:11] = thresh_cmp[0:2]
	64	* MMCRA[12:18] = thresh_cmp[3:9]
	65	*
8c31459d MS	66	* MMCR1[16] = cache_sel[2]
8c31459d MS	67	* MMCR1[17] = cache_sel[3]
18201b20 MS	68	*
	69	* if mark:
	70	* MMCRA[63] = 1 (SAMPLE_ENABLE)
	71	* MMCRA[57:59] = sample[0:2] (RAND_SAMP_ELIG)
	72	* MMCRA[61:62] = sample[3:4] (RAND_SAMP_MODE)
	73	*
	74	* if EBB and BHRB:
	75	* MMCRA[32:33] = IFM
	76	*
	77	* MMCRA[SDAR_MODE] = sm
	78	*/
	79
8c002dbd MS	80	/*
	81	* Some power9 event codes.
	82	*/
	83	#define EVENT(_name, _code) _name = _code,
	84
	85	enum {
	86	#include "power9-events-list.h"
	87	};
	88
	89	#undef EVENT
	90
	91	/* MMCRA IFM bits - POWER9 */
	92	#define POWER9_MMCRA_IFM1 0x0000000040000000UL
	93	#define POWER9_MMCRA_IFM2 0x0000000080000000UL
	94	#define POWER9_MMCRA_IFM3 0x00000000C0000000UL
	95
64acab4e MS	96	/* Nasty Power9 specific hack */
	97	#define PVR_POWER9_CUMULUS 0x00002000
	98
60b00025 MS	99	/* PowerISA v2.07 format attribute structure*/
	100	extern struct attribute_group isa207_pmu_format_group;
	101
64acab4e MS	102	int p9_dd21_bl_ev[] = {
	103	PM_MRK_ST_DONE_L2,
	104	PM_RADIX_PWC_L1_HIT,
	105	PM_FLOP_CMPL,
	106	PM_MRK_NTF_FIN,
	107	PM_RADIX_PWC_L2_HIT,
	108	PM_IFETCH_THROTTLE,
	109	PM_MRK_L2_TM_ST_ABORT_SISTER,
	110	PM_RADIX_PWC_L3_HIT,
	111	PM_RUN_CYC_SMT2_MODE,
	112	PM_TM_TX_PASS_RUN_INST,
	113	PM_DISP_HELD_SYNC_HOLD,
	114	};
	115
ac96588d MS	116	int p9_dd22_bl_ev[] = {
	117	PM_DTLB_MISS_16G,
	118	PM_DERAT_MISS_2M,
	119	PM_DTLB_MISS_2M,
	120	PM_MRK_DTLB_MISS_1G,
	121	PM_DTLB_MISS_4K,
	122	PM_DERAT_MISS_1G,
	123	PM_MRK_DERAT_MISS_2M,
	124	PM_MRK_DTLB_MISS_4K,
	125	PM_MRK_DTLB_MISS_16G,
	126	PM_DTLB_MISS_64K,
	127	PM_MRK_DERAT_MISS_1G,
	128	PM_MRK_DTLB_MISS_64K,
	129	PM_DISP_HELD_SYNC_HOLD,
	130	PM_DTLB_MISS_16M,
	131	PM_DTLB_MISS_1G,
	132	PM_MRK_DTLB_MISS_16M,
	133	};
	134
a114aca5 MS	135	/* Table of alternatives, sorted by column 0 */
	136	static const unsigned int power9_event_alternatives[][MAX_ALT] = {
	137	{ PM_INST_DISP, PM_INST_DISP_ALT },
3f0bd8da AB	138	{ PM_RUN_CYC_ALT, PM_RUN_CYC },
3f0bd8da AB	139	{ PM_RUN_INST_CMPL_ALT, PM_RUN_INST_CMPL },
91e0bd1e MS	140	{ PM_LD_MISS_L1, PM_LD_MISS_L1_ALT },
91e0bd1e MS	141	{ PM_BR_2PATH, PM_BR_2PATH_ALT },
a114aca5 MS	142	};
	143
	144	static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[])
	145	{
	146	int num_alt = 0;
	147
70a7e720 MS	148	num_alt = isa207_get_alternatives(event, alt,
	149	ARRAY_SIZE(power9_event_alternatives), flags,
	150	power9_event_alternatives);
a114aca5 MS	151
	152	return num_alt;
	153	}
	154
f1fb60bf MS	155	GENERIC_EVENT_ATTR(cpu-cycles, PM_CYC);
	156	GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC);
	157	GENERIC_EVENT_ATTR(stalled-cycles-backend, PM_CMPLU_STALL);
	158	GENERIC_EVENT_ATTR(instructions, PM_INST_CMPL);
93fc5ca9	159	GENERIC_EVENT_ATTR(branch-instructions, PM_BR_CMPL);
f1fb60bf MS	160	GENERIC_EVENT_ATTR(branch-misses, PM_BR_MPRED_CMPL);
	161	GENERIC_EVENT_ATTR(cache-references, PM_LD_REF_L1);
	162	GENERIC_EVENT_ATTR(cache-misses, PM_LD_MISS_L1_FIN);
ab4510e9 MS	163	GENERIC_EVENT_ATTR(mem-loads, MEM_LOADS);
ab4510e9 MS	164	GENERIC_EVENT_ATTR(mem-stores, MEM_STORES);
f1fb60bf MS	165
	166	CACHE_EVENT_ATTR(L1-dcache-load-misses, PM_LD_MISS_L1_FIN);
	167	CACHE_EVENT_ATTR(L1-dcache-loads, PM_LD_REF_L1);
	168	CACHE_EVENT_ATTR(L1-dcache-prefetches, PM_L1_PREF);
	169	CACHE_EVENT_ATTR(L1-dcache-store-misses, PM_ST_MISS_L1);
	170	CACHE_EVENT_ATTR(L1-icache-load-misses, PM_L1_ICACHE_MISS);
	171	CACHE_EVENT_ATTR(L1-icache-loads, PM_INST_FROM_L1);
	172	CACHE_EVENT_ATTR(L1-icache-prefetches, PM_IC_PREF_WRITE);
	173	CACHE_EVENT_ATTR(LLC-load-misses, PM_DATA_FROM_L3MISS);
	174	CACHE_EVENT_ATTR(LLC-loads, PM_DATA_FROM_L3);
	175	CACHE_EVENT_ATTR(LLC-prefetches, PM_L3_PREF_ALL);
f1fb60bf	176	CACHE_EVENT_ATTR(branch-load-misses, PM_BR_MPRED_CMPL);
93fc5ca9	177	CACHE_EVENT_ATTR(branch-loads, PM_BR_CMPL);
f1fb60bf MS	178	CACHE_EVENT_ATTR(dTLB-load-misses, PM_DTLB_MISS);
	179	CACHE_EVENT_ATTR(iTLB-load-misses, PM_ITLB_MISS);
	180
	181	static struct attribute *power9_events_attr[] = {
	182	GENERIC_EVENT_PTR(PM_CYC),
	183	GENERIC_EVENT_PTR(PM_ICT_NOSLOT_CYC),
	184	GENERIC_EVENT_PTR(PM_CMPLU_STALL),
	185	GENERIC_EVENT_PTR(PM_INST_CMPL),
93fc5ca9	186	GENERIC_EVENT_PTR(PM_BR_CMPL),
f1fb60bf MS	187	GENERIC_EVENT_PTR(PM_BR_MPRED_CMPL),
	188	GENERIC_EVENT_PTR(PM_LD_REF_L1),
	189	GENERIC_EVENT_PTR(PM_LD_MISS_L1_FIN),
ab4510e9 MS	190	GENERIC_EVENT_PTR(MEM_LOADS),
ab4510e9 MS	191	GENERIC_EVENT_PTR(MEM_STORES),
f1fb60bf MS	192	CACHE_EVENT_PTR(PM_LD_MISS_L1_FIN),
	193	CACHE_EVENT_PTR(PM_LD_REF_L1),
	194	CACHE_EVENT_PTR(PM_L1_PREF),
	195	CACHE_EVENT_PTR(PM_ST_MISS_L1),
	196	CACHE_EVENT_PTR(PM_L1_ICACHE_MISS),
	197	CACHE_EVENT_PTR(PM_INST_FROM_L1),
	198	CACHE_EVENT_PTR(PM_IC_PREF_WRITE),
	199	CACHE_EVENT_PTR(PM_DATA_FROM_L3MISS),
	200	CACHE_EVENT_PTR(PM_DATA_FROM_L3),
	201	CACHE_EVENT_PTR(PM_L3_PREF_ALL),
f1fb60bf	202	CACHE_EVENT_PTR(PM_BR_MPRED_CMPL),
93fc5ca9	203	CACHE_EVENT_PTR(PM_BR_CMPL),
f1fb60bf MS	204	CACHE_EVENT_PTR(PM_DTLB_MISS),
	205	CACHE_EVENT_PTR(PM_ITLB_MISS),
	206	NULL
	207	};
	208
	209	static struct attribute_group power9_pmu_events_group = {
	210	.name = "events",
	211	.attrs = power9_events_attr,
	212	};
8c002dbd	213
18201b20 MS	214	PMU_FORMAT_ATTR(event, "config:0-51");
	215	PMU_FORMAT_ATTR(pmcxsel, "config:0-7");
	216	PMU_FORMAT_ATTR(mark, "config:8");
	217	PMU_FORMAT_ATTR(combine, "config:10-11");
	218	PMU_FORMAT_ATTR(unit, "config:12-15");
	219	PMU_FORMAT_ATTR(pmc, "config:16-19");
	220	PMU_FORMAT_ATTR(cache_sel, "config:20-23");
	221	PMU_FORMAT_ATTR(sample_mode, "config:24-28");
	222	PMU_FORMAT_ATTR(thresh_sel, "config:29-31");
	223	PMU_FORMAT_ATTR(thresh_stop, "config:32-35");
	224	PMU_FORMAT_ATTR(thresh_start, "config:36-39");
	225	PMU_FORMAT_ATTR(thresh_cmp, "config:40-49");
	226	PMU_FORMAT_ATTR(sdar_mode, "config:50-51");
	227
	228	static struct attribute *power9_pmu_format_attr[] = {
	229	&format_attr_event.attr,
	230	&format_attr_pmcxsel.attr,
	231	&format_attr_mark.attr,
	232	&format_attr_combine.attr,
	233	&format_attr_unit.attr,
	234	&format_attr_pmc.attr,
	235	&format_attr_cache_sel.attr,
	236	&format_attr_sample_mode.attr,
	237	&format_attr_thresh_sel.attr,
	238	&format_attr_thresh_stop.attr,
	239	&format_attr_thresh_start.attr,
	240	&format_attr_thresh_cmp.attr,
	241	&format_attr_sdar_mode.attr,
	242	NULL,
	243	};
	244
	245	static struct attribute_group power9_pmu_format_group = {
	246	.name = "format",
	247	.attrs = power9_pmu_format_attr,
	248	};
	249
	250	static const struct attribute_group *power9_pmu_attr_groups[] = {
	251	&power9_pmu_format_group,
	252	&power9_pmu_events_group,
	253	NULL,
	254	};
	255
8c002dbd MS	256	static int power9_generic_events[] = {
	257	[PERF_COUNT_HW_CPU_CYCLES] = PM_CYC,
	258	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC,
	259	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = PM_CMPLU_STALL,
	260	[PERF_COUNT_HW_INSTRUCTIONS] = PM_INST_CMPL,
93fc5ca9	261	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_CMPL,
8c002dbd MS	262	[PERF_COUNT_HW_BRANCH_MISSES] = PM_BR_MPRED_CMPL,
	263	[PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1,
	264	[PERF_COUNT_HW_CACHE_MISSES] = PM_LD_MISS_L1_FIN,
	265	};
	266
	267	static u64 power9_bhrb_filter_map(u64 branch_sample_type)
	268	{
	269	u64 pmu_bhrb_filter = 0;
	270
	271	/* BHRB and regular PMU events share the same privilege state
	272	* filter configuration. BHRB is always recorded along with a
	273	* regular PMU event. As the privilege state filter is handled
	274	* in the basic PMC configuration of the accompanying regular
	275	* PMU event, we ignore any separate BHRB specific request.
	276	*/
	277
	278	/* No branch filter requested */
	279	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY)
	280	return pmu_bhrb_filter;
	281
	282	/* Invalid branch filter options - HW does not support */
	283	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
	284	return -1;
	285
	286	if (branch_sample_type & PERF_SAMPLE_BRANCH_IND_CALL)
	287	return -1;
	288
	289	if (branch_sample_type & PERF_SAMPLE_BRANCH_CALL)
	290	return -1;
	291
	292	if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
	293	pmu_bhrb_filter \|= POWER9_MMCRA_IFM1;
	294	return pmu_bhrb_filter;
	295	}
	296
	297	/* Every thing else is unsupported */
	298	return -1;
	299	}
	300
	301	static void power9_config_bhrb(u64 pmu_bhrb_filter)
	302	{
	303	/* Enable BHRB filter in PMU */
	304	mtspr(SPRN_MMCRA, (mfspr(SPRN_MMCRA) \| pmu_bhrb_filter));
	305	}
	306
	307	#define C(x) PERF_COUNT_HW_CACHE_##x
	308
	309	/*
	310	* Table of generalized cache-related events.
	311	* 0 means not supported, -1 means nonsensical, other values
	312	* are event codes.
	313	*/
	314	static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
	315	[ C(L1D) ] = {
	316	[ C(OP_READ) ] = {
	317	[ C(RESULT_ACCESS) ] = PM_LD_REF_L1,
	318	[ C(RESULT_MISS) ] = PM_LD_MISS_L1_FIN,
	319	},
	320	[ C(OP_WRITE) ] = {
	321	[ C(RESULT_ACCESS) ] = 0,
	322	[ C(RESULT_MISS) ] = PM_ST_MISS_L1,
	323	},
	324	[ C(OP_PREFETCH) ] = {
	325	[ C(RESULT_ACCESS) ] = PM_L1_PREF,
326	[ C(RESULT_MISS) ] = 0,
327	},
328	},
329	[ C(L1I) ] = {
330	[ C(OP_READ) ] = {
331	[ C(RESULT_ACCESS) ] = PM_INST_FROM_L1,
332	[ C(RESULT_MISS) ] = PM_L1_ICACHE_MISS,
333	},
334	[ C(OP_WRITE) ] = {
335	[ C(RESULT_ACCESS) ] = PM_L1_DEMAND_WRITE,
336	[ C(RESULT_MISS) ] = -1,
337	},
338	[ C(OP_PREFETCH) ] = {
339	[ C(RESULT_ACCESS) ] = PM_IC_PREF_WRITE,
340	[ C(RESULT_MISS) ] = 0,
341	},
342	},
343	[ C(LL) ] = {
344	[ C(OP_READ) ] = {
345	[ C(RESULT_ACCESS) ] = PM_DATA_FROM_L3,
346	[ C(RESULT_MISS) ] = PM_DATA_FROM_L3MISS,
347	},
348	[ C(OP_WRITE) ] = {
3757cba8 MS	349	[ C(RESULT_ACCESS) ] = 0,
3757cba8 MS	350	[ C(RESULT_MISS) ] = 0,
8c002dbd MS	351	},
	352	[ C(OP_PREFETCH) ] = {
	353	[ C(RESULT_ACCESS) ] = PM_L3_PREF_ALL,
	354	[ C(RESULT_MISS) ] = 0,
	355	},
	356	},
	357	[ C(DTLB) ] = {
	358	[ C(OP_READ) ] = {
	359	[ C(RESULT_ACCESS) ] = 0,
	360	[ C(RESULT_MISS) ] = PM_DTLB_MISS,
	361	},
	362	[ C(OP_WRITE) ] = {
	363	[ C(RESULT_ACCESS) ] = -1,
	364	[ C(RESULT_MISS) ] = -1,
	365	},
	366	[ C(OP_PREFETCH) ] = {
	367	[ C(RESULT_ACCESS) ] = -1,
	368	[ C(RESULT_MISS) ] = -1,
	369	},
	370	},
	371	[ C(ITLB) ] = {
	372	[ C(OP_READ) ] = {
	373	[ C(RESULT_ACCESS) ] = 0,
	374	[ C(RESULT_MISS) ] = PM_ITLB_MISS,
	375	},
	376	[ C(OP_WRITE) ] = {
	377	[ C(RESULT_ACCESS) ] = -1,
	378	[ C(RESULT_MISS) ] = -1,
	379	},
	380	[ C(OP_PREFETCH) ] = {
	381	[ C(RESULT_ACCESS) ] = -1,
	382	[ C(RESULT_MISS) ] = -1,
	383	},
	384	},
	385	[ C(BPU) ] = {
	386	[ C(OP_READ) ] = {
93fc5ca9	387	[ C(RESULT_ACCESS) ] = PM_BR_CMPL,
8c002dbd MS	388	[ C(RESULT_MISS) ] = PM_BR_MPRED_CMPL,
	389	},
	390	[ C(OP_WRITE) ] = {
	391	[ C(RESULT_ACCESS) ] = -1,
	392	[ C(RESULT_MISS) ] = -1,
	393	},
	394	[ C(OP_PREFETCH) ] = {
	395	[ C(RESULT_ACCESS) ] = -1,
	396	[ C(RESULT_MISS) ] = -1,
	397	},
	398	},
	399	[ C(NODE) ] = {
	400	[ C(OP_READ) ] = {
	401	[ C(RESULT_ACCESS) ] = -1,
	402	[ C(RESULT_MISS) ] = -1,
	403	},
	404	[ C(OP_WRITE) ] = {
	405	[ C(RESULT_ACCESS) ] = -1,
	406	[ C(RESULT_MISS) ] = -1,
	407	},
	408	[ C(OP_PREFETCH) ] = {
	409	[ C(RESULT_ACCESS) ] = -1,
	410	[ C(RESULT_MISS) ] = -1,
	411	},
	412	},
	413	};
	414
	415	#undef C
	416
18201b20 MS	417	static struct power_pmu power9_pmu = {
	418	.name = "POWER9",
	419	.n_counter = MAX_PMU_COUNTERS,
	420	.add_fields = ISA207_ADD_FIELDS,
8c218578	421	.test_adder = ISA207_TEST_ADDER,
59029136 MS	422	.group_constraint_mask = CNST_CACHE_PMC4_MASK,
59029136 MS	423	.group_constraint_val = CNST_CACHE_PMC4_VAL,
18201b20 MS	424	.compute_mmcr = isa207_compute_mmcr,
	425	.config_bhrb = power9_config_bhrb,
	426	.bhrb_filter_map = power9_bhrb_filter_map,
	427	.get_constraint = isa207_get_constraint,
a114aca5	428	.get_alternatives = power9_get_alternatives,
d148c94c MS	429	.get_mem_data_src = isa207_get_mem_data_src,
d148c94c MS	430	.get_mem_weight = isa207_get_mem_weight,
18201b20 MS	431	.disable_pmc = isa207_disable_pmc,
	432	.flags = PPMU_HAS_SIER \| PPMU_ARCH_207S,
	433	.n_generic = ARRAY_SIZE(power9_generic_events),
	434	.generic_events = power9_generic_events,
	435	.cache_events = &power9_cache_events,
	436	.attr_groups = power9_pmu_attr_groups,
	437	.bhrb_nr = 32,
	438	};
	439
8c002dbd MS	440	static int __init init_power9_pmu(void)
8c002dbd MS	441	{
520ed5b0	442	int rc = 0;
64acab4e	443	unsigned int pvr = mfspr(SPRN_PVR);
8c002dbd MS	444
	445	/* Comes from cpu_specs[] */
	446	if (!cur_cpu_spec->oprofile_cpu_type \|\|
	447	strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power9"))
	448	return -ENODEV;
	449
64acab4e MS	450	/* Blacklist events */
	451	if (!(pvr & PVR_POWER9_CUMULUS)) {
	452	if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 1)) {
	453	power9_pmu.blacklist_ev = p9_dd21_bl_ev;
	454	power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd21_bl_ev);
ac96588d MS	455	} else if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 2)) {
	456	power9_pmu.blacklist_ev = p9_dd22_bl_ev;
	457	power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd22_bl_ev);
64acab4e MS	458	}
	459	}
	460
2bf1071a	461	rc = register_power_pmu(&power9_pmu);
8c002dbd MS	462	if (rc)
	463	return rc;
	464
	465	/* Tell userspace that EBB is supported */
	466	cur_cpu_spec->cpu_user_features2 \|= PPC_FEATURE2_EBB;
	467
	468	return 0;
	469	}
	470	early_initcall(init_power9_pmu);