[linux-2.6-block.git] / arch / x86 / events / msr.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/perf_event.h>
#include <linux/nospec.h>
#include <asm/intel-family.h>

enum perf_msr_id {
	PERF_MSR_TSC			= 0,
	PERF_MSR_APERF			= 1,
	PERF_MSR_MPERF			= 2,
	PERF_MSR_PPERF			= 3,
	PERF_MSR_SMI			= 4,
	PERF_MSR_PTSC			= 5,
	PERF_MSR_IRPERF			= 6,
	PERF_MSR_THERM			= 7,
	PERF_MSR_THERM_SNAP		= 8,
	PERF_MSR_THERM_UNIT		= 9,
	PERF_MSR_EVENT_MAX,
};

static bool test_aperfmperf(int idx)
{
	return boot_cpu_has(X86_FEATURE_APERFMPERF);
}

static bool test_ptsc(int idx)
{
	return boot_cpu_has(X86_FEATURE_PTSC);
}

static bool test_irperf(int idx)
{
	return boot_cpu_has(X86_FEATURE_IRPERF);
}

static bool test_therm_status(int idx)
{
	return boot_cpu_has(X86_FEATURE_DTHERM);
}

static bool test_intel(int idx)
{
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
	    boot_cpu_data.x86 != 6)
		return false;

	switch (boot_cpu_data.x86_model) {
	case INTEL_FAM6_NEHALEM:
	case INTEL_FAM6_NEHALEM_G:
	case INTEL_FAM6_NEHALEM_EP:
	case INTEL_FAM6_NEHALEM_EX:

	case INTEL_FAM6_WESTMERE:
	case INTEL_FAM6_WESTMERE_EP:
	case INTEL_FAM6_WESTMERE_EX:

	case INTEL_FAM6_SANDYBRIDGE:
	case INTEL_FAM6_SANDYBRIDGE_X:

	case INTEL_FAM6_IVYBRIDGE:
	case INTEL_FAM6_IVYBRIDGE_X:

	case INTEL_FAM6_HASWELL_CORE:
	case INTEL_FAM6_HASWELL_X:
	case INTEL_FAM6_HASWELL_ULT:
	case INTEL_FAM6_HASWELL_GT3E:

	case INTEL_FAM6_BROADWELL_CORE:
	case INTEL_FAM6_BROADWELL_XEON_D:
	case INTEL_FAM6_BROADWELL_GT3E:
	case INTEL_FAM6_BROADWELL_X:

	case INTEL_FAM6_ATOM_SILVERMONT:
	case INTEL_FAM6_ATOM_SILVERMONT_X:
	case INTEL_FAM6_ATOM_AIRMONT:

	case INTEL_FAM6_ATOM_GOLDMONT:
	case INTEL_FAM6_ATOM_GOLDMONT_X:

	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:

	case INTEL_FAM6_XEON_PHI_KNL:
	case INTEL_FAM6_XEON_PHI_KNM:
		if (idx == PERF_MSR_SMI)
			return true;
		break;

	case INTEL_FAM6_SKYLAKE_MOBILE:
	case INTEL_FAM6_SKYLAKE_DESKTOP:
	case INTEL_FAM6_SKYLAKE_X:
	case INTEL_FAM6_KABYLAKE_MOBILE:
	case INTEL_FAM6_KABYLAKE_DESKTOP:
	case INTEL_FAM6_ICELAKE_MOBILE:
		if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF)
			return true;
		break;
	}

	return false;
}

struct perf_msr {
	u64	msr;
	struct	perf_pmu_events_attr *attr;
	bool	(*test)(int idx);
};

PMU_EVENT_ATTR_STRING(tsc,				evattr_tsc,		"event=0x00"	);
PMU_EVENT_ATTR_STRING(aperf,				evattr_aperf,		"event=0x01"	);
PMU_EVENT_ATTR_STRING(mperf,				evattr_mperf,		"event=0x02"	);
PMU_EVENT_ATTR_STRING(pperf,				evattr_pperf,		"event=0x03"	);
PMU_EVENT_ATTR_STRING(smi,				evattr_smi,		"event=0x04"	);
PMU_EVENT_ATTR_STRING(ptsc,				evattr_ptsc,		"event=0x05"	);
PMU_EVENT_ATTR_STRING(irperf,				evattr_irperf,		"event=0x06"	);
PMU_EVENT_ATTR_STRING(cpu_thermal_margin,		evattr_therm,		"event=0x07"	);
PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot,	evattr_therm_snap,	"1"		);
PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit,		evattr_therm_unit,	"C"		);

static struct perf_msr msr[] = {
	[PERF_MSR_TSC]		= { 0,				&evattr_tsc,		NULL,			},
	[PERF_MSR_APERF]	= { MSR_IA32_APERF,		&evattr_aperf,		test_aperfmperf,	},
	[PERF_MSR_MPERF]	= { MSR_IA32_MPERF,		&evattr_mperf,		test_aperfmperf,	},
	[PERF_MSR_PPERF]	= { MSR_PPERF,			&evattr_pperf,		test_intel,		},
	[PERF_MSR_SMI]		= { MSR_SMI_COUNT,		&evattr_smi,		test_intel,		},
	[PERF_MSR_PTSC]		= { MSR_F15H_PTSC,		&evattr_ptsc,		test_ptsc,		},
	[PERF_MSR_IRPERF]	= { MSR_F17H_IRPERF,		&evattr_irperf,		test_irperf,		},
	[PERF_MSR_THERM]	= { MSR_IA32_THERM_STATUS,	&evattr_therm,		test_therm_status,	},
	[PERF_MSR_THERM_SNAP]	= { MSR_IA32_THERM_STATUS,	&evattr_therm_snap,	test_therm_status,	},
	[PERF_MSR_THERM_UNIT]	= { MSR_IA32_THERM_STATUS,	&evattr_therm_unit,	test_therm_status,	},
};

static struct attribute *events_attrs[PERF_MSR_EVENT_MAX + 1] = {
	NULL,
};

static struct attribute_group events_attr_group = {
	.name = "events",
	.attrs = events_attrs,
};

PMU_FORMAT_ATTR(event, "config:0-63");
static struct attribute *format_attrs[] = {
	&format_attr_event.attr,
	NULL,
};
static struct attribute_group format_attr_group = {
	.name = "format",
	.attrs = format_attrs,
};

static const struct attribute_group *attr_groups[] = {
	&events_attr_group,
	&format_attr_group,
	NULL,
};

static int msr_event_init(struct perf_event *event)
{
	u64 cfg = event->attr.config;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/* unsupported modes and filters */
	if (event->attr.sample_period) /* no sampling */
		return -EINVAL;

	if (cfg >= PERF_MSR_EVENT_MAX)
		return -EINVAL;

	cfg = array_index_nospec((unsigned long)cfg, PERF_MSR_EVENT_MAX);

	if (!msr[cfg].attr)
		return -EINVAL;

	event->hw.idx		= -1;
	event->hw.event_base	= msr[cfg].msr;
	event->hw.config	= cfg;

	return 0;
}

static inline u64 msr_read_counter(struct perf_event *event)
{
	u64 now;

	if (event->hw.event_base)
		rdmsrl(event->hw.event_base, now);
	else
		now = rdtsc_ordered();

	return now;
}

static void msr_event_update(struct perf_event *event)
{
	u64 prev, now;
	s64 delta;

	/* Careful, an NMI might modify the previous event value: */
again:
	prev = local64_read(&event->hw.prev_count);
	now = msr_read_counter(event);

	if (local64_cmpxchg(&event->hw.prev_count, prev, now) != prev)
		goto again;

	delta = now - prev;
	if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) {
		delta = sign_extend64(delta, 31);
		local64_add(delta, &event->count);
	} else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) {
		/* If valid, extract digital readout, otherwise set to -1: */
		now = now & (1ULL << 31) ? (now >> 16) & 0x3f :  -1;
		local64_set(&event->count, now);
	} else {
		local64_add(delta, &event->count);
	}
}

static void msr_event_start(struct perf_event *event, int flags)
{
	u64 now = msr_read_counter(event);

	local64_set(&event->hw.prev_count, now);
}

static void msr_event_stop(struct perf_event *event, int flags)
{
	msr_event_update(event);
}

static void msr_event_del(struct perf_event *event, int flags)
{
	msr_event_stop(event, PERF_EF_UPDATE);
}

static int msr_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		msr_event_start(event, flags);

	return 0;
}

static struct pmu pmu_msr = {
	.task_ctx_nr	= perf_sw_context,
	.attr_groups	= attr_groups,
	.event_init	= msr_event_init,
	.add		= msr_event_add,
	.del		= msr_event_del,
	.start		= msr_event_start,
	.stop		= msr_event_stop,
	.read		= msr_event_update,
	.capabilities	= PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
};

static int __init msr_init(void)
{
	int i, j = 0;

	if (!boot_cpu_has(X86_FEATURE_TSC)) {
		pr_cont("no MSR PMU driver.\n");
		return 0;
	}

	/* Probe the MSRs. */
	for (i = PERF_MSR_TSC + 1; i < PERF_MSR_EVENT_MAX; i++) {
		u64 val;

		/* Virt sucks; you cannot tell if a R/O MSR is present :/ */
		if (!msr[i].test(i) || rdmsrl_safe(msr[i].msr, &val))
			msr[i].attr = NULL;
	}

	/* List remaining MSRs in the sysfs attrs. */
	for (i = 0; i < PERF_MSR_EVENT_MAX; i++) {
		if (msr[i].attr)
			events_attrs[j++] = &msr[i].attr->attr.attr;
	}
	events_attrs[j] = NULL;

	perf_pmu_register(&pmu_msr, "msr", -1);

	return 0;
}
device_initcall(msr_init);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
b7b7c782	2	#include <linux/perf_event.h>
06ce6e9b	3	#include <linux/nospec.h>
353bf605	4	#include <asm/intel-family.h>
b7b7c782 AL	5
	6	enum perf_msr_id {
	7	PERF_MSR_TSC = 0,
	8	PERF_MSR_APERF = 1,
	9	PERF_MSR_MPERF = 2,
	10	PERF_MSR_PPERF = 3,
	11	PERF_MSR_SMI = 4,
8a224261	12	PERF_MSR_PTSC = 5,
aaf24884	13	PERF_MSR_IRPERF = 6,
9ae21dd6 SE	14	PERF_MSR_THERM = 7,
	15	PERF_MSR_THERM_SNAP = 8,
	16	PERF_MSR_THERM_UNIT = 9,
b7b7c782 AL	17	PERF_MSR_EVENT_MAX,
	18	};
	19
7e5560a5	20	static bool test_aperfmperf(int idx)
19b3340c PZ	21	{
	22	return boot_cpu_has(X86_FEATURE_APERFMPERF);
	23	}
	24
8a224261 HR	25	static bool test_ptsc(int idx)
	26	{
	27	return boot_cpu_has(X86_FEATURE_PTSC);
	28	}
	29
aaf24884 HR	30	static bool test_irperf(int idx)
	31	{
	32	return boot_cpu_has(X86_FEATURE_IRPERF);
	33	}
	34
9ae21dd6 SE	35	static bool test_therm_status(int idx)
	36	{
	37	return boot_cpu_has(X86_FEATURE_DTHERM);
	38	}
	39
7e5560a5	40	static bool test_intel(int idx)
19b3340c PZ	41	{
	42	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL \|\|
	43	boot_cpu_data.x86 != 6)
	44	return false;
	45
	46	switch (boot_cpu_data.x86_model) {
353bf605	47	case INTEL_FAM6_NEHALEM:
b325e04e	48	case INTEL_FAM6_NEHALEM_G:
353bf605 DH	49	case INTEL_FAM6_NEHALEM_EP:
353bf605 DH	50	case INTEL_FAM6_NEHALEM_EX:
19b3340c	51
353bf605 DH	52	case INTEL_FAM6_WESTMERE:
	53	case INTEL_FAM6_WESTMERE_EP:
	54	case INTEL_FAM6_WESTMERE_EX:
19b3340c	55
353bf605 DH	56	case INTEL_FAM6_SANDYBRIDGE:
353bf605 DH	57	case INTEL_FAM6_SANDYBRIDGE_X:
19b3340c	58
353bf605 DH	59	case INTEL_FAM6_IVYBRIDGE:
353bf605 DH	60	case INTEL_FAM6_IVYBRIDGE_X:
19b3340c	61
353bf605 DH	62	case INTEL_FAM6_HASWELL_CORE:
	63	case INTEL_FAM6_HASWELL_X:
	64	case INTEL_FAM6_HASWELL_ULT:
	65	case INTEL_FAM6_HASWELL_GT3E:
19b3340c	66
353bf605 DH	67	case INTEL_FAM6_BROADWELL_CORE:
	68	case INTEL_FAM6_BROADWELL_XEON_D:
	69	case INTEL_FAM6_BROADWELL_GT3E:
	70	case INTEL_FAM6_BROADWELL_X:
19b3340c	71
f2c4db1b PZ	72	case INTEL_FAM6_ATOM_SILVERMONT:
f2c4db1b PZ	73	case INTEL_FAM6_ATOM_SILVERMONT_X:
353bf605	74	case INTEL_FAM6_ATOM_AIRMONT:
1aaccc40 KL	75
1aaccc40 KL	76	case INTEL_FAM6_ATOM_GOLDMONT:
f2c4db1b	77	case INTEL_FAM6_ATOM_GOLDMONT_X:
1aaccc40	78
f2c4db1b	79	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1aaccc40 KL	80
	81	case INTEL_FAM6_XEON_PHI_KNL:
	82	case INTEL_FAM6_XEON_PHI_KNM:
19b3340c PZ	83	if (idx == PERF_MSR_SMI)
	84	return true;
	85	break;
	86
353bf605 DH	87	case INTEL_FAM6_SKYLAKE_MOBILE:
353bf605 DH	88	case INTEL_FAM6_SKYLAKE_DESKTOP:
5134596c DH	89	case INTEL_FAM6_SKYLAKE_X:
	90	case INTEL_FAM6_KABYLAKE_MOBILE:
	91	case INTEL_FAM6_KABYLAKE_DESKTOP:
cf50d79a	92	case INTEL_FAM6_ICELAKE_MOBILE:
19b3340c PZ	93	if (idx == PERF_MSR_SMI \|\| idx == PERF_MSR_PPERF)
	94	return true;
	95	break;
	96	}
	97
	98	return false;
	99	}
	100
b7b7c782	101	struct perf_msr {
b7b7c782	102	u64 msr;
19b3340c PZ	103	struct perf_pmu_events_attr *attr;
19b3340c PZ	104	bool (*test)(int idx);
b7b7c782 AL	105	};
b7b7c782 AL	106
9128d3ed IM	107	PMU_EVENT_ATTR_STRING(tsc, evattr_tsc, "event=0x00" );
	108	PMU_EVENT_ATTR_STRING(aperf, evattr_aperf, "event=0x01" );
	109	PMU_EVENT_ATTR_STRING(mperf, evattr_mperf, "event=0x02" );
	110	PMU_EVENT_ATTR_STRING(pperf, evattr_pperf, "event=0x03" );
	111	PMU_EVENT_ATTR_STRING(smi, evattr_smi, "event=0x04" );
	112	PMU_EVENT_ATTR_STRING(ptsc, evattr_ptsc, "event=0x05" );
	113	PMU_EVENT_ATTR_STRING(irperf, evattr_irperf, "event=0x06" );
	114	PMU_EVENT_ATTR_STRING(cpu_thermal_margin, evattr_therm, "event=0x07" );
	115	PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot, evattr_therm_snap, "1" );
	116	PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit, evattr_therm_unit, "C" );
b7b7c782	117
19b3340c	118	static struct perf_msr msr[] = {
9128d3ed IM	119	[PERF_MSR_TSC] = { 0, &evattr_tsc, NULL, },
	120	[PERF_MSR_APERF] = { MSR_IA32_APERF, &evattr_aperf, test_aperfmperf, },
	121	[PERF_MSR_MPERF] = { MSR_IA32_MPERF, &evattr_mperf, test_aperfmperf, },
	122	[PERF_MSR_PPERF] = { MSR_PPERF, &evattr_pperf, test_intel, },
	123	[PERF_MSR_SMI] = { MSR_SMI_COUNT, &evattr_smi, test_intel, },
	124	[PERF_MSR_PTSC] = { MSR_F15H_PTSC, &evattr_ptsc, test_ptsc, },
	125	[PERF_MSR_IRPERF] = { MSR_F17H_IRPERF, &evattr_irperf, test_irperf, },
	126	[PERF_MSR_THERM] = { MSR_IA32_THERM_STATUS, &evattr_therm, test_therm_status, },
	127	[PERF_MSR_THERM_SNAP] = { MSR_IA32_THERM_STATUS, &evattr_therm_snap, test_therm_status, },
	128	[PERF_MSR_THERM_UNIT] = { MSR_IA32_THERM_STATUS, &evattr_therm_unit, test_therm_status, },
19b3340c PZ	129	};
19b3340c PZ	130
b7b7c782	131	static struct attribute *events_attrs[PERF_MSR_EVENT_MAX + 1] = {
19b3340c	132	NULL,
b7b7c782 AL	133	};
	134
	135	static struct attribute_group events_attr_group = {
	136	.name = "events",
	137	.attrs = events_attrs,
	138	};
	139
	140	PMU_FORMAT_ATTR(event, "config:0-63");
	141	static struct attribute *format_attrs[] = {
	142	&format_attr_event.attr,
	143	NULL,
	144	};
	145	static struct attribute_group format_attr_group = {
	146	.name = "format",
	147	.attrs = format_attrs,
	148	};
	149
	150	static const struct attribute_group *attr_groups[] = {
	151	&events_attr_group,
	152	&format_attr_group,
	153	NULL,
	154	};
	155
	156	static int msr_event_init(struct perf_event *event)
	157	{
	158	u64 cfg = event->attr.config;
	159
	160	if (event->attr.type != event->pmu->type)
	161	return -ENOENT;
	162
b7b7c782	163	/* unsupported modes and filters */
2ff40250	164	if (event->attr.sample_period) /* no sampling */
b7b7c782 AL	165	return -EINVAL;
b7b7c782 AL	166
06ce6e9b PZ	167	if (cfg >= PERF_MSR_EVENT_MAX)
	168	return -EINVAL;
	169
	170	cfg = array_index_nospec((unsigned long)cfg, PERF_MSR_EVENT_MAX);
	171
19b3340c PZ	172	if (!msr[cfg].attr)
	173	return -EINVAL;
	174
9128d3ed IM	175	event->hw.idx = -1;
	176	event->hw.event_base = msr[cfg].msr;
	177	event->hw.config = cfg;
b7b7c782 AL	178
	179	return 0;
	180	}
	181
	182	static inline u64 msr_read_counter(struct perf_event *event)
	183	{
	184	u64 now;
	185
	186	if (event->hw.event_base)
	187	rdmsrl(event->hw.event_base, now);
	188	else
ea89c065	189	now = rdtsc_ordered();
b7b7c782 AL	190
	191	return now;
	192	}
ea89c065	193
b7b7c782 AL	194	static void msr_event_update(struct perf_event *event)
	195	{
	196	u64 prev, now;
	197	s64 delta;
	198
9128d3ed	199	/* Careful, an NMI might modify the previous event value: */
b7b7c782 AL	200	again:
	201	prev = local64_read(&event->hw.prev_count);
	202	now = msr_read_counter(event);
	203
	204	if (local64_cmpxchg(&event->hw.prev_count, prev, now) != prev)
	205	goto again;
	206
	207	delta = now - prev;
9ae21dd6	208	if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) {
78e3c795	209	delta = sign_extend64(delta, 31);
9ae21dd6 SE	210	local64_add(delta, &event->count);
9ae21dd6 SE	211	} else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) {
9128d3ed	212	/* If valid, extract digital readout, otherwise set to -1: */
9ae21dd6 SE	213	now = now & (1ULL << 31) ? (now >> 16) & 0x3f : -1;
9ae21dd6 SE	214	local64_set(&event->count, now);
9128d3ed	215	} else {
9ae21dd6	216	local64_add(delta, &event->count);
9128d3ed	217	}
b7b7c782 AL	218	}
	219
	220	static void msr_event_start(struct perf_event *event, int flags)
	221	{
9128d3ed	222	u64 now = msr_read_counter(event);
b7b7c782	223
b7b7c782 AL	224	local64_set(&event->hw.prev_count, now);
	225	}
	226
	227	static void msr_event_stop(struct perf_event *event, int flags)
	228	{
	229	msr_event_update(event);
	230	}
	231
	232	static void msr_event_del(struct perf_event *event, int flags)
	233	{
	234	msr_event_stop(event, PERF_EF_UPDATE);
	235	}
	236
	237	static int msr_event_add(struct perf_event *event, int flags)
	238	{
	239	if (flags & PERF_EF_START)
	240	msr_event_start(event, flags);
	241
	242	return 0;
	243	}
	244
	245	static struct pmu pmu_msr = {
	246	.task_ctx_nr = perf_sw_context,
	247	.attr_groups = attr_groups,
	248	.event_init = msr_event_init,
	249	.add = msr_event_add,
	250	.del = msr_event_del,
	251	.start = msr_event_start,
	252	.stop = msr_event_stop,
	253	.read = msr_event_update,
2ff40250	254	.capabilities = PERF_PMU_CAP_NO_INTERRUPT \| PERF_PMU_CAP_NO_EXCLUDE,
b7b7c782 AL	255	};
b7b7c782 AL	256
b7b7c782 AL	257	static int __init msr_init(void)
b7b7c782 AL	258	{
19b3340c	259	int i, j = 0;
b7b7c782	260
19b3340c PZ	261	if (!boot_cpu_has(X86_FEATURE_TSC)) {
	262	pr_cont("no MSR PMU driver.\n");
	263	return 0;
b7b7c782 AL	264	}
b7b7c782 AL	265
19b3340c PZ	266	/* Probe the MSRs. */
	267	for (i = PERF_MSR_TSC + 1; i < PERF_MSR_EVENT_MAX; i++) {
	268	u64 val;
b7b7c782	269
9128d3ed	270	/* Virt sucks; you cannot tell if a R/O MSR is present :/ */
19b3340c PZ	271	if (!msr[i].test(i) \|\| rdmsrl_safe(msr[i].msr, &val))
19b3340c PZ	272	msr[i].attr = NULL;
b7b7c782 AL	273	}
b7b7c782 AL	274
19b3340c PZ	275	/* List remaining MSRs in the sysfs attrs. */
	276	for (i = 0; i < PERF_MSR_EVENT_MAX; i++) {
	277	if (msr[i].attr)
	278	events_attrs[j++] = &msr[i].attr->attr.attr;
b7b7c782	279	}
19b3340c	280	events_attrs[j] = NULL;
b7b7c782 AL	281
	282	perf_pmu_register(&pmu_msr, "msr", -1);
	283
	284	return 0;
	285	}
	286	device_initcall(msr_init);