[linux-2.6-block.git] / arch / x86 / kernel / cpu / perfctr-watchdog.c

// SPDX-License-Identifier: GPL-2.0
/*
 * local apic based NMI watchdog for various CPUs.
 *
 * This file also handles reservation of performance counters for coordination
 * with other users.
 *
 * Note that these events normally don't tick when the CPU idles. This means
 * the frequency varies with CPU load.
 *
 * Original code for K7/P6 written by Keith Owens
 *
 */

#include <linux/percpu.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/smp.h>
#include <asm/nmi.h>
#include <linux/kprobes.h>

#include <asm/apic.h>
#include <asm/perf_event.h>

/*
 * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
 * offset from MSR_P4_BSU_ESCR0.
 *
 * It will be the max for all platforms (for now)
 */
#define NMI_MAX_COUNTER_BITS 66

/*
 * perfctr_nmi_owner tracks the ownership of the perfctr registers:
 * evtsel_nmi_owner tracks the ownership of the event selection
 * - different performance counters/ event selection may be reserved for
 *   different subsystems this reservation system just tries to coordinate
 *   things a little
 */
static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);

/* converts an msr to an appropriate reservation bit */
static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
{
	/* returns the bit offset of the performance counter register */
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_HYGON:
	case X86_VENDOR_AMD:
		if (msr >= MSR_F15H_PERF_CTR)
			return (msr - MSR_F15H_PERF_CTR) >> 1;
		return msr - MSR_K7_PERFCTR0;
	case X86_VENDOR_INTEL:
		if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
			return msr - MSR_ARCH_PERFMON_PERFCTR0;

		switch (boot_cpu_data.x86) {
		case 6:
			return msr - MSR_P6_PERFCTR0;
		case 11:
			return msr - MSR_KNC_PERFCTR0;
		case 15:
			return msr - MSR_P4_BPU_PERFCTR0;
		}
		break;
	case X86_VENDOR_ZHAOXIN:
	case X86_VENDOR_CENTAUR:
		return msr - MSR_ARCH_PERFMON_PERFCTR0;
	}
	return 0;
}

/*
 * converts an msr to an appropriate reservation bit
 * returns the bit offset of the event selection register
 */
static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
{
	/* returns the bit offset of the event selection register */
	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_HYGON:
	case X86_VENDOR_AMD:
		if (msr >= MSR_F15H_PERF_CTL)
			return (msr - MSR_F15H_PERF_CTL) >> 1;
		return msr - MSR_K7_EVNTSEL0;
	case X86_VENDOR_INTEL:
		if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
			return msr - MSR_ARCH_PERFMON_EVENTSEL0;

		switch (boot_cpu_data.x86) {
		case 6:
			return msr - MSR_P6_EVNTSEL0;
		case 11:
			return msr - MSR_KNC_EVNTSEL0;
		case 15:
			return msr - MSR_P4_BSU_ESCR0;
		}
		break;
	case X86_VENDOR_ZHAOXIN:
	case X86_VENDOR_CENTAUR:
		return msr - MSR_ARCH_PERFMON_EVENTSEL0;
	}
	return 0;

}

int reserve_perfctr_nmi(unsigned int msr)
{
	unsigned int counter;

	counter = nmi_perfctr_msr_to_bit(msr);
	/* register not managed by the allocator? */
	if (counter > NMI_MAX_COUNTER_BITS)
		return 1;

	if (!test_and_set_bit(counter, perfctr_nmi_owner))
		return 1;
	return 0;
}
EXPORT_SYMBOL(reserve_perfctr_nmi);

void release_perfctr_nmi(unsigned int msr)
{
	unsigned int counter;

	counter = nmi_perfctr_msr_to_bit(msr);
	/* register not managed by the allocator? */
	if (counter > NMI_MAX_COUNTER_BITS)
		return;

	clear_bit(counter, perfctr_nmi_owner);
}
EXPORT_SYMBOL(release_perfctr_nmi);

int reserve_evntsel_nmi(unsigned int msr)
{
	unsigned int counter;

	counter = nmi_evntsel_msr_to_bit(msr);
	/* register not managed by the allocator? */
	if (counter > NMI_MAX_COUNTER_BITS)
		return 1;

	if (!test_and_set_bit(counter, evntsel_nmi_owner))
		return 1;
	return 0;
}
EXPORT_SYMBOL(reserve_evntsel_nmi);

void release_evntsel_nmi(unsigned int msr)
{
	unsigned int counter;

	counter = nmi_evntsel_msr_to_bit(msr);
	/* register not managed by the allocator? */
	if (counter > NMI_MAX_COUNTER_BITS)
		return;

	clear_bit(counter, evntsel_nmi_owner);
}
EXPORT_SYMBOL(release_evntsel_nmi);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
47a486cc CG	2	/*
	3	* local apic based NMI watchdog for various CPUs.
	4	*
	5	* This file also handles reservation of performance counters for coordination
a6a0683b	6	* with other users.
47a486cc CG	7	*
	8	* Note that these events normally don't tick when the CPU idles. This means
	9	* the frequency varies with CPU load.
	10	*
	11	* Original code for K7/P6 written by Keith Owens
	12	*
	13	*/
09198e68 AK	14
09198e68 AK	15	#include <linux/percpu.h>
186f4360	16	#include <linux/export.h>
09198e68 AK	17	#include <linux/kernel.h>
	18	#include <linux/bitops.h>
	19	#include <linux/smp.h>
4a7863cc	20	#include <asm/nmi.h>
8b1fa1d7 IM	21	#include <linux/kprobes.h>
8b1fa1d7 IM	22
09198e68	23	#include <asm/apic.h>
cdd6c482	24	#include <asm/perf_event.h>
09198e68	25
47a486cc CG	26	/*
	27	* this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
	28	* offset from MSR_P4_BSU_ESCR0.
	29	*
	30	* It will be the max for all platforms (for now)
09198e68 AK	31	*/
	32	#define NMI_MAX_COUNTER_BITS 66
	33
47a486cc CG	34	/*
47a486cc CG	35	* perfctr_nmi_owner tracks the ownership of the perfctr registers:
09198e68 AK	36	* evtsel_nmi_owner tracks the ownership of the event selection
	37	* - different performance counters/ event selection may be reserved for
	38	* different subsystems this reservation system just tries to coordinate
	39	* things a little
	40	*/
	41	static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
	42	static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
	43
09198e68 AK	44	/* converts an msr to an appropriate reservation bit */
	45	static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
	46	{
5dcccd8d AK	47	/* returns the bit offset of the performance counter register */
5dcccd8d AK	48	switch (boot_cpu_data.x86_vendor) {
6d0ef316	49	case X86_VENDOR_HYGON:
5dcccd8d	50	case X86_VENDOR_AMD:
69d8e1e8 RR	51	if (msr >= MSR_F15H_PERF_CTR)
69d8e1e8 RR	52	return (msr - MSR_F15H_PERF_CTR) >> 1;
8bdbd962	53	return msr - MSR_K7_PERFCTR0;
5dcccd8d AK	54	case X86_VENDOR_INTEL:
5dcccd8d AK	55	if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
8bdbd962	56	return msr - MSR_ARCH_PERFMON_PERFCTR0;
5dcccd8d AK	57
	58	switch (boot_cpu_data.x86) {
	59	case 6:
8bdbd962	60	return msr - MSR_P6_PERFCTR0;
e717bf4e VW	61	case 11:
e717bf4e VW	62	return msr - MSR_KNC_PERFCTR0;
5dcccd8d	63	case 15:
8bdbd962	64	return msr - MSR_P4_BPU_PERFCTR0;
5dcccd8d	65	}
a8383dfb	66	break;
3a4ac121 C	67	case X86_VENDOR_ZHAOXIN:
	68	case X86_VENDOR_CENTAUR:
	69	return msr - MSR_ARCH_PERFMON_PERFCTR0;
5dcccd8d AK	70	}
5dcccd8d AK	71	return 0;
09198e68 AK	72	}
09198e68 AK	73
47a486cc CG	74	/*
	75	* converts an msr to an appropriate reservation bit
	76	* returns the bit offset of the event selection register
	77	*/
09198e68 AK	78	static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
09198e68 AK	79	{
5dcccd8d AK	80	/* returns the bit offset of the event selection register */
5dcccd8d AK	81	switch (boot_cpu_data.x86_vendor) {
6d0ef316	82	case X86_VENDOR_HYGON:
5dcccd8d	83	case X86_VENDOR_AMD:
69d8e1e8 RR	84	if (msr >= MSR_F15H_PERF_CTL)
69d8e1e8 RR	85	return (msr - MSR_F15H_PERF_CTL) >> 1;
8bdbd962	86	return msr - MSR_K7_EVNTSEL0;
5dcccd8d AK	87	case X86_VENDOR_INTEL:
5dcccd8d AK	88	if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
8bdbd962	89	return msr - MSR_ARCH_PERFMON_EVENTSEL0;
5dcccd8d AK	90
	91	switch (boot_cpu_data.x86) {
	92	case 6:
8bdbd962	93	return msr - MSR_P6_EVNTSEL0;
e717bf4e VW	94	case 11:
e717bf4e VW	95	return msr - MSR_KNC_EVNTSEL0;
5dcccd8d	96	case 15:
8bdbd962	97	return msr - MSR_P4_BSU_ESCR0;
5dcccd8d	98	}
a8383dfb	99	break;
3a4ac121 C	100	case X86_VENDOR_ZHAOXIN:
	101	case X86_VENDOR_CENTAUR:
	102	return msr - MSR_ARCH_PERFMON_EVENTSEL0;
5dcccd8d AK	103	}
	104	return 0;
	105
09198e68 AK	106	}
09198e68 AK	107
09198e68 AK	108	int reserve_perfctr_nmi(unsigned int msr)
	109	{
	110	unsigned int counter;
	111
	112	counter = nmi_perfctr_msr_to_bit(msr);
124d395f SE	113	/* register not managed by the allocator? */
	114	if (counter > NMI_MAX_COUNTER_BITS)
	115	return 1;
09198e68 AK	116
	117	if (!test_and_set_bit(counter, perfctr_nmi_owner))
	118	return 1;
	119	return 0;
	120	}
47a486cc	121	EXPORT_SYMBOL(reserve_perfctr_nmi);
09198e68 AK	122
	123	void release_perfctr_nmi(unsigned int msr)
	124	{
	125	unsigned int counter;
	126
	127	counter = nmi_perfctr_msr_to_bit(msr);
124d395f SE	128	/* register not managed by the allocator? */
	129	if (counter > NMI_MAX_COUNTER_BITS)
	130	return;
09198e68 AK	131
	132	clear_bit(counter, perfctr_nmi_owner);
	133	}
47a486cc	134	EXPORT_SYMBOL(release_perfctr_nmi);
09198e68 AK	135
	136	int reserve_evntsel_nmi(unsigned int msr)
	137	{
	138	unsigned int counter;
	139
	140	counter = nmi_evntsel_msr_to_bit(msr);
124d395f SE	141	/* register not managed by the allocator? */
	142	if (counter > NMI_MAX_COUNTER_BITS)
	143	return 1;
09198e68 AK	144
	145	if (!test_and_set_bit(counter, evntsel_nmi_owner))
	146	return 1;
	147	return 0;
	148	}
47a486cc	149	EXPORT_SYMBOL(reserve_evntsel_nmi);
09198e68 AK	150
	151	void release_evntsel_nmi(unsigned int msr)
	152	{
	153	unsigned int counter;
	154
	155	counter = nmi_evntsel_msr_to_bit(msr);
124d395f SE	156	/* register not managed by the allocator? */
	157	if (counter > NMI_MAX_COUNTER_BITS)
	158	return;
09198e68 AK	159
	160	clear_bit(counter, evntsel_nmi_owner);
	161	}
09198e68	162	EXPORT_SYMBOL(release_evntsel_nmi);