[linux-2.6-block.git] / arch / x86 / xen / pmu.c

#include <linux/types.h>
#include <linux/interrupt.h>

#include <asm/xen/hypercall.h>
#include <xen/page.h>
#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>
#include <xen/interface/xenpmu.h>

#include "xen-ops.h"
#include "pmu.h"

/* x86_pmu.handle_irq definition */
#include "../events/perf_event.h"

#define XENPMU_IRQ_PROCESSING    1
struct xenpmu {
	/* Shared page between hypervisor and domain */
	struct xen_pmu_data *xenpmu_data;

	uint8_t flags;
};
static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
#define get_xenpmu_data()    (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
#define get_xenpmu_flags()   (this_cpu_ptr(&xenpmu_shared)->flags)

/* Macro for computing address of a PMU MSR bank */
#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
					    (uintptr_t)ctxt->field))

/* AMD PMU */
#define F15H_NUM_COUNTERS   6
#define F10H_NUM_COUNTERS   4

static __read_mostly uint32_t amd_counters_base;
static __read_mostly uint32_t amd_ctrls_base;
static __read_mostly int amd_msr_step;
static __read_mostly int k7_counters_mirrored;
static __read_mostly int amd_num_counters;

/* Intel PMU */
#define MSR_TYPE_COUNTER            0
#define MSR_TYPE_CTRL               1
#define MSR_TYPE_GLOBAL             2
#define MSR_TYPE_ARCH_COUNTER       3
#define MSR_TYPE_ARCH_CTRL          4

/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
#define PMU_GENERAL_NR_SHIFT        8
#define PMU_GENERAL_NR_BITS         8
#define PMU_GENERAL_NR_MASK         (((1 << PMU_GENERAL_NR_BITS) - 1) \
				     << PMU_GENERAL_NR_SHIFT)

/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
#define PMU_FIXED_NR_SHIFT          0
#define PMU_FIXED_NR_BITS           5
#define PMU_FIXED_NR_MASK           (((1 << PMU_FIXED_NR_BITS) - 1) \
				     << PMU_FIXED_NR_SHIFT)

/* Alias registers (0x4c1) for full-width writes to PMCs */
#define MSR_PMC_ALIAS_MASK          (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))

#define INTEL_PMC_TYPE_SHIFT        30

static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;


static void xen_pmu_arch_init(void)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {

		switch (boot_cpu_data.x86) {
		case 0x15:
			amd_num_counters = F15H_NUM_COUNTERS;
			amd_counters_base = MSR_F15H_PERF_CTR;
			amd_ctrls_base = MSR_F15H_PERF_CTL;
			amd_msr_step = 2;
			k7_counters_mirrored = 1;
			break;
		case 0x10:
		case 0x12:
		case 0x14:
		case 0x16:
		default:
			amd_num_counters = F10H_NUM_COUNTERS;
			amd_counters_base = MSR_K7_PERFCTR0;
			amd_ctrls_base = MSR_K7_EVNTSEL0;
			amd_msr_step = 1;
			k7_counters_mirrored = 0;
			break;
		}
	} else {
		uint32_t eax, ebx, ecx, edx;

		cpuid(0xa, &eax, &ebx, &ecx, &edx);

		intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
			PMU_GENERAL_NR_SHIFT;
		intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
			PMU_FIXED_NR_SHIFT;
	}
}

static inline uint32_t get_fam15h_addr(u32 addr)
{
	switch (addr) {
	case MSR_K7_PERFCTR0:
	case MSR_K7_PERFCTR1:
	case MSR_K7_PERFCTR2:
	case MSR_K7_PERFCTR3:
		return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
	case MSR_K7_EVNTSEL0:
	case MSR_K7_EVNTSEL1:
	case MSR_K7_EVNTSEL2:
	case MSR_K7_EVNTSEL3:
		return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
	default:
		break;
	}

	return addr;
}

static inline bool is_amd_pmu_msr(unsigned int msr)
{
	if ((msr >= MSR_F15H_PERF_CTL &&
	     msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
	    (msr >= MSR_K7_EVNTSEL0 &&
	     msr < MSR_K7_PERFCTR0 + amd_num_counters))
		return true;

	return false;
}

static int is_intel_pmu_msr(u32 msr_index, int *type, int *index)
{
	u32 msr_index_pmc;

	switch (msr_index) {
	case MSR_CORE_PERF_FIXED_CTR_CTRL:
	case MSR_IA32_DS_AREA:
	case MSR_IA32_PEBS_ENABLE:
		*type = MSR_TYPE_CTRL;
		return true;

	case MSR_CORE_PERF_GLOBAL_CTRL:
	case MSR_CORE_PERF_GLOBAL_STATUS:
	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
		*type = MSR_TYPE_GLOBAL;
		return true;

	default:

		if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
		    (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
				 intel_num_fixed_counters)) {
			*index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
			*type = MSR_TYPE_COUNTER;
			return true;
		}

		if ((msr_index >= MSR_P6_EVNTSEL0) &&
		    (msr_index < MSR_P6_EVNTSEL0 +  intel_num_arch_counters)) {
			*index = msr_index - MSR_P6_EVNTSEL0;
			*type = MSR_TYPE_ARCH_CTRL;
			return true;
		}

		msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
		if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
		    (msr_index_pmc < MSR_IA32_PERFCTR0 +
				     intel_num_arch_counters)) {
			*type = MSR_TYPE_ARCH_COUNTER;
			*index = msr_index_pmc - MSR_IA32_PERFCTR0;
			return true;
		}
		return false;
	}
}

static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
				  int index, bool is_read)
{
	uint64_t *reg = NULL;
	struct xen_pmu_intel_ctxt *ctxt;
	uint64_t *fix_counters;
	struct xen_pmu_cntr_pair *arch_cntr_pair;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	uint8_t xenpmu_flags = get_xenpmu_flags();


	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
		return false;

	ctxt = &xenpmu_data->pmu.c.intel;

	switch (msr) {
	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
		reg = &ctxt->global_ovf_ctrl;
		break;
	case MSR_CORE_PERF_GLOBAL_STATUS:
		reg = &ctxt->global_status;
		break;
	case MSR_CORE_PERF_GLOBAL_CTRL:
		reg = &ctxt->global_ctrl;
		break;
	case MSR_CORE_PERF_FIXED_CTR_CTRL:
		reg = &ctxt->fixed_ctrl;
		break;
	default:
		switch (type) {
		case MSR_TYPE_COUNTER:
			fix_counters = field_offset(ctxt, fixed_counters);
			reg = &fix_counters[index];
			break;
		case MSR_TYPE_ARCH_COUNTER:
			arch_cntr_pair = field_offset(ctxt, arch_counters);
			reg = &arch_cntr_pair[index].counter;
			break;
		case MSR_TYPE_ARCH_CTRL:
			arch_cntr_pair = field_offset(ctxt, arch_counters);
			reg = &arch_cntr_pair[index].control;
			break;
		default:
			return false;
		}
	}

	if (reg) {
		if (is_read)
			*val = *reg;
		else {
			*reg = *val;

			if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
				ctxt->global_status &= (~(*val));
		}
		return true;
	}

	return false;
}

static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
{
	uint64_t *reg = NULL;
	int i, off = 0;
	struct xen_pmu_amd_ctxt *ctxt;
	uint64_t *counter_regs, *ctrl_regs;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	uint8_t xenpmu_flags = get_xenpmu_flags();

	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
		return false;

	if (k7_counters_mirrored &&
	    ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
		msr = get_fam15h_addr(msr);

	ctxt = &xenpmu_data->pmu.c.amd;
	for (i = 0; i < amd_num_counters; i++) {
		if (msr == amd_ctrls_base + off) {
			ctrl_regs = field_offset(ctxt, ctrls);
			reg = &ctrl_regs[i];
			break;
		} else if (msr == amd_counters_base + off) {
			counter_regs = field_offset(ctxt, counters);
			reg = &counter_regs[i];
			break;
		}
		off += amd_msr_step;
	}

	if (reg) {
		if (is_read)
			*val = *reg;
		else
			*reg = *val;

		return true;
	}
	return false;
}

bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
		if (is_amd_pmu_msr(msr)) {
			if (!xen_amd_pmu_emulate(msr, val, 1))
				*val = native_read_msr_safe(msr, err);
			return true;
		}
	} else {
		int type, index;

		if (is_intel_pmu_msr(msr, &type, &index)) {
			if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
				*val = native_read_msr_safe(msr, err);
			return true;
		}
	}

	return false;
}

bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
{
	uint64_t val = ((uint64_t)high << 32) | low;

	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
		if (is_amd_pmu_msr(msr)) {
			if (!xen_amd_pmu_emulate(msr, &val, 0))
				*err = native_write_msr_safe(msr, low, high);
			return true;
		}
	} else {
		int type, index;

		if (is_intel_pmu_msr(msr, &type, &index)) {
			if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
				*err = native_write_msr_safe(msr, low, high);
			return true;
		}
	}

	return false;
}

static unsigned long long xen_amd_read_pmc(int counter)
{
	struct xen_pmu_amd_ctxt *ctxt;
	uint64_t *counter_regs;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	uint8_t xenpmu_flags = get_xenpmu_flags();

	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
		uint32_t msr;
		int err;

		msr = amd_counters_base + (counter * amd_msr_step);
		return native_read_msr_safe(msr, &err);
	}

	ctxt = &xenpmu_data->pmu.c.amd;
	counter_regs = field_offset(ctxt, counters);
	return counter_regs[counter];
}

static unsigned long long xen_intel_read_pmc(int counter)
{
	struct xen_pmu_intel_ctxt *ctxt;
	uint64_t *fixed_counters;
	struct xen_pmu_cntr_pair *arch_cntr_pair;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	uint8_t xenpmu_flags = get_xenpmu_flags();

	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
		uint32_t msr;
		int err;

		if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
			msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
		else
			msr = MSR_IA32_PERFCTR0 + counter;

		return native_read_msr_safe(msr, &err);
	}

	ctxt = &xenpmu_data->pmu.c.intel;
	if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
		fixed_counters = field_offset(ctxt, fixed_counters);
		return fixed_counters[counter & 0xffff];
	}

	arch_cntr_pair = field_offset(ctxt, arch_counters);
	return arch_cntr_pair[counter].counter;
}

unsigned long long xen_read_pmc(int counter)
{
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
		return xen_amd_read_pmc(counter);
	else
		return xen_intel_read_pmc(counter);
}

int pmu_apic_update(uint32_t val)
{
	int ret;
	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return -EINVAL;
	}

	xenpmu_data->pmu.l.lapic_lvtpc = val;

	if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
		return 0;

	ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);

	return ret;
}

/* perf callbacks */
static int xen_is_in_guest(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
		return 0;

	return 1;
}

static int xen_is_user_mode(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
		return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
	else
		return !!(xenpmu_data->pmu.r.regs.cpl & 3);
}

static unsigned long xen_get_guest_ip(void)
{
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return 0;
	}

	return xenpmu_data->pmu.r.regs.ip;
}

static struct perf_guest_info_callbacks xen_guest_cbs = {
	.is_in_guest            = xen_is_in_guest,
	.is_user_mode           = xen_is_user_mode,
	.get_guest_ip           = xen_get_guest_ip,
};

/* Convert registers from Xen's format to Linux' */
static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
			     struct pt_regs *regs, uint64_t pmu_flags)
{
	regs->ip = xen_regs->ip;
	regs->cs = xen_regs->cs;
	regs->sp = xen_regs->sp;

	if (pmu_flags & PMU_SAMPLE_PV) {
		if (pmu_flags & PMU_SAMPLE_USER)
			regs->cs |= 3;
		else
			regs->cs &= ~3;
	} else {
		if (xen_regs->cpl)
			regs->cs |= 3;
		else
			regs->cs &= ~3;
	}
}

irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
{
	int err, ret = IRQ_NONE;
	struct pt_regs regs;
	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	uint8_t xenpmu_flags = get_xenpmu_flags();

	if (!xenpmu_data) {
		pr_warn_once("%s: pmudata not initialized\n", __func__);
		return ret;
	}

	this_cpu_ptr(&xenpmu_shared)->flags =
		xenpmu_flags | XENPMU_IRQ_PROCESSING;
	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
			 xenpmu_data->pmu.pmu_flags);
	if (x86_pmu.handle_irq(&regs))
		ret = IRQ_HANDLED;

	/* Write out cached context to HW */
	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
	this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
	if (err) {
		pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
		return IRQ_NONE;
	}

	return ret;
}

bool is_xen_pmu(int cpu)
{
	return (get_xenpmu_data() != NULL);
}

void xen_pmu_init(int cpu)
{
	int err;
	struct xen_pmu_params xp;
	unsigned long pfn;
	struct xen_pmu_data *xenpmu_data;

	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);

	if (xen_hvm_domain())
		return;

	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
	if (!xenpmu_data) {
		pr_err("VPMU init: No memory\n");
		return;
	}
	pfn = virt_to_pfn(xenpmu_data);

	xp.val = pfn_to_mfn(pfn);
	xp.vcpu = cpu;
	xp.version.maj = XENPMU_VER_MAJ;
	xp.version.min = XENPMU_VER_MIN;
	err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
	if (err)
		goto fail;

	per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
	per_cpu(xenpmu_shared, cpu).flags = 0;

	if (cpu == 0) {
		perf_register_guest_info_callbacks(&xen_guest_cbs);
		xen_pmu_arch_init();
	}

	return;

fail:
	pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n",
		cpu, err);
	free_pages((unsigned long)xenpmu_data, 0);
}

void xen_pmu_finish(int cpu)
{
	struct xen_pmu_params xp;

	if (xen_hvm_domain())
		return;

	xp.vcpu = cpu;
	xp.version.maj = XENPMU_VER_MAJ;
	xp.version.min = XENPMU_VER_MIN;

	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);

	free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
	per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
}
Commit	Line	Data
65d0cf0b BO	1	#include <linux/types.h>
	2	#include <linux/interrupt.h>
	3
	4	#include <asm/xen/hypercall.h>
	5	#include <xen/page.h>
	6	#include <xen/interface/xen.h>
	7	#include <xen/interface/vcpu.h>
	8	#include <xen/interface/xenpmu.h>
	9
	10	#include "xen-ops.h"
	11	#include "pmu.h"
	12
	13	/* x86_pmu.handle_irq definition */
27f6d22b	14	#include "../events/perf_event.h"
65d0cf0b	15
bf6dfb15 BO	16	#define XENPMU_IRQ_PROCESSING 1
	17	struct xenpmu {
	18	/* Shared page between hypervisor and domain */
	19	struct xen_pmu_data *xenpmu_data;
65d0cf0b	20
bf6dfb15 BO	21	uint8_t flags;
	22	};
	23	static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
	24	#define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
	25	#define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags)
65d0cf0b	26
bf6dfb15 BO	27	/* Macro for computing address of a PMU MSR bank */
	28	#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
	29	(uintptr_t)ctxt->field))
e27b72df BO	30
	31	/* AMD PMU */
	32	#define F15H_NUM_COUNTERS 6
	33	#define F10H_NUM_COUNTERS 4
	34
	35	static __read_mostly uint32_t amd_counters_base;
	36	static __read_mostly uint32_t amd_ctrls_base;
	37	static __read_mostly int amd_msr_step;
	38	static __read_mostly int k7_counters_mirrored;
	39	static __read_mostly int amd_num_counters;
	40
	41	/* Intel PMU */
	42	#define MSR_TYPE_COUNTER 0
	43	#define MSR_TYPE_CTRL 1
	44	#define MSR_TYPE_GLOBAL 2
	45	#define MSR_TYPE_ARCH_COUNTER 3
	46	#define MSR_TYPE_ARCH_CTRL 4
	47
	48	/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
	49	#define PMU_GENERAL_NR_SHIFT 8
	50	#define PMU_GENERAL_NR_BITS 8
	51	#define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \
	52	<< PMU_GENERAL_NR_SHIFT)
	53
	54	/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
	55	#define PMU_FIXED_NR_SHIFT 0
	56	#define PMU_FIXED_NR_BITS 5
	57	#define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \
	58	<< PMU_FIXED_NR_SHIFT)
	59
	60	/* Alias registers (0x4c1) for full-width writes to PMCs */
	61	#define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
	62
6b08cd63 BO	63	#define INTEL_PMC_TYPE_SHIFT 30
6b08cd63 BO	64
e27b72df BO	65	static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
	66
	67
	68	static void xen_pmu_arch_init(void)
	69	{
	70	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
	71
	72	switch (boot_cpu_data.x86) {
	73	case 0x15:
	74	amd_num_counters = F15H_NUM_COUNTERS;
	75	amd_counters_base = MSR_F15H_PERF_CTR;
	76	amd_ctrls_base = MSR_F15H_PERF_CTL;
	77	amd_msr_step = 2;
	78	k7_counters_mirrored = 1;
	79	break;
	80	case 0x10:
	81	case 0x12:
	82	case 0x14:
	83	case 0x16:
	84	default:
	85	amd_num_counters = F10H_NUM_COUNTERS;
	86	amd_counters_base = MSR_K7_PERFCTR0;
	87	amd_ctrls_base = MSR_K7_EVNTSEL0;
	88	amd_msr_step = 1;
	89	k7_counters_mirrored = 0;
	90	break;
	91	}
	92	} else {
	93	uint32_t eax, ebx, ecx, edx;
	94
	95	cpuid(0xa, &eax, &ebx, &ecx, &edx);
	96
	97	intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
	98	PMU_GENERAL_NR_SHIFT;
	99	intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
	100	PMU_FIXED_NR_SHIFT;
	101	}
	102	}
	103
	104	static inline uint32_t get_fam15h_addr(u32 addr)
	105	{
	106	switch (addr) {
	107	case MSR_K7_PERFCTR0:
	108	case MSR_K7_PERFCTR1:
	109	case MSR_K7_PERFCTR2:
	110	case MSR_K7_PERFCTR3:
	111	return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
	112	case MSR_K7_EVNTSEL0:
	113	case MSR_K7_EVNTSEL1:
	114	case MSR_K7_EVNTSEL2:
	115	case MSR_K7_EVNTSEL3:
	116	return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
	117	default:
	118	break;
	119	}
	120
	121	return addr;
	122	}
	123
	124	static inline bool is_amd_pmu_msr(unsigned int msr)
	125	{
	126	if ((msr >= MSR_F15H_PERF_CTL &&
	127	msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) \|\|
	128	(msr >= MSR_K7_EVNTSEL0 &&
129	msr < MSR_K7_PERFCTR0 + amd_num_counters))
130	return true;
131
132	return false;
133	}
134
135	static int is_intel_pmu_msr(u32 msr_index, int type, int index)
136	{
137	u32 msr_index_pmc;
138
139	switch (msr_index) {
140	case MSR_CORE_PERF_FIXED_CTR_CTRL:
141	case MSR_IA32_DS_AREA:
142	case MSR_IA32_PEBS_ENABLE:
143	*type = MSR_TYPE_CTRL;
144	return true;
145
146	case MSR_CORE_PERF_GLOBAL_CTRL:
147	case MSR_CORE_PERF_GLOBAL_STATUS:
148	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
149	*type = MSR_TYPE_GLOBAL;
150	return true;
151
152	default:
153
154	if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
155	(msr_index < MSR_CORE_PERF_FIXED_CTR0 +
156	intel_num_fixed_counters)) {
157	*index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
158	*type = MSR_TYPE_COUNTER;
159	return true;
160	}
161
162	if ((msr_index >= MSR_P6_EVNTSEL0) &&
163	(msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) {
164	*index = msr_index - MSR_P6_EVNTSEL0;
165	*type = MSR_TYPE_ARCH_CTRL;
166	return true;
167	}
168
169	msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
170	if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
171	(msr_index_pmc < MSR_IA32_PERFCTR0 +
172	intel_num_arch_counters)) {
173	*type = MSR_TYPE_ARCH_COUNTER;
174	*index = msr_index_pmc - MSR_IA32_PERFCTR0;
175	return true;
176	}
177	return false;
178	}
179	}
180
bf6dfb15 BO	181	static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
bf6dfb15 BO	182	int index, bool is_read)
6b08cd63	183	{
bf6dfb15 BO	184	uint64_t *reg = NULL;
	185	struct xen_pmu_intel_ctxt *ctxt;
	186	uint64_t *fix_counters;
	187	struct xen_pmu_cntr_pair *arch_cntr_pair;
	188	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	189	uint8_t xenpmu_flags = get_xenpmu_flags();
	190
6b08cd63	191
bf6dfb15 BO	192	if (!xenpmu_data \|\| !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
	193	return false;
	194
	195	ctxt = &xenpmu_data->pmu.c.intel;
	196
	197	switch (msr) {
	198	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
	199	reg = &ctxt->global_ovf_ctrl;
	200	break;
	201	case MSR_CORE_PERF_GLOBAL_STATUS:
	202	reg = &ctxt->global_status;
	203	break;
	204	case MSR_CORE_PERF_GLOBAL_CTRL:
	205	reg = &ctxt->global_ctrl;
	206	break;
	207	case MSR_CORE_PERF_FIXED_CTR_CTRL:
	208	reg = &ctxt->fixed_ctrl;
	209	break;
	210	default:
	211	switch (type) {
	212	case MSR_TYPE_COUNTER:
	213	fix_counters = field_offset(ctxt, fixed_counters);
	214	reg = &fix_counters[index];
	215	break;
	216	case MSR_TYPE_ARCH_COUNTER:
	217	arch_cntr_pair = field_offset(ctxt, arch_counters);
	218	reg = &arch_cntr_pair[index].counter;
	219	break;
	220	case MSR_TYPE_ARCH_CTRL:
	221	arch_cntr_pair = field_offset(ctxt, arch_counters);
	222	reg = &arch_cntr_pair[index].control;
	223	break;
	224	default:
	225	return false;
	226	}
	227	}
	228
	229	if (reg) {
	230	if (is_read)
	231	val = reg;
	232	else {
	233	reg = val;
	234
	235	if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
	236	ctxt->global_status &= (~(*val));
	237	}
	238	return true;
	239	}
	240
	241	return false;
	242	}
	243
	244	static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
	245	{
	246	uint64_t *reg = NULL;
	247	int i, off = 0;
	248	struct xen_pmu_amd_ctxt *ctxt;
	249	uint64_t counter_regs, ctrl_regs;
	250	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	251	uint8_t xenpmu_flags = get_xenpmu_flags();
	252
	253	if (!xenpmu_data \|\| !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
	254	return false;
	255
256	if (k7_counters_mirrored &&
257	((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
258	msr = get_fam15h_addr(msr);
259
260	ctxt = &xenpmu_data->pmu.c.amd;
261	for (i = 0; i < amd_num_counters; i++) {
262	if (msr == amd_ctrls_base + off) {
263	ctrl_regs = field_offset(ctxt, ctrls);
264	reg = &ctrl_regs[i];
265	break;
266	} else if (msr == amd_counters_base + off) {
267	counter_regs = field_offset(ctxt, counters);
268	reg = &counter_regs[i];
269	break;
270	}
271	off += amd_msr_step;
272	}
273
274	if (reg) {
275	if (is_read)
276	val = reg;
277	else
278	reg = val;
279
280	return true;
281	}
282	return false;
283	}
284
285	bool pmu_msr_read(unsigned int msr, uint64_t val, int err)
286	{
6b08cd63 BO	287	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
6b08cd63 BO	288	if (is_amd_pmu_msr(msr)) {
bf6dfb15 BO	289	if (!xen_amd_pmu_emulate(msr, val, 1))
bf6dfb15 BO	290	*val = native_read_msr_safe(msr, err);
6b08cd63 BO	291	return true;
	292	}
	293	} else {
	294	int type, index;
	295
	296	if (is_intel_pmu_msr(msr, &type, &index)) {
bf6dfb15 BO	297	if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
bf6dfb15 BO	298	*val = native_read_msr_safe(msr, err);
6b08cd63 BO	299	return true;
	300	}
	301	}
	302
	303	return false;
	304	}
	305
	306	bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
	307	{
bf6dfb15 BO	308	uint64_t val = ((uint64_t)high << 32) \| low;
bf6dfb15 BO	309
6b08cd63 BO	310	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
6b08cd63 BO	311	if (is_amd_pmu_msr(msr)) {
bf6dfb15 BO	312	if (!xen_amd_pmu_emulate(msr, &val, 0))
bf6dfb15 BO	313	*err = native_write_msr_safe(msr, low, high);
6b08cd63 BO	314	return true;
	315	}
	316	} else {
	317	int type, index;
	318
	319	if (is_intel_pmu_msr(msr, &type, &index)) {
bf6dfb15 BO	320	if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
bf6dfb15 BO	321	*err = native_write_msr_safe(msr, low, high);
6b08cd63 BO	322	return true;
	323	}
	324	}
	325
	326	return false;
	327	}
	328
	329	static unsigned long long xen_amd_read_pmc(int counter)
	330	{
bf6dfb15 BO	331	struct xen_pmu_amd_ctxt *ctxt;
	332	uint64_t *counter_regs;
	333	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	334	uint8_t xenpmu_flags = get_xenpmu_flags();
	335
	336	if (!xenpmu_data \|\| !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
	337	uint32_t msr;
	338	int err;
6b08cd63	339
bf6dfb15 BO	340	msr = amd_counters_base + (counter * amd_msr_step);
	341	return native_read_msr_safe(msr, &err);
	342	}
	343
	344	ctxt = &xenpmu_data->pmu.c.amd;
	345	counter_regs = field_offset(ctxt, counters);
	346	return counter_regs[counter];
6b08cd63 BO	347	}
	348
	349	static unsigned long long xen_intel_read_pmc(int counter)
	350	{
bf6dfb15 BO	351	struct xen_pmu_intel_ctxt *ctxt;
	352	uint64_t *fixed_counters;
	353	struct xen_pmu_cntr_pair *arch_cntr_pair;
	354	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	355	uint8_t xenpmu_flags = get_xenpmu_flags();
6b08cd63	356
bf6dfb15 BO	357	if (!xenpmu_data \|\| !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
	358	uint32_t msr;
	359	int err;
6b08cd63	360
bf6dfb15 BO	361	if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
	362	msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
	363	else
	364	msr = MSR_IA32_PERFCTR0 + counter;
	365
	366	return native_read_msr_safe(msr, &err);
	367	}
	368
	369	ctxt = &xenpmu_data->pmu.c.intel;
	370	if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
	371	fixed_counters = field_offset(ctxt, fixed_counters);
	372	return fixed_counters[counter & 0xffff];
	373	}
	374
	375	arch_cntr_pair = field_offset(ctxt, arch_counters);
	376	return arch_cntr_pair[counter].counter;
6b08cd63 BO	377	}
	378
	379	unsigned long long xen_read_pmc(int counter)
	380	{
	381	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
	382	return xen_amd_read_pmc(counter);
	383	else
	384	return xen_intel_read_pmc(counter);
	385	}
	386
	387	int pmu_apic_update(uint32_t val)
	388	{
	389	int ret;
	390	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	391
	392	if (!xenpmu_data) {
	393	pr_warn_once("%s: pmudata not initialized\n", __func__);
	394	return -EINVAL;
	395	}
	396
	397	xenpmu_data->pmu.l.lapic_lvtpc = val;
bf6dfb15 BO	398
	399	if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
	400	return 0;
	401
6b08cd63 BO	402	ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
	403
	404	return ret;
	405	}
	406
65d0cf0b BO	407	/* perf callbacks */
	408	static int xen_is_in_guest(void)
	409	{
	410	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	411
	412	if (!xenpmu_data) {
	413	pr_warn_once("%s: pmudata not initialized\n", __func__);
	414	return 0;
	415	}
	416
	417	if (!xen_initial_domain() \|\| (xenpmu_data->domain_id >= DOMID_SELF))
	418	return 0;
	419
	420	return 1;
	421	}
	422
	423	static int xen_is_user_mode(void)
	424	{
	425	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	426
	427	if (!xenpmu_data) {
	428	pr_warn_once("%s: pmudata not initialized\n", __func__);
	429	return 0;
	430	}
	431
	432	if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
	433	return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
	434	else
	435	return !!(xenpmu_data->pmu.r.regs.cpl & 3);
	436	}
	437
	438	static unsigned long xen_get_guest_ip(void)
	439	{
	440	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
	441
	442	if (!xenpmu_data) {
	443	pr_warn_once("%s: pmudata not initialized\n", __func__);
	444	return 0;
	445	}
	446
	447	return xenpmu_data->pmu.r.regs.ip;
	448	}
	449
	450	static struct perf_guest_info_callbacks xen_guest_cbs = {
	451	.is_in_guest = xen_is_in_guest,
	452	.is_user_mode = xen_is_user_mode,
	453	.get_guest_ip = xen_get_guest_ip,
	454	};
	455
	456	/* Convert registers from Xen's format to Linux' */
	457	static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
	458	struct pt_regs *regs, uint64_t pmu_flags)
	459	{
	460	regs->ip = xen_regs->ip;
	461	regs->cs = xen_regs->cs;
	462	regs->sp = xen_regs->sp;
	463
	464	if (pmu_flags & PMU_SAMPLE_PV) {
	465	if (pmu_flags & PMU_SAMPLE_USER)
	466	regs->cs \|= 3;
	467	else
	468	regs->cs &= ~3;
	469	} else {
	470	if (xen_regs->cpl)
471	regs->cs \|= 3;
472	else
473	regs->cs &= ~3;
474	}
475	}
476
477	irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
478	{
6b08cd63	479	int err, ret = IRQ_NONE;
65d0cf0b BO	480	struct pt_regs regs;
65d0cf0b BO	481	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
bf6dfb15	482	uint8_t xenpmu_flags = get_xenpmu_flags();
65d0cf0b BO	483
	484	if (!xenpmu_data) {
	485	pr_warn_once("%s: pmudata not initialized\n", __func__);
	486	return ret;
	487	}
	488
bf6dfb15 BO	489	this_cpu_ptr(&xenpmu_shared)->flags =
bf6dfb15 BO	490	xenpmu_flags \| XENPMU_IRQ_PROCESSING;
65d0cf0b BO	491	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
	492	xenpmu_data->pmu.pmu_flags);
	493	if (x86_pmu.handle_irq(&regs))
	494	ret = IRQ_HANDLED;
	495
bf6dfb15 BO	496	/* Write out cached context to HW */
	497	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
	498	this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
	499	if (err) {
	500	pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
	501	return IRQ_NONE;
	502	}
	503
65d0cf0b BO	504	return ret;
	505	}
	506
	507	bool is_xen_pmu(int cpu)
	508	{
bf6dfb15	509	return (get_xenpmu_data() != NULL);
65d0cf0b BO	510	}
	511
	512	void xen_pmu_init(int cpu)
	513	{
	514	int err;
	515	struct xen_pmu_params xp;
	516	unsigned long pfn;
	517	struct xen_pmu_data *xenpmu_data;
	518
	519	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
	520
	521	if (xen_hvm_domain())
	522	return;
	523
	524	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
	525	if (!xenpmu_data) {
	526	pr_err("VPMU init: No memory\n");
	527	return;
	528	}
	529	pfn = virt_to_pfn(xenpmu_data);
	530
	531	xp.val = pfn_to_mfn(pfn);
	532	xp.vcpu = cpu;
	533	xp.version.maj = XENPMU_VER_MAJ;
	534	xp.version.min = XENPMU_VER_MIN;
	535	err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
	536	if (err)
	537	goto fail;
	538
bf6dfb15 BO	539	per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
bf6dfb15 BO	540	per_cpu(xenpmu_shared, cpu).flags = 0;
65d0cf0b	541
e27b72df	542	if (cpu == 0) {
65d0cf0b	543	perf_register_guest_info_callbacks(&xen_guest_cbs);
e27b72df BO	544	xen_pmu_arch_init();
e27b72df BO	545	}
65d0cf0b BO	546
	547	return;
	548
	549	fail:
	550	pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n",
	551	cpu, err);
	552	free_pages((unsigned long)xenpmu_data, 0);
	553	}
	554
	555	void xen_pmu_finish(int cpu)
	556	{
	557	struct xen_pmu_params xp;
	558
	559	if (xen_hvm_domain())
	560	return;
	561
	562	xp.vcpu = cpu;
	563	xp.version.maj = XENPMU_VER_MAJ;
	564	xp.version.min = XENPMU_VER_MIN;
	565
	566	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
	567
bf6dfb15 BO	568	free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
bf6dfb15 BO	569	per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
65d0cf0b	570	}