[linux-2.6-block.git] / arch / x86 / events / amd / uncore.c

/*
 * Copyright (C) 2013 Advanced Micro Devices, Inc.
 *
 * Author: Jacob Shin <jacob.shin@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/perf_event.h>
#include <linux/percpu.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>

#include <asm/cpufeature.h>
#include <asm/perf_event.h>
#include <asm/msr.h>

#define NUM_COUNTERS_NB		4
#define NUM_COUNTERS_L2		4
#define MAX_COUNTERS		NUM_COUNTERS_NB

#define RDPMC_BASE_NB		6
#define RDPMC_BASE_L2		10

#define COUNTER_SHIFT		16

struct amd_uncore {
	int id;
	int refcnt;
	int cpu;
	int num_counters;
	int rdpmc_base;
	u32 msr_base;
	cpumask_t *active_mask;
	struct pmu *pmu;
	struct perf_event *events[MAX_COUNTERS];
	struct amd_uncore *free_when_cpu_online;
};

static struct amd_uncore * __percpu *amd_uncore_nb;
static struct amd_uncore * __percpu *amd_uncore_l2;

static struct pmu amd_nb_pmu;
static struct pmu amd_l2_pmu;

static cpumask_t amd_nb_active_mask;
static cpumask_t amd_l2_active_mask;

static bool is_nb_event(struct perf_event *event)
{
	return event->pmu->type == amd_nb_pmu.type;
}

static bool is_l2_event(struct perf_event *event)
{
	return event->pmu->type == amd_l2_pmu.type;
}

static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
{
	if (is_nb_event(event) && amd_uncore_nb)
		return *per_cpu_ptr(amd_uncore_nb, event->cpu);
	else if (is_l2_event(event) && amd_uncore_l2)
		return *per_cpu_ptr(amd_uncore_l2, event->cpu);

	return NULL;
}

static void amd_uncore_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev, new;
	s64 delta;

	/*
	 * since we do not enable counter overflow interrupts,
	 * we do not have to worry about prev_count changing on us
	 */

	prev = local64_read(&hwc->prev_count);
	rdpmcl(hwc->event_base_rdpmc, new);
	local64_set(&hwc->prev_count, new);
	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
	delta >>= COUNTER_SHIFT;
	local64_add(delta, &event->count);
}

static void amd_uncore_start(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	if (flags & PERF_EF_RELOAD)
		wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));

	hwc->state = 0;
	wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
	perf_event_update_userpage(event);
}

static void amd_uncore_stop(struct perf_event *event, int flags)
{
	struct hw_perf_event *hwc = &event->hw;

	wrmsrl(hwc->config_base, hwc->config);
	hwc->state |= PERF_HES_STOPPED;

	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
		amd_uncore_read(event);
		hwc->state |= PERF_HES_UPTODATE;
	}
}

static int amd_uncore_add(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	/* are we already assigned? */
	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
		goto out;

	for (i = 0; i < uncore->num_counters; i++) {
		if (uncore->events[i] == event) {
			hwc->idx = i;
			goto out;
		}
	}

	/* if not, take the first available counter */
	hwc->idx = -1;
	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
			hwc->idx = i;
			break;
		}
	}

out:
	if (hwc->idx == -1)
		return -EBUSY;

	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;

	if (flags & PERF_EF_START)
		amd_uncore_start(event, PERF_EF_RELOAD);

	return 0;
}

static void amd_uncore_del(struct perf_event *event, int flags)
{
	int i;
	struct amd_uncore *uncore = event_to_amd_uncore(event);
	struct hw_perf_event *hwc = &event->hw;

	amd_uncore_stop(event, PERF_EF_UPDATE);

	for (i = 0; i < uncore->num_counters; i++) {
		if (cmpxchg(&uncore->events[i], event, NULL) == event)
			break;
	}

	hwc->idx = -1;
}

static int amd_uncore_event_init(struct perf_event *event)
{
	struct amd_uncore *uncore;
	struct hw_perf_event *hwc = &event->hw;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/*
	 * NB and L2 counters (MSRs) are shared across all cores that share the
	 * same NB / L2 cache. Interrupts can be directed to a single target
	 * core, however, event counts generated by processes running on other
	 * cores cannot be masked out. So we do not support sampling and
	 * per-thread events.
	 */
	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
		return -EINVAL;

	/* NB and L2 counters do not have usr/os/guest/host bits */
	if (event->attr.exclude_user || event->attr.exclude_kernel ||
	    event->attr.exclude_host || event->attr.exclude_guest)
		return -EINVAL;

	/* and we do not enable counter overflow interrupts */
	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
	hwc->idx = -1;

	if (event->cpu < 0)
		return -EINVAL;

	uncore = event_to_amd_uncore(event);
	if (!uncore)
		return -ENODEV;

	/*
	 * since request can come in to any of the shared cores, we will remap
	 * to a single common cpu.
	 */
	event->cpu = uncore->cpu;

	return 0;
}

static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
					    struct device_attribute *attr,
					    char *buf)
{
	cpumask_t *active_mask;
	struct pmu *pmu = dev_get_drvdata(dev);

	if (pmu->type == amd_nb_pmu.type)
		active_mask = &amd_nb_active_mask;
	else if (pmu->type == amd_l2_pmu.type)
		active_mask = &amd_l2_active_mask;
	else
		return 0;

	return cpumap_print_to_pagebuf(true, buf, active_mask);
}
static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);

static struct attribute *amd_uncore_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

static struct attribute_group amd_uncore_attr_group = {
	.attrs = amd_uncore_attrs,
};

PMU_FORMAT_ATTR(event, "config:0-7,32-35");
PMU_FORMAT_ATTR(umask, "config:8-15");

static struct attribute *amd_uncore_format_attr[] = {
	&format_attr_event.attr,
	&format_attr_umask.attr,
	NULL,
};

static struct attribute_group amd_uncore_format_group = {
	.name = "format",
	.attrs = amd_uncore_format_attr,
};

static const struct attribute_group *amd_uncore_attr_groups[] = {
	&amd_uncore_attr_group,
	&amd_uncore_format_group,
	NULL,
};

static struct pmu amd_nb_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_nb",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct pmu amd_l2_pmu = {
	.task_ctx_nr	= perf_invalid_context,
	.attr_groups	= amd_uncore_attr_groups,
	.name		= "amd_l2",
	.event_init	= amd_uncore_event_init,
	.add		= amd_uncore_add,
	.del		= amd_uncore_del,
	.start		= amd_uncore_start,
	.stop		= amd_uncore_stop,
	.read		= amd_uncore_read,
};

static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
{
	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
			cpu_to_node(cpu));
}

static int amd_uncore_cpu_up_prepare(unsigned int cpu)
{
	struct amd_uncore *uncore_nb = NULL, *uncore_l2;

	if (amd_uncore_nb) {
		uncore_nb = amd_uncore_alloc(cpu);
		if (!uncore_nb)
			goto fail;
		uncore_nb->cpu = cpu;
		uncore_nb->num_counters = NUM_COUNTERS_NB;
		uncore_nb->rdpmc_base = RDPMC_BASE_NB;
		uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
		uncore_nb->active_mask = &amd_nb_active_mask;
		uncore_nb->pmu = &amd_nb_pmu;
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
	}

	if (amd_uncore_l2) {
		uncore_l2 = amd_uncore_alloc(cpu);
		if (!uncore_l2)
			goto fail;
		uncore_l2->cpu = cpu;
		uncore_l2->num_counters = NUM_COUNTERS_L2;
		uncore_l2->rdpmc_base = RDPMC_BASE_L2;
		uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
		uncore_l2->active_mask = &amd_l2_active_mask;
		uncore_l2->pmu = &amd_l2_pmu;
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
	}

	return 0;

fail:
	if (amd_uncore_nb)
		*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
	kfree(uncore_nb);
	return -ENOMEM;
}

static struct amd_uncore *
amd_uncore_find_online_sibling(struct amd_uncore *this,
			       struct amd_uncore * __percpu *uncores)
{
	unsigned int cpu;
	struct amd_uncore *that;

	for_each_online_cpu(cpu) {
		that = *per_cpu_ptr(uncores, cpu);

		if (!that)
			continue;

		if (this == that)
			continue;

		if (this->id == that->id) {
			that->free_when_cpu_online = this;
			this = that;
			break;
		}
	}

	this->refcnt++;
	return this;
}

static void amd_uncore_cpu_starting(unsigned int cpu)
{
	unsigned int eax, ebx, ecx, edx;
	struct amd_uncore *uncore;

	if (amd_uncore_nb) {
		uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
		cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
		uncore->id = ecx & 0xff;

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
		*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	}

	if (amd_uncore_l2) {
		unsigned int apicid = cpu_data(cpu).apicid;
		unsigned int nshared;

		uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
		cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
		nshared = ((eax >> 14) & 0xfff) + 1;
		uncore->id = apicid - (apicid % nshared);

		uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
		*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	}
}

static void uncore_online(unsigned int cpu,
			  struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	kfree(uncore->free_when_cpu_online);
	uncore->free_when_cpu_online = NULL;

	if (cpu == uncore->cpu)
		cpumask_set_cpu(cpu, uncore->active_mask);
}

static void amd_uncore_cpu_online(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_online(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_online(cpu, amd_uncore_l2);
}

static void uncore_down_prepare(unsigned int cpu,
				struct amd_uncore * __percpu *uncores)
{
	unsigned int i;
	struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);

	if (this->cpu != cpu)
		return;

	/* this cpu is going down, migrate to a shared sibling if possible */
	for_each_online_cpu(i) {
		struct amd_uncore *that = *per_cpu_ptr(uncores, i);

		if (cpu == i)
			continue;

		if (this == that) {
			perf_pmu_migrate_context(this->pmu, cpu, i);
			cpumask_clear_cpu(cpu, that->active_mask);
			cpumask_set_cpu(i, that->active_mask);
			that->cpu = i;
			break;
		}
	}
}

static void amd_uncore_cpu_down_prepare(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_down_prepare(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_down_prepare(cpu, amd_uncore_l2);
}

static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
{
	struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);

	if (cpu == uncore->cpu)
		cpumask_clear_cpu(cpu, uncore->active_mask);

	if (!--uncore->refcnt)
		kfree(uncore);
	*per_cpu_ptr(uncores, cpu) = NULL;
}

static void amd_uncore_cpu_dead(unsigned int cpu)
{
	if (amd_uncore_nb)
		uncore_dead(cpu, amd_uncore_nb);

	if (amd_uncore_l2)
		uncore_dead(cpu, amd_uncore_l2);
}

static int
amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action,
			void *hcpu)
{
	unsigned int cpu = (long)hcpu;

	switch (action & ~CPU_TASKS_FROZEN) {
	case CPU_UP_PREPARE:
		if (amd_uncore_cpu_up_prepare(cpu))
			return notifier_from_errno(-ENOMEM);
		break;

	case CPU_STARTING:
		amd_uncore_cpu_starting(cpu);
		break;

	case CPU_ONLINE:
		amd_uncore_cpu_online(cpu);
		break;

	case CPU_DOWN_PREPARE:
		amd_uncore_cpu_down_prepare(cpu);
		break;

	case CPU_UP_CANCELED:
	case CPU_DEAD:
		amd_uncore_cpu_dead(cpu);
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

static struct notifier_block amd_uncore_cpu_notifier_block = {
	.notifier_call	= amd_uncore_cpu_notifier,
	.priority	= CPU_PRI_PERF + 1,
};

static void __init init_cpu_already_online(void *dummy)
{
	unsigned int cpu = smp_processor_id();

	amd_uncore_cpu_starting(cpu);
	amd_uncore_cpu_online(cpu);
}

static void cleanup_cpu_online(void *dummy)
{
	unsigned int cpu = smp_processor_id();

	amd_uncore_cpu_dead(cpu);
}

static int __init amd_uncore_init(void)
{
	unsigned int cpu, cpu2;
	int ret = -ENODEV;

	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
		goto fail_nodev;

	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
		goto fail_nodev;

	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
		amd_uncore_nb = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_nb) {
			ret = -ENOMEM;
			goto fail_nb;
		}
		ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
		if (ret)
			goto fail_nb;

		pr_info("perf: AMD NB counters detected\n");
		ret = 0;
	}

	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
		amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
		if (!amd_uncore_l2) {
			ret = -ENOMEM;
			goto fail_l2;
		}
		ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
		if (ret)
			goto fail_l2;

		pr_info("perf: AMD L2I counters detected\n");
		ret = 0;
	}

	if (ret)
		goto fail_nodev;

	cpu_notifier_register_begin();

	/* init cpus already online before registering for hotplug notifier */
	for_each_online_cpu(cpu) {
		ret = amd_uncore_cpu_up_prepare(cpu);
		if (ret)
			goto fail_online;
		smp_call_function_single(cpu, init_cpu_already_online, NULL, 1);
	}

	__register_cpu_notifier(&amd_uncore_cpu_notifier_block);
	cpu_notifier_register_done();

	return 0;


fail_online:
	for_each_online_cpu(cpu2) {
		if (cpu2 == cpu)
			break;
		smp_call_function_single(cpu, cleanup_cpu_online, NULL, 1);
	}
	cpu_notifier_register_done();

	/* amd_uncore_nb/l2 should have been freed by cleanup_cpu_online */
	amd_uncore_nb = amd_uncore_l2 = NULL;

	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2))
		perf_pmu_unregister(&amd_l2_pmu);
fail_l2:
	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
		perf_pmu_unregister(&amd_nb_pmu);
	if (amd_uncore_l2)
		free_percpu(amd_uncore_l2);
fail_nb:
	if (amd_uncore_nb)
		free_percpu(amd_uncore_nb);

fail_nodev:
	return ret;
}
device_initcall(amd_uncore_init);
Commit	Line	Data
c43ca509 JS	1	/*
	2	* Copyright (C) 2013 Advanced Micro Devices, Inc.
	3	*
	4	* Author: Jacob Shin <jacob.shin@amd.com>
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*/
	10
	11	#include <linux/perf_event.h>
	12	#include <linux/percpu.h>
	13	#include <linux/types.h>
	14	#include <linux/slab.h>
	15	#include <linux/init.h>
	16	#include <linux/cpu.h>
	17	#include <linux/cpumask.h>
	18
	19	#include <asm/cpufeature.h>
	20	#include <asm/perf_event.h>
	21	#include <asm/msr.h>
	22
	23	#define NUM_COUNTERS_NB 4
	24	#define NUM_COUNTERS_L2 4
	25	#define MAX_COUNTERS NUM_COUNTERS_NB
	26
	27	#define RDPMC_BASE_NB 6
	28	#define RDPMC_BASE_L2 10
	29
	30	#define COUNTER_SHIFT 16
	31
	32	struct amd_uncore {
	33	int id;
	34	int refcnt;
	35	int cpu;
	36	int num_counters;
	37	int rdpmc_base;
	38	u32 msr_base;
	39	cpumask_t *active_mask;
	40	struct pmu *pmu;
	41	struct perf_event *events[MAX_COUNTERS];
	42	struct amd_uncore *free_when_cpu_online;
	43	};
	44
	45	static struct amd_uncore * __percpu *amd_uncore_nb;
	46	static struct amd_uncore * __percpu *amd_uncore_l2;
	47
	48	static struct pmu amd_nb_pmu;
	49	static struct pmu amd_l2_pmu;
	50
	51	static cpumask_t amd_nb_active_mask;
	52	static cpumask_t amd_l2_active_mask;
	53
	54	static bool is_nb_event(struct perf_event *event)
	55	{
	56	return event->pmu->type == amd_nb_pmu.type;
	57	}
	58
	59	static bool is_l2_event(struct perf_event *event)
	60	{
	61	return event->pmu->type == amd_l2_pmu.type;
	62	}
	63
	64	static struct amd_uncore event_to_amd_uncore(struct perf_event event)
65	{
66	if (is_nb_event(event) && amd_uncore_nb)
67	return *per_cpu_ptr(amd_uncore_nb, event->cpu);
68	else if (is_l2_event(event) && amd_uncore_l2)
69	return *per_cpu_ptr(amd_uncore_l2, event->cpu);
70
71	return NULL;
72	}
73
74	static void amd_uncore_read(struct perf_event *event)
75	{
76	struct hw_perf_event *hwc = &event->hw;
77	u64 prev, new;
78	s64 delta;
79
80	/*
81	* since we do not enable counter overflow interrupts,
82	* we do not have to worry about prev_count changing on us
83	*/
84
85	prev = local64_read(&hwc->prev_count);
86	rdpmcl(hwc->event_base_rdpmc, new);
87	local64_set(&hwc->prev_count, new);
88	delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
89	delta >>= COUNTER_SHIFT;
90	local64_add(delta, &event->count);
91	}
92
93	static void amd_uncore_start(struct perf_event *event, int flags)
94	{
95	struct hw_perf_event *hwc = &event->hw;
96
97	if (flags & PERF_EF_RELOAD)
98	wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
99
100	hwc->state = 0;
101	wrmsrl(hwc->config_base, (hwc->config \| ARCH_PERFMON_EVENTSEL_ENABLE));
102	perf_event_update_userpage(event);
103	}
104
105	static void amd_uncore_stop(struct perf_event *event, int flags)
106	{
107	struct hw_perf_event *hwc = &event->hw;
108
109	wrmsrl(hwc->config_base, hwc->config);
110	hwc->state \|= PERF_HES_STOPPED;
111
112	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
113	amd_uncore_read(event);
114	hwc->state \|= PERF_HES_UPTODATE;
115	}
116	}
117
118	static int amd_uncore_add(struct perf_event *event, int flags)
119	{
120	int i;
121	struct amd_uncore *uncore = event_to_amd_uncore(event);
122	struct hw_perf_event *hwc = &event->hw;
123
124	/* are we already assigned? */
125	if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
126	goto out;
127
128	for (i = 0; i < uncore->num_counters; i++) {
129	if (uncore->events[i] == event) {
130	hwc->idx = i;
131	goto out;
132	}
133	}
134
135	/* if not, take the first available counter */
136	hwc->idx = -1;
137	for (i = 0; i < uncore->num_counters; i++) {
138	if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
139	hwc->idx = i;
140	break;
141	}
142	}
143
144	out:
145	if (hwc->idx == -1)
146	return -EBUSY;
147
148	hwc->config_base = uncore->msr_base + (2 * hwc->idx);
149	hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
150	hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
151	hwc->state = PERF_HES_UPTODATE \| PERF_HES_STOPPED;
152
153	if (flags & PERF_EF_START)
154	amd_uncore_start(event, PERF_EF_RELOAD);
155
156	return 0;
157	}
158
159	static void amd_uncore_del(struct perf_event *event, int flags)
160	{
161	int i;
162	struct amd_uncore *uncore = event_to_amd_uncore(event);
163	struct hw_perf_event *hwc = &event->hw;
164
165	amd_uncore_stop(event, PERF_EF_UPDATE);
166
167	for (i = 0; i < uncore->num_counters; i++) {
168	if (cmpxchg(&uncore->events[i], event, NULL) == event)
169	break;
170	}
171
172	hwc->idx = -1;
173	}
174
175	static int amd_uncore_event_init(struct perf_event *event)
176	{
177	struct amd_uncore *uncore;
178	struct hw_perf_event *hwc = &event->hw;
179
180	if (event->attr.type != event->pmu->type)
181	return -ENOENT;
182
183	/*
184	* NB and L2 counters (MSRs) are shared across all cores that share the
185	* same NB / L2 cache. Interrupts can be directed to a single target
186	* core, however, event counts generated by processes running on other
187	* cores cannot be masked out. So we do not support sampling and
188	* per-thread events.
189	*/
190	if (is_sampling_event(event) \|\| event->attach_state & PERF_ATTACH_TASK)
191	return -EINVAL;
192
193	/* NB and L2 counters do not have usr/os/guest/host bits */
194	if (event->attr.exclude_user \|\| event->attr.exclude_kernel \|\|
195	event->attr.exclude_host \|\| event->attr.exclude_guest)
196	return -EINVAL;
197
198	/* and we do not enable counter overflow interrupts */
199	hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
200	hwc->idx = -1;
201
202	if (event->cpu < 0)
203	return -EINVAL;
204
205	uncore = event_to_amd_uncore(event);
206	if (!uncore)
207	return -ENODEV;
208
209	/*
210	* since request can come in to any of the shared cores, we will remap
211	* to a single common cpu.
212	*/
213	event->cpu = uncore->cpu;
214
215	return 0;
216	}
217
218	static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
219	struct device_attribute *attr,
220	char *buf)
221	{
c43ca509 JS	222	cpumask_t *active_mask;
	223	struct pmu *pmu = dev_get_drvdata(dev);
	224
	225	if (pmu->type == amd_nb_pmu.type)
	226	active_mask = &amd_nb_active_mask;
	227	else if (pmu->type == amd_l2_pmu.type)
	228	active_mask = &amd_l2_active_mask;
	229	else
	230	return 0;
	231
5aaba363	232	return cpumap_print_to_pagebuf(true, buf, active_mask);
c43ca509 JS	233	}
	234	static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
	235
	236	static struct attribute *amd_uncore_attrs[] = {
	237	&dev_attr_cpumask.attr,
	238	NULL,
	239	};
	240
	241	static struct attribute_group amd_uncore_attr_group = {
	242	.attrs = amd_uncore_attrs,
	243	};
	244
	245	PMU_FORMAT_ATTR(event, "config:0-7,32-35");
	246	PMU_FORMAT_ATTR(umask, "config:8-15");
	247
	248	static struct attribute *amd_uncore_format_attr[] = {
	249	&format_attr_event.attr,
	250	&format_attr_umask.attr,
	251	NULL,
	252	};
	253
	254	static struct attribute_group amd_uncore_format_group = {
	255	.name = "format",
	256	.attrs = amd_uncore_format_attr,
	257	};
	258
	259	static const struct attribute_group *amd_uncore_attr_groups[] = {
	260	&amd_uncore_attr_group,
	261	&amd_uncore_format_group,
	262	NULL,
	263	};
	264
	265	static struct pmu amd_nb_pmu = {
31d50c55	266	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	267	.attr_groups = amd_uncore_attr_groups,
	268	.name = "amd_nb",
	269	.event_init = amd_uncore_event_init,
	270	.add = amd_uncore_add,
	271	.del = amd_uncore_del,
	272	.start = amd_uncore_start,
	273	.stop = amd_uncore_stop,
	274	.read = amd_uncore_read,
	275	};
	276
	277	static struct pmu amd_l2_pmu = {
31d50c55	278	.task_ctx_nr = perf_invalid_context,
c43ca509 JS	279	.attr_groups = amd_uncore_attr_groups,
	280	.name = "amd_l2",
	281	.event_init = amd_uncore_event_init,
	282	.add = amd_uncore_add,
	283	.del = amd_uncore_del,
	284	.start = amd_uncore_start,
	285	.stop = amd_uncore_stop,
	286	.read = amd_uncore_read,
	287	};
	288
148f9bb8	289	static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
c43ca509 JS	290	{
	291	return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
	292	cpu_to_node(cpu));
	293	}
	294
503d3291	295	static int amd_uncore_cpu_up_prepare(unsigned int cpu)
c43ca509	296	{
503d3291	297	struct amd_uncore uncore_nb = NULL, uncore_l2;
c43ca509 JS	298
c43ca509 JS	299	if (amd_uncore_nb) {
503d3291 ZZ	300	uncore_nb = amd_uncore_alloc(cpu);
	301	if (!uncore_nb)
	302	goto fail;
	303	uncore_nb->cpu = cpu;
	304	uncore_nb->num_counters = NUM_COUNTERS_NB;
	305	uncore_nb->rdpmc_base = RDPMC_BASE_NB;
	306	uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
	307	uncore_nb->active_mask = &amd_nb_active_mask;
	308	uncore_nb->pmu = &amd_nb_pmu;
	309	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
c43ca509 JS	310	}
	311
	312	if (amd_uncore_l2) {
503d3291 ZZ	313	uncore_l2 = amd_uncore_alloc(cpu);
	314	if (!uncore_l2)
	315	goto fail;
	316	uncore_l2->cpu = cpu;
	317	uncore_l2->num_counters = NUM_COUNTERS_L2;
	318	uncore_l2->rdpmc_base = RDPMC_BASE_L2;
	319	uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL;
	320	uncore_l2->active_mask = &amd_l2_active_mask;
	321	uncore_l2->pmu = &amd_l2_pmu;
	322	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2;
c43ca509	323	}
503d3291 ZZ	324
	325	return 0;
	326
	327	fail:
8bc9162c TG	328	if (amd_uncore_nb)
8bc9162c TG	329	*per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
503d3291 ZZ	330	kfree(uncore_nb);
503d3291 ZZ	331	return -ENOMEM;
c43ca509 JS	332	}
	333
	334	static struct amd_uncore *
148f9bb8 PG	335	amd_uncore_find_online_sibling(struct amd_uncore *this,
148f9bb8 PG	336	struct amd_uncore * __percpu *uncores)
c43ca509 JS	337	{
	338	unsigned int cpu;
	339	struct amd_uncore *that;
	340
	341	for_each_online_cpu(cpu) {
	342	that = *per_cpu_ptr(uncores, cpu);
	343
	344	if (!that)
	345	continue;
	346
	347	if (this == that)
	348	continue;
	349
	350	if (this->id == that->id) {
	351	that->free_when_cpu_online = this;
	352	this = that;
	353	break;
	354	}
	355	}
	356
	357	this->refcnt++;
	358	return this;
	359	}
	360
148f9bb8	361	static void amd_uncore_cpu_starting(unsigned int cpu)
c43ca509 JS	362	{
	363	unsigned int eax, ebx, ecx, edx;
	364	struct amd_uncore *uncore;
	365
	366	if (amd_uncore_nb) {
	367	uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
	368	cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
	369	uncore->id = ecx & 0xff;
	370
	371	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
	372	*per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
	373	}
	374
	375	if (amd_uncore_l2) {
	376	unsigned int apicid = cpu_data(cpu).apicid;
	377	unsigned int nshared;
	378
	379	uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
	380	cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
	381	nshared = ((eax >> 14) & 0xfff) + 1;
	382	uncore->id = apicid - (apicid % nshared);
	383
	384	uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
	385	*per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
	386	}
	387	}
	388
148f9bb8 PG	389	static void uncore_online(unsigned int cpu,
148f9bb8 PG	390	struct amd_uncore * __percpu *uncores)
c43ca509 JS	391	{
	392	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	393
	394	kfree(uncore->free_when_cpu_online);
	395	uncore->free_when_cpu_online = NULL;
	396
	397	if (cpu == uncore->cpu)
	398	cpumask_set_cpu(cpu, uncore->active_mask);
	399	}
	400
148f9bb8	401	static void amd_uncore_cpu_online(unsigned int cpu)
c43ca509 JS	402	{
	403	if (amd_uncore_nb)
	404	uncore_online(cpu, amd_uncore_nb);
	405
	406	if (amd_uncore_l2)
	407	uncore_online(cpu, amd_uncore_l2);
	408	}
	409
148f9bb8 PG	410	static void uncore_down_prepare(unsigned int cpu,
148f9bb8 PG	411	struct amd_uncore * __percpu *uncores)
c43ca509 JS	412	{
	413	unsigned int i;
	414	struct amd_uncore this = per_cpu_ptr(uncores, cpu);
	415
	416	if (this->cpu != cpu)
	417	return;
	418
	419	/* this cpu is going down, migrate to a shared sibling if possible */
	420	for_each_online_cpu(i) {
	421	struct amd_uncore that = per_cpu_ptr(uncores, i);
	422
	423	if (cpu == i)
	424	continue;
	425
	426	if (this == that) {
	427	perf_pmu_migrate_context(this->pmu, cpu, i);
	428	cpumask_clear_cpu(cpu, that->active_mask);
	429	cpumask_set_cpu(i, that->active_mask);
	430	that->cpu = i;
	431	break;
	432	}
	433	}
	434	}
	435
148f9bb8	436	static void amd_uncore_cpu_down_prepare(unsigned int cpu)
c43ca509 JS	437	{
	438	if (amd_uncore_nb)
	439	uncore_down_prepare(cpu, amd_uncore_nb);
	440
	441	if (amd_uncore_l2)
	442	uncore_down_prepare(cpu, amd_uncore_l2);
	443	}
	444
148f9bb8	445	static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
c43ca509 JS	446	{
	447	struct amd_uncore uncore = per_cpu_ptr(uncores, cpu);
	448
	449	if (cpu == uncore->cpu)
	450	cpumask_clear_cpu(cpu, uncore->active_mask);
	451
	452	if (!--uncore->refcnt)
	453	kfree(uncore);
503d3291	454	*per_cpu_ptr(uncores, cpu) = NULL;
c43ca509 JS	455	}
c43ca509 JS	456
148f9bb8	457	static void amd_uncore_cpu_dead(unsigned int cpu)
c43ca509 JS	458	{
	459	if (amd_uncore_nb)
	460	uncore_dead(cpu, amd_uncore_nb);
	461
	462	if (amd_uncore_l2)
	463	uncore_dead(cpu, amd_uncore_l2);
	464	}
	465
148f9bb8	466	static int
c43ca509 JS	467	amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action,
	468	void *hcpu)
	469	{
	470	unsigned int cpu = (long)hcpu;
	471
	472	switch (action & ~CPU_TASKS_FROZEN) {
	473	case CPU_UP_PREPARE:
503d3291 ZZ	474	if (amd_uncore_cpu_up_prepare(cpu))
503d3291 ZZ	475	return notifier_from_errno(-ENOMEM);
c43ca509 JS	476	break;
	477
	478	case CPU_STARTING:
	479	amd_uncore_cpu_starting(cpu);
	480	break;
	481
	482	case CPU_ONLINE:
	483	amd_uncore_cpu_online(cpu);
	484	break;
	485
	486	case CPU_DOWN_PREPARE:
	487	amd_uncore_cpu_down_prepare(cpu);
	488	break;
	489
	490	case CPU_UP_CANCELED:
	491	case CPU_DEAD:
	492	amd_uncore_cpu_dead(cpu);
	493	break;
	494
	495	default:
	496	break;
	497	}
	498
	499	return NOTIFY_OK;
	500	}
	501
148f9bb8	502	static struct notifier_block amd_uncore_cpu_notifier_block = {
c43ca509 JS	503	.notifier_call = amd_uncore_cpu_notifier,
	504	.priority = CPU_PRI_PERF + 1,
	505	};
	506
	507	static void __init init_cpu_already_online(void *dummy)
	508	{
	509	unsigned int cpu = smp_processor_id();
	510
c43ca509 JS	511	amd_uncore_cpu_starting(cpu);
	512	amd_uncore_cpu_online(cpu);
	513	}
	514
503d3291 ZZ	515	static void cleanup_cpu_online(void *dummy)
	516	{
	517	unsigned int cpu = smp_processor_id();
	518
	519	amd_uncore_cpu_dead(cpu);
	520	}
	521
c43ca509 JS	522	static int __init amd_uncore_init(void)
c43ca509 JS	523	{
503d3291	524	unsigned int cpu, cpu2;
c43ca509 JS	525	int ret = -ENODEV;
	526
	527	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
503d3291	528	goto fail_nodev;
c43ca509	529
362f924b	530	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
503d3291	531	goto fail_nodev;
c43ca509	532
362f924b	533	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
c43ca509	534	amd_uncore_nb = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	535	if (!amd_uncore_nb) {
	536	ret = -ENOMEM;
	537	goto fail_nb;
	538	}
	539	ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
	540	if (ret)
	541	goto fail_nb;
c43ca509	542
1b74dde7	543	pr_info("perf: AMD NB counters detected\n");
c43ca509 JS	544	ret = 0;
	545	}
	546
362f924b	547	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) {
c43ca509	548	amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
503d3291 ZZ	549	if (!amd_uncore_l2) {
	550	ret = -ENOMEM;
	551	goto fail_l2;
	552	}
	553	ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
	554	if (ret)
	555	goto fail_l2;
c43ca509	556
1b74dde7	557	pr_info("perf: AMD L2I counters detected\n");
c43ca509 JS	558	ret = 0;
	559	}
	560
	561	if (ret)
503d3291	562	goto fail_nodev;
c43ca509	563
a8c17c29 SB	564	cpu_notifier_register_begin();
a8c17c29 SB	565
c43ca509	566	/* init cpus already online before registering for hotplug notifier */
94f4db35	567	for_each_online_cpu(cpu) {
503d3291 ZZ	568	ret = amd_uncore_cpu_up_prepare(cpu);
	569	if (ret)
	570	goto fail_online;
c43ca509	571	smp_call_function_single(cpu, init_cpu_already_online, NULL, 1);
94f4db35	572	}
c43ca509	573
a8c17c29 SB	574	__register_cpu_notifier(&amd_uncore_cpu_notifier_block);
a8c17c29 SB	575	cpu_notifier_register_done();
c43ca509 JS	576
c43ca509 JS	577	return 0;
503d3291 ZZ	578
	579
	580	fail_online:
	581	for_each_online_cpu(cpu2) {
	582	if (cpu2 == cpu)
	583	break;
	584	smp_call_function_single(cpu, cleanup_cpu_online, NULL, 1);
	585	}
	586	cpu_notifier_register_done();
	587
	588	/* amd_uncore_nb/l2 should have been freed by cleanup_cpu_online */
	589	amd_uncore_nb = amd_uncore_l2 = NULL;
362f924b BP	590
362f924b BP	591	if (boot_cpu_has(X86_FEATURE_PERFCTR_L2))
503d3291 ZZ	592	perf_pmu_unregister(&amd_l2_pmu);
503d3291 ZZ	593	fail_l2:
362f924b	594	if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
503d3291 ZZ	595	perf_pmu_unregister(&amd_nb_pmu);
	596	if (amd_uncore_l2)
	597	free_percpu(amd_uncore_l2);
	598	fail_nb:
	599	if (amd_uncore_nb)
	600	free_percpu(amd_uncore_nb);
	601
	602	fail_nodev:
	603	return ret;
c43ca509 JS	604	}
c43ca509 JS	605	device_initcall(amd_uncore_init);