projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf_event: x86: Optimize the fast path a little more
[linux-block.git] / arch / x86 / kernel / cpu / perf_event.c
CommitLineData
241771ef 1/*
cdd6c482 2 * Performance events x86 architecture code
241771ef 3 *
98144511
IM		 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
30dd568c 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
1da53e02 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
241771ef
IM		 11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
cdd6c482 15#include <linux/perf_event.h>
241771ef
IM		 16#include <linux/capability.h>
17#include <linux/notifier.h>
18#include <linux/hardirq.h>
19#include <linux/kprobes.h>
4ac13294 20#include <linux/module.h>
241771ef
IM		 21#include <linux/kdebug.h>
22#include <linux/sched.h>
d7d59fb3 23#include <linux/uaccess.h>
74193ef0 24#include <linux/highmem.h>
30dd568c 25#include <linux/cpu.h>
272d30be 26#include <linux/bitops.h>
241771ef 27
241771ef 28#include <asm/apic.h>
d7d59fb3 29#include <asm/stacktrace.h>
4e935e47 30#include <asm/nmi.h>
241771ef 31
cdd6c482 32static u64 perf_event_mask __read_mostly;
703e937c 33
cdd6c482
IM		 34/* The maximal number of PEBS events: */
35#define MAX_PEBS_EVENTS 4
30dd568c
MM		 36
37/* The size of a BTS record in bytes: */
38#define BTS_RECORD_SIZE 24
39
40/* The size of a per-cpu BTS buffer in bytes: */
5622f295 41#define BTS_BUFFER_SIZE (BTS_RECORD_SIZE * 2048)
30dd568c
MM		 42
43/* The BTS overflow threshold in bytes from the end of the buffer: */
5622f295 44#define BTS_OVFL_TH (BTS_RECORD_SIZE * 128)
30dd568c
MM		 45
46
47/*
48 * Bits in the debugctlmsr controlling branch tracing.
49 */
50#define X86_DEBUGCTL_TR (1 << 6)
51#define X86_DEBUGCTL_BTS (1 << 7)
52#define X86_DEBUGCTL_BTINT (1 << 8)
53#define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
54#define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
55
56/*
57 * A debug store configuration.
58 *
59 * We only support architectures that use 64bit fields.
60 */
61struct debug_store {
62 u64 bts_buffer_base;
63 u64 bts_index;
64 u64 bts_absolute_maximum;
65 u64 bts_interrupt_threshold;
66 u64 pebs_buffer_base;
67 u64 pebs_index;
68 u64 pebs_absolute_maximum;
69 u64 pebs_interrupt_threshold;
cdd6c482 70 u64 pebs_event_reset[MAX_PEBS_EVENTS];
30dd568c
MM		 71};
72
1da53e02 73struct event_constraint {
c91e0f5d
PZ		 74 union {
75 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
76 u64 idxmsk64[1];
77 };
1da53e02
SE		 78 int code;
79 int cmask;
272d30be 80 int weight;
1da53e02
SE		 81};
82
cdd6c482 83struct cpu_hw_events {
1da53e02 84 struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
43f6201a 85 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
4b39fd96 86 unsigned long interrupts;
b0f3f28e 87 int enabled;
30dd568c 88 struct debug_store *ds;
241771ef 89
1da53e02
SE		 90 int n_events;
91 int n_added;
92 int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
93 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
b690081d
SE		 94};
95
c91e0f5d
PZ		 96#define EVENT_CONSTRAINT(c, n, m) { \
97 { .idxmsk64[0] = (n) }, \
98 .code = (c), \
99 .cmask = (m), \
272d30be 100 .weight = HWEIGHT64((u64)(n)), \
c91e0f5d 101}
b690081d 102
8433be11
PZ		 103#define INTEL_EVENT_CONSTRAINT(c, n) \
104 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
105
106#define FIXED_EVENT_CONSTRAINT(c, n) \
107 EVENT_CONSTRAINT(c, n, INTEL_ARCH_FIXED_MASK)
108
1da53e02 109#define EVENT_CONSTRAINT_END \
c91e0f5d 110 EVENT_CONSTRAINT(0, 0, 0)
b690081d 111
1da53e02
SE		 112#define for_each_event_constraint(e, c) \
113 for ((e) = (c); (e)->cmask; (e)++)
b690081d 114
241771ef 115/*
5f4ec28f 116 * struct x86_pmu - generic x86 pmu
241771ef 117 */
5f4ec28f 118struct x86_pmu {
faa28ae0
RR		 119 const char *name;
120 int version;
a3288106 121 int (*handle_irq)(struct pt_regs *);
9e35ad38
PZ		 122 void (*disable_all)(void);
123 void (*enable_all)(void);
cdd6c482
IM		 124 void (*enable)(struct hw_perf_event *, int);
125 void (*disable)(struct hw_perf_event *, int);
169e41eb
JSR		 126 unsigned eventsel;
127 unsigned perfctr;
b0f3f28e
PZ		 128 u64 (*event_map)(int);
129 u64 (*raw_event)(u64);
169e41eb 130 int max_events;
cdd6c482
IM		 131 int num_events;
132 int num_events_fixed;
133 int event_bits;
134 u64 event_mask;
04da8a43 135 int apic;
c619b8ff 136 u64 max_period;
9e35ad38 137 u64 intel_ctrl;
30dd568c
MM		 138 void (*enable_bts)(u64 config);
139 void (*disable_bts)(void);
63b14649
PZ		 140
141 struct event_constraint *
142 (*get_event_constraints)(struct cpu_hw_events *cpuc,
143 struct perf_event *event);
144
c91e0f5d
PZ		 145 void (*put_event_constraints)(struct cpu_hw_events *cpuc,
146 struct perf_event *event);
63b14649 147 struct event_constraint *event_constraints;
b56a3802
JSR		 148};
149
4a06bd85 150static struct x86_pmu x86_pmu __read_mostly;
b56a3802 151
cdd6c482 152static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
b0f3f28e
PZ		 153 .enabled = 1,
154};
241771ef 155
1da53e02
SE		 156static int x86_perf_event_set_period(struct perf_event *event,
157 struct hw_perf_event *hwc, int idx);
b690081d 158
11d1578f
VW		 159/*
160 * Not sure about some of these
161 */
162static const u64 p6_perfmon_event_map[] =
163{
164 [PERF_COUNT_HW_CPU_CYCLES] = 0x0079,
165 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
f64ccccb
IM		 166 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0f2e,
167 [PERF_COUNT_HW_CACHE_MISSES] = 0x012e,
11d1578f
VW		 168 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
169 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
170 [PERF_COUNT_HW_BUS_CYCLES] = 0x0062,
171};
172
dfc65094 173static u64 p6_pmu_event_map(int hw_event)
11d1578f 174{
dfc65094 175 return p6_perfmon_event_map[hw_event];
11d1578f
VW		 176}
177
9c74fb50 178/*
cdd6c482 179 * Event setting that is specified not to count anything.
9c74fb50
PZ		 180 * We use this to effectively disable a counter.
181 *
182 * L2_RQSTS with 0 MESI unit mask.
183 */
cdd6c482 184#define P6_NOP_EVENT 0x0000002EULL
9c74fb50 185
dfc65094 186static u64 p6_pmu_raw_event(u64 hw_event)
11d1578f
VW		 187{
188#define P6_EVNTSEL_EVENT_MASK 0x000000FFULL
189#define P6_EVNTSEL_UNIT_MASK 0x0000FF00ULL
190#define P6_EVNTSEL_EDGE_MASK 0x00040000ULL
191#define P6_EVNTSEL_INV_MASK 0x00800000ULL
cdd6c482 192#define P6_EVNTSEL_REG_MASK 0xFF000000ULL
11d1578f
VW		 193
194#define P6_EVNTSEL_MASK \
195 (P6_EVNTSEL_EVENT_MASK | \
196 P6_EVNTSEL_UNIT_MASK | \
197 P6_EVNTSEL_EDGE_MASK | \
198 P6_EVNTSEL_INV_MASK | \
cdd6c482 199 P6_EVNTSEL_REG_MASK)
11d1578f 200
dfc65094 201 return hw_event & P6_EVNTSEL_MASK;
11d1578f
VW		 202}
203
1da53e02 204static struct event_constraint intel_p6_event_constraints[] =
b690081d 205{
8433be11
PZ		 206 INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FLOPS */
207 INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
208 INTEL_EVENT_CONSTRAINT(0x11, 0x1), /* FP_ASSIST */
209 INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
210 INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
211 INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
b690081d
SE		 212 EVENT_CONSTRAINT_END
213};
11d1578f 214
b56a3802
JSR		 215/*
216 * Intel PerfMon v3. Used on Core2 and later.
217 */
b0f3f28e 218static const u64 intel_perfmon_event_map[] =
241771ef 219{
f4dbfa8f
PZ		 220 [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
221 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
222 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
223 [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
224 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
225 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
226 [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
241771ef
IM		 227};
228
1da53e02
SE		 229static struct event_constraint intel_core_event_constraints[] =
230{
8433be11
PZ		 231 FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
232 FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
233 INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
234 INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
235 INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
236 INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
237 INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
238 INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
239 INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
240 INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
241 INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
b690081d
SE		 242 EVENT_CONSTRAINT_END
243};
244
1da53e02
SE		 245static struct event_constraint intel_nehalem_event_constraints[] =
246{
8433be11
PZ		 247 FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
248 FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
249 INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
250 INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
251 INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
252 INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
253 INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
254 INTEL_EVENT_CONSTRAINT(0x4c, 0x3), /* LOAD_HIT_PRE */
255 INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
256 INTEL_EVENT_CONSTRAINT(0x52, 0x3), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
257 INTEL_EVENT_CONSTRAINT(0x53, 0x3), /* L1D_CACHE_LOCK_FB_HIT */
258 INTEL_EVENT_CONSTRAINT(0xc5, 0x3), /* CACHE_LOCK_CYCLES */
1da53e02
SE		 259 EVENT_CONSTRAINT_END
260};
261
262static struct event_constraint intel_gen_event_constraints[] =
263{
8433be11
PZ		 264 FIXED_EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32))), /* INSTRUCTIONS_RETIRED */
265 FIXED_EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33))), /* UNHALTED_CORE_CYCLES */
b690081d
SE		 266 EVENT_CONSTRAINT_END
267};
268
dfc65094 269static u64 intel_pmu_event_map(int hw_event)
b56a3802 270{
dfc65094 271 return intel_perfmon_event_map[hw_event];
b56a3802 272}
241771ef 273
8326f44d 274/*
dfc65094 275 * Generalized hw caching related hw_event table, filled
8326f44d 276 * in on a per model basis. A value of 0 means
dfc65094
IM		 277 * 'not supported', -1 means 'hw_event makes no sense on
278 * this CPU', any other value means the raw hw_event
8326f44d
IM		 279 * ID.
280 */
281
282#define C(x) PERF_COUNT_HW_CACHE_##x
283
284static u64 __read_mostly hw_cache_event_ids
285 [PERF_COUNT_HW_CACHE_MAX]
286 [PERF_COUNT_HW_CACHE_OP_MAX]
287 [PERF_COUNT_HW_CACHE_RESULT_MAX];
288
db48cccc 289static __initconst u64 nehalem_hw_cache_event_ids
8326f44d
IM		 290 [PERF_COUNT_HW_CACHE_MAX]
291 [PERF_COUNT_HW_CACHE_OP_MAX]
292 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
293{
294 [ C(L1D) ] = {
295 [ C(OP_READ) ] = {
296 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
297 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
298 },
299 [ C(OP_WRITE) ] = {
300 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
301 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
302 },
303 [ C(OP_PREFETCH) ] = {
304 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
305 [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
306 },
307 },
308 [ C(L1I ) ] = {
309 [ C(OP_READ) ] = {
fecc8ac8 310 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
8326f44d
IM		 311 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
312 },
313 [ C(OP_WRITE) ] = {
314 [ C(RESULT_ACCESS) ] = -1,
315 [ C(RESULT_MISS) ] = -1,
316 },
317 [ C(OP_PREFETCH) ] = {
318 [ C(RESULT_ACCESS) ] = 0x0,
319 [ C(RESULT_MISS) ] = 0x0,
320 },
321 },
8be6e8f3 322 [ C(LL ) ] = {
8326f44d
IM		 323 [ C(OP_READ) ] = {
324 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS */
325 [ C(RESULT_MISS) ] = 0x0224, /* L2_RQSTS.LD_MISS */
326 },
327 [ C(OP_WRITE) ] = {
328 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS */
329 [ C(RESULT_MISS) ] = 0x0824, /* L2_RQSTS.RFO_MISS */
330 },
331 [ C(OP_PREFETCH) ] = {
8be6e8f3
PZ		 332 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference */
333 [ C(RESULT_MISS) ] = 0x412e, /* LLC Misses */
8326f44d
IM		 334 },
335 },
336 [ C(DTLB) ] = {
337 [ C(OP_READ) ] = {
338 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
339 [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
340 },
341 [ C(OP_WRITE) ] = {
342 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
343 [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
344 },
345 [ C(OP_PREFETCH) ] = {
346 [ C(RESULT_ACCESS) ] = 0x0,
347 [ C(RESULT_MISS) ] = 0x0,
348 },
349 },
350 [ C(ITLB) ] = {
351 [ C(OP_READ) ] = {
352 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
fecc8ac8 353 [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
8326f44d
IM		 354 },
355 [ C(OP_WRITE) ] = {
356 [ C(RESULT_ACCESS) ] = -1,
357 [ C(RESULT_MISS) ] = -1,
358 },
359 [ C(OP_PREFETCH) ] = {
360 [ C(RESULT_ACCESS) ] = -1,
361 [ C(RESULT_MISS) ] = -1,
362 },
363 },
364 [ C(BPU ) ] = {
365 [ C(OP_READ) ] = {
366 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
367 [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
368 },
369 [ C(OP_WRITE) ] = {
370 [ C(RESULT_ACCESS) ] = -1,
371 [ C(RESULT_MISS) ] = -1,
372 },
373 [ C(OP_PREFETCH) ] = {
374 [ C(RESULT_ACCESS) ] = -1,
375 [ C(RESULT_MISS) ] = -1,
376 },
377 },
378};
379
db48cccc 380static __initconst u64 core2_hw_cache_event_ids
8326f44d
IM		 381 [PERF_COUNT_HW_CACHE_MAX]
382 [PERF_COUNT_HW_CACHE_OP_MAX]
383 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
384{
0312af84
TG		 385 [ C(L1D) ] = {
386 [ C(OP_READ) ] = {
387 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
388 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
389 },
390 [ C(OP_WRITE) ] = {
391 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
392 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
393 },
394 [ C(OP_PREFETCH) ] = {
395 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
396 [ C(RESULT_MISS) ] = 0,
397 },
398 },
399 [ C(L1I ) ] = {
400 [ C(OP_READ) ] = {
401 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
402 [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
403 },
404 [ C(OP_WRITE) ] = {
405 [ C(RESULT_ACCESS) ] = -1,
406 [ C(RESULT_MISS) ] = -1,
407 },
408 [ C(OP_PREFETCH) ] = {
409 [ C(RESULT_ACCESS) ] = 0,
410 [ C(RESULT_MISS) ] = 0,
411 },
412 },
8be6e8f3 413 [ C(LL ) ] = {
0312af84
TG		 414 [ C(OP_READ) ] = {
415 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
416 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
417 },
418 [ C(OP_WRITE) ] = {
419 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
420 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
421 },
422 [ C(OP_PREFETCH) ] = {
423 [ C(RESULT_ACCESS) ] = 0,
424 [ C(RESULT_MISS) ] = 0,
425 },
426 },
427 [ C(DTLB) ] = {
428 [ C(OP_READ) ] = {
429 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
430 [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
431 },
432 [ C(OP_WRITE) ] = {
433 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
434 [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
435 },
436 [ C(OP_PREFETCH) ] = {
437 [ C(RESULT_ACCESS) ] = 0,
438 [ C(RESULT_MISS) ] = 0,
439 },
440 },
441 [ C(ITLB) ] = {
442 [ C(OP_READ) ] = {
443 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
444 [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
445 },
446 [ C(OP_WRITE) ] = {
447 [ C(RESULT_ACCESS) ] = -1,
448 [ C(RESULT_MISS) ] = -1,
449 },
450 [ C(OP_PREFETCH) ] = {
451 [ C(RESULT_ACCESS) ] = -1,
452 [ C(RESULT_MISS) ] = -1,
453 },
454 },
455 [ C(BPU ) ] = {
456 [ C(OP_READ) ] = {
457 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
458 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
459 },
460 [ C(OP_WRITE) ] = {
461 [ C(RESULT_ACCESS) ] = -1,
462 [ C(RESULT_MISS) ] = -1,
463 },
464 [ C(OP_PREFETCH) ] = {
465 [ C(RESULT_ACCESS) ] = -1,
466 [ C(RESULT_MISS) ] = -1,
467 },
468 },
8326f44d
IM		 469};
470
db48cccc 471static __initconst u64 atom_hw_cache_event_ids
8326f44d
IM		 472 [PERF_COUNT_HW_CACHE_MAX]
473 [PERF_COUNT_HW_CACHE_OP_MAX]
474 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
475{
ad689220
TG		 476 [ C(L1D) ] = {
477 [ C(OP_READ) ] = {
478 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
479 [ C(RESULT_MISS) ] = 0,
480 },
481 [ C(OP_WRITE) ] = {
fecc8ac8 482 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
ad689220
TG		 483 [ C(RESULT_MISS) ] = 0,
484 },
485 [ C(OP_PREFETCH) ] = {
486 [ C(RESULT_ACCESS) ] = 0x0,
487 [ C(RESULT_MISS) ] = 0,
488 },
489 },
490 [ C(L1I ) ] = {
491 [ C(OP_READ) ] = {
fecc8ac8
YW		 492 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
493 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
ad689220
TG		 494 },
495 [ C(OP_WRITE) ] = {
496 [ C(RESULT_ACCESS) ] = -1,
497 [ C(RESULT_MISS) ] = -1,
498 },
499 [ C(OP_PREFETCH) ] = {
500 [ C(RESULT_ACCESS) ] = 0,
501 [ C(RESULT_MISS) ] = 0,
502 },
503 },
8be6e8f3 504 [ C(LL ) ] = {
ad689220
TG		 505 [ C(OP_READ) ] = {
506 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
507 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
508 },
509 [ C(OP_WRITE) ] = {
510 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
511 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
512 },
513 [ C(OP_PREFETCH) ] = {
514 [ C(RESULT_ACCESS) ] = 0,
515 [ C(RESULT_MISS) ] = 0,
516 },
517 },
518 [ C(DTLB) ] = {
519 [ C(OP_READ) ] = {
fecc8ac8 520 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
ad689220
TG		 521 [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
522 },
523 [ C(OP_WRITE) ] = {
fecc8ac8 524 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
ad689220
TG		 525 [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
526 },
527 [ C(OP_PREFETCH) ] = {
528 [ C(RESULT_ACCESS) ] = 0,
529 [ C(RESULT_MISS) ] = 0,
530 },
531 },
532 [ C(ITLB) ] = {
533 [ C(OP_READ) ] = {
534 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
535 [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
536 },
537 [ C(OP_WRITE) ] = {
538 [ C(RESULT_ACCESS) ] = -1,
539 [ C(RESULT_MISS) ] = -1,
540 },
541 [ C(OP_PREFETCH) ] = {
542 [ C(RESULT_ACCESS) ] = -1,
543 [ C(RESULT_MISS) ] = -1,
544 },
545 },
546 [ C(BPU ) ] = {
547 [ C(OP_READ) ] = {
548 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
549 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
550 },
551 [ C(OP_WRITE) ] = {
552 [ C(RESULT_ACCESS) ] = -1,
553 [ C(RESULT_MISS) ] = -1,
554 },
555 [ C(OP_PREFETCH) ] = {
556 [ C(RESULT_ACCESS) ] = -1,
557 [ C(RESULT_MISS) ] = -1,
558 },
559 },
8326f44d
IM		 560};
561
dfc65094 562static u64 intel_pmu_raw_event(u64 hw_event)
b0f3f28e 563{
82bae4f8
PZ		 564#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
565#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
ff99be57
PZ		 566#define CORE_EVNTSEL_EDGE_MASK 0x00040000ULL
567#define CORE_EVNTSEL_INV_MASK 0x00800000ULL
fe9081cc 568#define CORE_EVNTSEL_REG_MASK 0xFF000000ULL
b0f3f28e 569
128f048f 570#define CORE_EVNTSEL_MASK \
1da53e02
SE		 571 (INTEL_ARCH_EVTSEL_MASK | \
572 INTEL_ARCH_UNIT_MASK | \
573 INTEL_ARCH_EDGE_MASK | \
574 INTEL_ARCH_INV_MASK | \
575 INTEL_ARCH_CNT_MASK)
b0f3f28e 576
dfc65094 577 return hw_event & CORE_EVNTSEL_MASK;
b0f3f28e
PZ		 578}
579
db48cccc 580static __initconst u64 amd_hw_cache_event_ids
f86748e9
TG		 581 [PERF_COUNT_HW_CACHE_MAX]
582 [PERF_COUNT_HW_CACHE_OP_MAX]
583 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
584{
585 [ C(L1D) ] = {
586 [ C(OP_READ) ] = {
f4db43a3
JSR		 587 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
588 [ C(RESULT_MISS) ] = 0x0041, /* Data Cache Misses */
f86748e9
TG		 589 },
590 [ C(OP_WRITE) ] = {
d9f2a5ec 591 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
f86748e9
TG		 592 [ C(RESULT_MISS) ] = 0,
593 },
594 [ C(OP_PREFETCH) ] = {
f4db43a3
JSR		 595 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts */
596 [ C(RESULT_MISS) ] = 0x0167, /* Data Prefetcher :cancelled */
f86748e9
TG		 597 },
598 },
599 [ C(L1I ) ] = {
600 [ C(OP_READ) ] = {
601 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
602 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
603 },
604 [ C(OP_WRITE) ] = {
605 [ C(RESULT_ACCESS) ] = -1,
606 [ C(RESULT_MISS) ] = -1,
607 },
608 [ C(OP_PREFETCH) ] = {
f4db43a3 609 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
f86748e9
TG		 610 [ C(RESULT_MISS) ] = 0,
611 },
612 },
8be6e8f3 613 [ C(LL ) ] = {
f86748e9 614 [ C(OP_READ) ] = {
f4db43a3
JSR		 615 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
616 [ C(RESULT_MISS) ] = 0x037E, /* L2 Cache Misses : IC+DC */
f86748e9
TG		 617 },
618 [ C(OP_WRITE) ] = {
f4db43a3 619 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback */
f86748e9
TG		 620 [ C(RESULT_MISS) ] = 0,
621 },
622 [ C(OP_PREFETCH) ] = {
623 [ C(RESULT_ACCESS) ] = 0,
624 [ C(RESULT_MISS) ] = 0,
625 },
626 },
627 [ C(DTLB) ] = {
628 [ C(OP_READ) ] = {
f4db43a3
JSR		 629 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses */
630 [ C(RESULT_MISS) ] = 0x0046, /* L1 DTLB and L2 DLTB Miss */
f86748e9
TG		 631 },
632 [ C(OP_WRITE) ] = {
633 [ C(RESULT_ACCESS) ] = 0,
634 [ C(RESULT_MISS) ] = 0,
635 },
636 [ C(OP_PREFETCH) ] = {
637 [ C(RESULT_ACCESS) ] = 0,
638 [ C(RESULT_MISS) ] = 0,
639 },
640 },
641 [ C(ITLB) ] = {
642 [ C(OP_READ) ] = {
643 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
644 [ C(RESULT_MISS) ] = 0x0085, /* Instr. fetch ITLB misses */
645 },
646 [ C(OP_WRITE) ] = {
647 [ C(RESULT_ACCESS) ] = -1,
648 [ C(RESULT_MISS) ] = -1,
649 },
650 [ C(OP_PREFETCH) ] = {
651 [ C(RESULT_ACCESS) ] = -1,
652 [ C(RESULT_MISS) ] = -1,
653 },
654 },
655 [ C(BPU ) ] = {
656 [ C(OP_READ) ] = {
657 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
658 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
659 },
660 [ C(OP_WRITE) ] = {
661 [ C(RESULT_ACCESS) ] = -1,
662 [ C(RESULT_MISS) ] = -1,
663 },
664 [ C(OP_PREFETCH) ] = {
665 [ C(RESULT_ACCESS) ] = -1,
666 [ C(RESULT_MISS) ] = -1,
667 },
668 },
669};
670
f87ad35d
JSR		 671/*
672 * AMD Performance Monitor K7 and later.
673 */
b0f3f28e 674static const u64 amd_perfmon_event_map[] =
f87ad35d 675{
f4dbfa8f
PZ		 676 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
677 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
678 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
679 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
680 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
681 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
f87ad35d
JSR		 682};
683
dfc65094 684static u64 amd_pmu_event_map(int hw_event)
f87ad35d 685{
dfc65094 686 return amd_perfmon_event_map[hw_event];
f87ad35d
JSR		 687}
688
dfc65094 689static u64 amd_pmu_raw_event(u64 hw_event)
b0f3f28e 690{
82bae4f8
PZ		 691#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
692#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
ff99be57
PZ		 693#define K7_EVNTSEL_EDGE_MASK 0x000040000ULL
694#define K7_EVNTSEL_INV_MASK 0x000800000ULL
cdd6c482 695#define K7_EVNTSEL_REG_MASK 0x0FF000000ULL
b0f3f28e
PZ		 696
697#define K7_EVNTSEL_MASK \
698 (K7_EVNTSEL_EVENT_MASK | \
699 K7_EVNTSEL_UNIT_MASK | \
ff99be57
PZ		 700 K7_EVNTSEL_EDGE_MASK | \
701 K7_EVNTSEL_INV_MASK | \
cdd6c482 702 K7_EVNTSEL_REG_MASK)
b0f3f28e 703
dfc65094 704 return hw_event & K7_EVNTSEL_MASK;
b0f3f28e
PZ		 705}
706
ee06094f 707/*
cdd6c482
IM		 708 * Propagate event elapsed time into the generic event.
709 * Can only be executed on the CPU where the event is active.
ee06094f
IM		 710 * Returns the delta events processed.
711 */
4b7bfd0d 712static u64
cdd6c482
IM		 713x86_perf_event_update(struct perf_event *event,
714 struct hw_perf_event *hwc, int idx)
ee06094f 715{
cdd6c482 716 int shift = 64 - x86_pmu.event_bits;
ec3232bd
PZ		 717 u64 prev_raw_count, new_raw_count;
718 s64 delta;
ee06094f 719
30dd568c
MM		 720 if (idx == X86_PMC_IDX_FIXED_BTS)
721 return 0;
722
ee06094f 723 /*
cdd6c482 724 * Careful: an NMI might modify the previous event value.
ee06094f
IM		 725 *
726 * Our tactic to handle this is to first atomically read and
727 * exchange a new raw count - then add that new-prev delta
cdd6c482 728 * count to the generic event atomically:
ee06094f
IM		 729 */
730again:
731 prev_raw_count = atomic64_read(&hwc->prev_count);
cdd6c482 732 rdmsrl(hwc->event_base + idx, new_raw_count);
ee06094f
IM		 733
734 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
735 new_raw_count) != prev_raw_count)
736 goto again;
737
738 /*
739 * Now we have the new raw value and have updated the prev
740 * timestamp already. We can now calculate the elapsed delta
cdd6c482 741 * (event-)time and add that to the generic event.
ee06094f
IM		 742 *
743 * Careful, not all hw sign-extends above the physical width
ec3232bd 744 * of the count.
ee06094f 745 */
ec3232bd
PZ		 746 delta = (new_raw_count << shift) - (prev_raw_count << shift);
747 delta >>= shift;
ee06094f 748
cdd6c482 749 atomic64_add(delta, &event->count);
ee06094f 750 atomic64_sub(delta, &hwc->period_left);
4b7bfd0d
RR		 751
752 return new_raw_count;
ee06094f
IM		 753}
754
cdd6c482 755static atomic_t active_events;
4e935e47
PZ		 756static DEFINE_MUTEX(pmc_reserve_mutex);
757
758static bool reserve_pmc_hardware(void)
759{
04da8a43 760#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 761 int i;
762
763 if (nmi_watchdog == NMI_LOCAL_APIC)
764 disable_lapic_nmi_watchdog();
765
cdd6c482 766 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85 767 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
4e935e47
PZ		 768 goto perfctr_fail;
769 }
770
cdd6c482 771 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85 772 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
4e935e47
PZ		 773 goto eventsel_fail;
774 }
04da8a43 775#endif
4e935e47
PZ		 776
777 return true;
778
04da8a43 779#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 780eventsel_fail:
781 for (i--; i >= 0; i--)
4a06bd85 782 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47 783
cdd6c482 784 i = x86_pmu.num_events;
4e935e47
PZ		 785
786perfctr_fail:
787 for (i--; i >= 0; i--)
4a06bd85 788 release_perfctr_nmi(x86_pmu.perfctr + i);
4e935e47
PZ		 789
790 if (nmi_watchdog == NMI_LOCAL_APIC)
791 enable_lapic_nmi_watchdog();
792
793 return false;
04da8a43 794#endif
4e935e47
PZ		 795}
796
797static void release_pmc_hardware(void)
798{
04da8a43 799#ifdef CONFIG_X86_LOCAL_APIC
4e935e47
PZ		 800 int i;
801
cdd6c482 802 for (i = 0; i < x86_pmu.num_events; i++) {
4a06bd85
RR		 803 release_perfctr_nmi(x86_pmu.perfctr + i);
804 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47
PZ		 805 }
806
807 if (nmi_watchdog == NMI_LOCAL_APIC)
808 enable_lapic_nmi_watchdog();
04da8a43 809#endif
4e935e47
PZ		 810}
811
30dd568c
MM		 812static inline bool bts_available(void)
813{
814 return x86_pmu.enable_bts != NULL;
815}
816
817static inline void init_debug_store_on_cpu(int cpu)
818{
cdd6c482 819 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
30dd568c
MM		 820
821 if (!ds)
822 return;
823
824 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
596da17f 825 (u32)((u64)(unsigned long)ds),
826 (u32)((u64)(unsigned long)ds >> 32));
30dd568c
MM		 827}
828
829static inline void fini_debug_store_on_cpu(int cpu)
830{
cdd6c482 831 if (!per_cpu(cpu_hw_events, cpu).ds)
30dd568c
MM		 832 return;
833
834 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
835}
836
837static void release_bts_hardware(void)
838{
839 int cpu;
840
841 if (!bts_available())
842 return;
843
844 get_online_cpus();
845
846 for_each_online_cpu(cpu)
847 fini_debug_store_on_cpu(cpu);
848
849 for_each_possible_cpu(cpu) {
cdd6c482 850 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
30dd568c
MM		 851
852 if (!ds)
853 continue;
854
cdd6c482 855 per_cpu(cpu_hw_events, cpu).ds = NULL;
30dd568c 856
596da17f 857 kfree((void *)(unsigned long)ds->bts_buffer_base);
30dd568c
MM		 858 kfree(ds);
859 }
860
861 put_online_cpus();
862}
863
864static int reserve_bts_hardware(void)
865{
866 int cpu, err = 0;
867
868 if (!bts_available())
747b50aa 869 return 0;
30dd568c
MM		 870
871 get_online_cpus();
872
873 for_each_possible_cpu(cpu) {
874 struct debug_store *ds;
875 void *buffer;
876
877 err = -ENOMEM;
878 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
879 if (unlikely(!buffer))
880 break;
881
882 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
883 if (unlikely(!ds)) {
884 kfree(buffer);
885 break;
886 }
887
596da17f 888 ds->bts_buffer_base = (u64)(unsigned long)buffer;
30dd568c
MM		 889 ds->bts_index = ds->bts_buffer_base;
890 ds->bts_absolute_maximum =
891 ds->bts_buffer_base + BTS_BUFFER_SIZE;
892 ds->bts_interrupt_threshold =
893 ds->bts_absolute_maximum - BTS_OVFL_TH;
894
cdd6c482 895 per_cpu(cpu_hw_events, cpu).ds = ds;
30dd568c
MM		 896 err = 0;
897 }
898
899 if (err)
900 release_bts_hardware();
901 else {
902 for_each_online_cpu(cpu)
903 init_debug_store_on_cpu(cpu);
904 }
905
906 put_online_cpus();
907
908 return err;
909}
910
cdd6c482 911static void hw_perf_event_destroy(struct perf_event *event)
4e935e47 912{
cdd6c482 913 if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
4e935e47 914 release_pmc_hardware();
30dd568c 915 release_bts_hardware();
4e935e47
PZ		 916 mutex_unlock(&pmc_reserve_mutex);
917 }
918}
919
85cf9dba
RR		 920static inline int x86_pmu_initialized(void)
921{
922 return x86_pmu.handle_irq != NULL;
923}
924
8326f44d 925static inline int
cdd6c482 926set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
8326f44d
IM		 927{
928 unsigned int cache_type, cache_op, cache_result;
929 u64 config, val;
930
931 config = attr->config;
932
933 cache_type = (config >> 0) & 0xff;
934 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
935 return -EINVAL;
936
937 cache_op = (config >> 8) & 0xff;
938 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
939 return -EINVAL;
940
941 cache_result = (config >> 16) & 0xff;
942 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
943 return -EINVAL;
944
945 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
946
947 if (val == 0)
948 return -ENOENT;
949
950 if (val == -1)
951 return -EINVAL;
952
953 hwc->config |= val;
954
955 return 0;
956}
957
30dd568c
MM		 958static void intel_pmu_enable_bts(u64 config)
959{
960 unsigned long debugctlmsr;
961
962 debugctlmsr = get_debugctlmsr();
963
964 debugctlmsr |= X86_DEBUGCTL_TR;
965 debugctlmsr |= X86_DEBUGCTL_BTS;
966 debugctlmsr |= X86_DEBUGCTL_BTINT;
967
968 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
969 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
970
971 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
972 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
973
974 update_debugctlmsr(debugctlmsr);
975}
976
977static void intel_pmu_disable_bts(void)
978{
cdd6c482 979 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 980 unsigned long debugctlmsr;
981
982 if (!cpuc->ds)
983 return;
984
985 debugctlmsr = get_debugctlmsr();
986
987 debugctlmsr &=
988 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
989 X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
990
991 update_debugctlmsr(debugctlmsr);
992}
993
241771ef 994/*
0d48696f 995 * Setup the hardware configuration for a given attr_type
241771ef 996 */
cdd6c482 997static int __hw_perf_event_init(struct perf_event *event)
241771ef 998{
cdd6c482
IM		 999 struct perf_event_attr *attr = &event->attr;
1000 struct hw_perf_event *hwc = &event->hw;
9c74fb50 1001 u64 config;
4e935e47 1002 int err;
241771ef 1003
85cf9dba
RR		 1004 if (!x86_pmu_initialized())
1005 return -ENODEV;
241771ef 1006
4e935e47 1007 err = 0;
cdd6c482 1008 if (!atomic_inc_not_zero(&active_events)) {
4e935e47 1009 mutex_lock(&pmc_reserve_mutex);
cdd6c482 1010 if (atomic_read(&active_events) == 0) {
30dd568c
MM		 1011 if (!reserve_pmc_hardware())
1012 err = -EBUSY;
1013 else
747b50aa 1014 err = reserve_bts_hardware();
30dd568c
MM		 1015 }
1016 if (!err)
cdd6c482 1017 atomic_inc(&active_events);
4e935e47
PZ		 1018 mutex_unlock(&pmc_reserve_mutex);
1019 }
1020 if (err)
1021 return err;
1022
cdd6c482 1023 event->destroy = hw_perf_event_destroy;
a1792cda 1024
241771ef 1025 /*
0475f9ea 1026 * Generate PMC IRQs:
241771ef
IM		 1027 * (keep 'enabled' bit clear for now)
1028 */
0475f9ea 1029 hwc->config = ARCH_PERFMON_EVENTSEL_INT;
241771ef 1030
b690081d
SE		 1031 hwc->idx = -1;
1032
241771ef 1033 /*
0475f9ea 1034 * Count user and OS events unless requested not to.
241771ef 1035 */
0d48696f 1036 if (!attr->exclude_user)
0475f9ea 1037 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
0d48696f 1038 if (!attr->exclude_kernel)
241771ef 1039 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
0475f9ea 1040
bd2b5b12 1041 if (!hwc->sample_period) {
b23f3325 1042 hwc->sample_period = x86_pmu.max_period;
9e350de3 1043 hwc->last_period = hwc->sample_period;
bd2b5b12 1044 atomic64_set(&hwc->period_left, hwc->sample_period);
04da8a43
IM		 1045 } else {
1046 /*
1047 * If we have a PMU initialized but no APIC
1048 * interrupts, we cannot sample hardware
cdd6c482
IM		 1049 * events (user-space has to fall back and
1050 * sample via a hrtimer based software event):
04da8a43
IM		 1051 */
1052 if (!x86_pmu.apic)
1053 return -EOPNOTSUPP;
bd2b5b12 1054 }
d2517a49 1055
241771ef 1056 /*
dfc65094 1057 * Raw hw_event type provide the config in the hw_event structure
241771ef 1058 */
a21ca2ca
IM		 1059 if (attr->type == PERF_TYPE_RAW) {
1060 hwc->config |= x86_pmu.raw_event(attr->config);
8326f44d 1061 return 0;
241771ef 1062 }
241771ef 1063
8326f44d
IM		 1064 if (attr->type == PERF_TYPE_HW_CACHE)
1065 return set_ext_hw_attr(hwc, attr);
1066
1067 if (attr->config >= x86_pmu.max_events)
1068 return -EINVAL;
9c74fb50 1069
8326f44d
IM		 1070 /*
1071 * The generic map:
1072 */
9c74fb50
PZ		 1073 config = x86_pmu.event_map(attr->config);
1074
1075 if (config == 0)
1076 return -ENOENT;
1077
1078 if (config == -1LL)
1079 return -EINVAL;
1080
747b50aa 1081 /*
1082 * Branch tracing:
1083 */
1084 if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1653192f 1085 (hwc->sample_period == 1)) {
1086 /* BTS is not supported by this architecture. */
1087 if (!bts_available())
1088 return -EOPNOTSUPP;
1089
1090 /* BTS is currently only allowed for user-mode. */
1091 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1092 return -EOPNOTSUPP;
1093 }
747b50aa 1094
9c74fb50 1095 hwc->config |= config;
4e935e47 1096
241771ef
IM		 1097 return 0;
1098}
1099
11d1578f
VW		 1100static void p6_pmu_disable_all(void)
1101{
cdd6c482 1102 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
9c74fb50 1103 u64 val;
11d1578f
VW		 1104
1105 if (!cpuc->enabled)
1106 return;
1107
1108 cpuc->enabled = 0;
1109 barrier();
1110
1111 /* p6 only has one enable register */
1112 rdmsrl(MSR_P6_EVNTSEL0, val);
1113 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1114 wrmsrl(MSR_P6_EVNTSEL0, val);
1115}
1116
9e35ad38 1117static void intel_pmu_disable_all(void)
4ac13294 1118{
cdd6c482 1119 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 1120
1121 if (!cpuc->enabled)
1122 return;
1123
1124 cpuc->enabled = 0;
1125 barrier();
1126
862a1a5f 1127 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
30dd568c
MM		 1128
1129 if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1130 intel_pmu_disable_bts();
241771ef 1131}
b56a3802 1132
9e35ad38 1133static void amd_pmu_disable_all(void)
f87ad35d 1134{
cdd6c482 1135 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
9e35ad38
PZ		 1136 int idx;
1137
1138 if (!cpuc->enabled)
1139 return;
b0f3f28e 1140
b0f3f28e 1141 cpuc->enabled = 0;
60b3df9c
PZ		 1142 /*
1143 * ensure we write the disable before we start disabling the
cdd6c482 1144 * events proper, so that amd_pmu_enable_event() does the
5f4ec28f 1145 * right thing.
60b3df9c 1146 */
b0f3f28e 1147 barrier();
f87ad35d 1148
cdd6c482 1149 for (idx = 0; idx < x86_pmu.num_events; idx++) {
b0f3f28e
PZ		 1150 u64 val;
1151
43f6201a 1152 if (!test_bit(idx, cpuc->active_mask))
4295ee62 1153 continue;
f87ad35d 1154 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
4295ee62
RR		 1155 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1156 continue;
1157 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1158 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d 1159 }
f87ad35d
JSR		 1160}
1161
9e35ad38 1162void hw_perf_disable(void)
b56a3802 1163{
1da53e02
SE		 1164 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1165
85cf9dba 1166 if (!x86_pmu_initialized())
9e35ad38 1167 return;
1da53e02
SE		 1168
1169 if (cpuc->enabled)
1170 cpuc->n_added = 0;
1171
1172 x86_pmu.disable_all();
b56a3802 1173}
241771ef 1174
11d1578f
VW		 1175static void p6_pmu_enable_all(void)
1176{
cdd6c482 1177 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
11d1578f
VW		 1178 unsigned long val;
1179
1180 if (cpuc->enabled)
1181 return;
1182
1183 cpuc->enabled = 1;
1184 barrier();
1185
1186 /* p6 only has one enable register */
1187 rdmsrl(MSR_P6_EVNTSEL0, val);
1188 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1189 wrmsrl(MSR_P6_EVNTSEL0, val);
1190}
1191
9e35ad38 1192static void intel_pmu_enable_all(void)
b56a3802 1193{
cdd6c482 1194 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
30dd568c
MM		 1195
1196 if (cpuc->enabled)
1197 return;
1198
1199 cpuc->enabled = 1;
1200 barrier();
1201
9e35ad38 1202 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
30dd568c
MM		 1203
1204 if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
cdd6c482
IM		 1205 struct perf_event *event =
1206 cpuc->events[X86_PMC_IDX_FIXED_BTS];
30dd568c 1207
cdd6c482 1208 if (WARN_ON_ONCE(!event))
30dd568c
MM		 1209 return;
1210
cdd6c482 1211 intel_pmu_enable_bts(event->hw.config);
30dd568c 1212 }
b56a3802
JSR		 1213}
1214
9e35ad38 1215static void amd_pmu_enable_all(void)
f87ad35d 1216{
cdd6c482 1217 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
f87ad35d
JSR		 1218 int idx;
1219
9e35ad38 1220 if (cpuc->enabled)
b0f3f28e
PZ		 1221 return;
1222
9e35ad38
PZ		 1223 cpuc->enabled = 1;
1224 barrier();
1225
cdd6c482
IM		 1226 for (idx = 0; idx < x86_pmu.num_events; idx++) {
1227 struct perf_event *event = cpuc->events[idx];
4295ee62 1228 u64 val;
b0f3f28e 1229
43f6201a 1230 if (!test_bit(idx, cpuc->active_mask))
4295ee62 1231 continue;
984b838c 1232
cdd6c482 1233 val = event->hw.config;
4295ee62
RR		 1234 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1235 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d
JSR		 1236 }
1237}
1238
1da53e02
SE		 1239static const struct pmu pmu;
1240
1241static inline int is_x86_event(struct perf_event *event)
1242{
1243 return event->pmu == &pmu;
1244}
1245
1246static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
1247{
63b14649 1248 struct event_constraint *c, *constraints[X86_PMC_IDX_MAX];
1da53e02 1249 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
c933c1a6 1250 int i, j, w, wmax, num = 0;
1da53e02
SE		 1251 struct hw_perf_event *hwc;
1252
1253 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
1254
1255 for (i = 0; i < n; i++) {
63b14649
PZ		 1256 constraints[i] =
1257 x86_pmu.get_event_constraints(cpuc, cpuc->event_list[i]);
1da53e02
SE		 1258 }
1259
8113070d
SE		 1260 /*
1261 * fastpath, try to reuse previous register
1262 */
c933c1a6 1263 for (i = 0; i < n; i++) {
8113070d 1264 hwc = &cpuc->event_list[i]->hw;
81269a08 1265 c = constraints[i];
8113070d
SE		 1266
1267 /* never assigned */
1268 if (hwc->idx == -1)
1269 break;
1270
1271 /* constraint still honored */
63b14649 1272 if (!test_bit(hwc->idx, c->idxmsk))
8113070d
SE		 1273 break;
1274
1275 /* not already used */
1276 if (test_bit(hwc->idx, used_mask))
1277 break;
1278
1279#if 0
1280 pr_debug("CPU%d fast config=0x%llx idx=%d assign=%c\n",
1281 smp_processor_id(),
1282 hwc->config,
1283 hwc->idx,
1284 assign ? 'y' : 'n');
1285#endif
1286
1287 set_bit(hwc->idx, used_mask);
1288 if (assign)
1289 assign[i] = hwc->idx;
1290 }
c933c1a6 1291 if (i == n)
8113070d
SE		 1292 goto done;
1293
1294 /*
1295 * begin slow path
1296 */
1297
1298 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
1299
1da53e02
SE		 1300 /*
1301 * weight = number of possible counters
1302 *
1303 * 1 = most constrained, only works on one counter
1304 * wmax = least constrained, works on any counter
1305 *
1306 * assign events to counters starting with most
1307 * constrained events.
1308 */
1309 wmax = x86_pmu.num_events;
1310
1311 /*
1312 * when fixed event counters are present,
1313 * wmax is incremented by 1 to account
1314 * for one more choice
1315 */
1316 if (x86_pmu.num_events_fixed)
1317 wmax++;
1318
8113070d 1319 for (w = 1, num = n; num && w <= wmax; w++) {
1da53e02 1320 /* for each event */
8113070d 1321 for (i = 0; num && i < n; i++) {
81269a08 1322 c = constraints[i];
1da53e02
SE		 1323 hwc = &cpuc->event_list[i]->hw;
1324
272d30be 1325 if (c->weight != w)
1da53e02
SE		 1326 continue;
1327
63b14649 1328 for_each_bit(j, c->idxmsk, X86_PMC_IDX_MAX) {
1da53e02
SE		 1329 if (!test_bit(j, used_mask))
1330 break;
1331 }
1332
1333 if (j == X86_PMC_IDX_MAX)
1334 break;
1da53e02
SE		 1335
1336#if 0
8113070d 1337 pr_debug("CPU%d slow config=0x%llx idx=%d assign=%c\n",
1da53e02
SE		 1338 smp_processor_id(),
1339 hwc->config,
1340 j,
1341 assign ? 'y' : 'n');
1342#endif
1343
8113070d
SE		 1344 set_bit(j, used_mask);
1345
1da53e02
SE		 1346 if (assign)
1347 assign[i] = j;
1348 num--;
1349 }
1350 }
8113070d 1351done:
1da53e02
SE		 1352 /*
1353 * scheduling failed or is just a simulation,
1354 * free resources if necessary
1355 */
1356 if (!assign || num) {
1357 for (i = 0; i < n; i++) {
1358 if (x86_pmu.put_event_constraints)
1359 x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
1360 }
1361 }
1362 return num ? -ENOSPC : 0;
1363}
1364
1365/*
1366 * dogrp: true if must collect siblings events (group)
1367 * returns total number of events and error code
1368 */
1369static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
1370{
1371 struct perf_event *event;
1372 int n, max_count;
1373
1374 max_count = x86_pmu.num_events + x86_pmu.num_events_fixed;
1375
1376 /* current number of events already accepted */
1377 n = cpuc->n_events;
1378
1379 if (is_x86_event(leader)) {
1380 if (n >= max_count)
1381 return -ENOSPC;
1382 cpuc->event_list[n] = leader;
1383 n++;
1384 }
1385 if (!dogrp)
1386 return n;
1387
1388 list_for_each_entry(event, &leader->sibling_list, group_entry) {
1389 if (!is_x86_event(event) ||
8113070d 1390 event->state <= PERF_EVENT_STATE_OFF)
1da53e02
SE		 1391 continue;
1392
1393 if (n >= max_count)
1394 return -ENOSPC;
1395
1396 cpuc->event_list[n] = event;
1397 n++;
1398 }
1399 return n;
1400}
1401
1402
1403static inline void x86_assign_hw_event(struct perf_event *event,
1404 struct hw_perf_event *hwc, int idx)
1405{
1406 hwc->idx = idx;
1407
1408 if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
1409 hwc->config_base = 0;
1410 hwc->event_base = 0;
1411 } else if (hwc->idx >= X86_PMC_IDX_FIXED) {
1412 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1413 /*
1414 * We set it so that event_base + idx in wrmsr/rdmsr maps to
1415 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1416 */
1417 hwc->event_base =
1418 MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1419 } else {
1420 hwc->config_base = x86_pmu.eventsel;
1421 hwc->event_base = x86_pmu.perfctr;
1422 }
1423}
1424
9e35ad38 1425void hw_perf_enable(void)
ee06094f 1426{
1da53e02
SE		 1427 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1428 struct perf_event *event;
1429 struct hw_perf_event *hwc;
1430 int i;
1431
85cf9dba 1432 if (!x86_pmu_initialized())
2b9ff0db 1433 return;
1da53e02
SE		 1434 if (cpuc->n_added) {
1435 /*
1436 * apply assignment obtained either from
1437 * hw_perf_group_sched_in() or x86_pmu_enable()
1438 *
1439 * step1: save events moving to new counters
1440 * step2: reprogram moved events into new counters
1441 */
1442 for (i = 0; i < cpuc->n_events; i++) {
1443
1444 event = cpuc->event_list[i];
1445 hwc = &event->hw;
1446
1447 if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
1448 continue;
1449
1450 x86_pmu.disable(hwc, hwc->idx);
1451
1452 clear_bit(hwc->idx, cpuc->active_mask);
1453 barrier();
1454 cpuc->events[hwc->idx] = NULL;
1455
1456 x86_perf_event_update(event, hwc, hwc->idx);
1457
1458 hwc->idx = -1;
1459 }
1460
1461 for (i = 0; i < cpuc->n_events; i++) {
1462
1463 event = cpuc->event_list[i];
1464 hwc = &event->hw;
1465
1466 if (hwc->idx == -1) {
1467 x86_assign_hw_event(event, hwc, cpuc->assign[i]);
1468 x86_perf_event_set_period(event, hwc, hwc->idx);
1469 }
1470 /*
1471 * need to mark as active because x86_pmu_disable()
1472 * clear active_mask and eventsp[] yet it preserves
1473 * idx
1474 */
1475 set_bit(hwc->idx, cpuc->active_mask);
1476 cpuc->events[hwc->idx] = event;
1477
1478 x86_pmu.enable(hwc, hwc->idx);
1479 perf_event_update_userpage(event);
1480 }
1481 cpuc->n_added = 0;
1482 perf_events_lapic_init();
1483 }
9e35ad38 1484 x86_pmu.enable_all();
ee06094f 1485}
ee06094f 1486
19d84dab 1487static inline u64 intel_pmu_get_status(void)
b0f3f28e
PZ		 1488{
1489 u64 status;
1490
b7f8859a 1491 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
b0f3f28e 1492
b7f8859a 1493 return status;
b0f3f28e
PZ		 1494}
1495
dee5d906 1496static inline void intel_pmu_ack_status(u64 ack)
b0f3f28e
PZ		 1497{
1498 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1499}
1500
cdd6c482 1501static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
b0f3f28e 1502{
11d1578f 1503 (void)checking_wrmsrl(hwc->config_base + idx,
7c90cc45 1504 hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
b0f3f28e
PZ		 1505}
1506
cdd6c482 1507static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
b0f3f28e 1508{
11d1578f 1509 (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
b0f3f28e
PZ		 1510}
1511
2f18d1e8 1512static inline void
cdd6c482 1513intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
2f18d1e8
IM		 1514{
1515 int idx = __idx - X86_PMC_IDX_FIXED;
1516 u64 ctrl_val, mask;
2f18d1e8
IM		 1517
1518 mask = 0xfULL << (idx * 4);
1519
1520 rdmsrl(hwc->config_base, ctrl_val);
1521 ctrl_val &= ~mask;
11d1578f
VW		 1522 (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1523}
1524
1525static inline void
cdd6c482 1526p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
11d1578f 1527{
cdd6c482
IM		 1528 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1529 u64 val = P6_NOP_EVENT;
11d1578f 1530
9c74fb50
PZ		 1531 if (cpuc->enabled)
1532 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
11d1578f
VW		 1533
1534 (void)checking_wrmsrl(hwc->config_base + idx, val);
2f18d1e8
IM		 1535}
1536
7e2ae347 1537static inline void
cdd6c482 1538intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
7e2ae347 1539{
30dd568c
MM		 1540 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1541 intel_pmu_disable_bts();
1542 return;
1543 }
1544
d4369891
RR		 1545 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1546 intel_pmu_disable_fixed(hwc, idx);
1547 return;
1548 }
1549
cdd6c482 1550 x86_pmu_disable_event(hwc, idx);
d4369891
RR		 1551}
1552
1553static inline void
cdd6c482 1554amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
d4369891 1555{
cdd6c482 1556 x86_pmu_disable_event(hwc, idx);
7e2ae347
IM		 1557}
1558
245b2e70 1559static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
241771ef 1560
ee06094f
IM		 1561/*
1562 * Set the next IRQ period, based on the hwc->period_left value.
cdd6c482 1563 * To be called with the event disabled in hw:
ee06094f 1564 */
e4abb5d4 1565static int
cdd6c482
IM		 1566x86_perf_event_set_period(struct perf_event *event,
1567 struct hw_perf_event *hwc, int idx)
241771ef 1568{
2f18d1e8 1569 s64 left = atomic64_read(&hwc->period_left);
e4abb5d4
PZ		 1570 s64 period = hwc->sample_period;
1571 int err, ret = 0;
ee06094f 1572
30dd568c
MM		 1573 if (idx == X86_PMC_IDX_FIXED_BTS)
1574 return 0;
1575
ee06094f 1576 /*
af901ca1 1577 * If we are way outside a reasonable range then just skip forward:
ee06094f
IM		 1578 */
1579 if (unlikely(left <= -period)) {
1580 left = period;
1581 atomic64_set(&hwc->period_left, left);
9e350de3 1582 hwc->last_period = period;
e4abb5d4 1583 ret = 1;
ee06094f
IM		 1584 }
1585
1586 if (unlikely(left <= 0)) {
1587 left += period;
1588 atomic64_set(&hwc->period_left, left);
9e350de3 1589 hwc->last_period = period;
e4abb5d4 1590 ret = 1;
ee06094f 1591 }
1c80f4b5 1592 /*
dfc65094 1593 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1c80f4b5
IM		 1594 */
1595 if (unlikely(left < 2))
1596 left = 2;
241771ef 1597
e4abb5d4
PZ		 1598 if (left > x86_pmu.max_period)
1599 left = x86_pmu.max_period;
1600
245b2e70 1601 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
ee06094f
IM		 1602
1603 /*
cdd6c482 1604 * The hw event starts counting from this event offset,
ee06094f
IM		 1605 * mark it to be able to extra future deltas:
1606 */
2f18d1e8 1607 atomic64_set(&hwc->prev_count, (u64)-left);
ee06094f 1608
cdd6c482
IM		 1609 err = checking_wrmsrl(hwc->event_base + idx,
1610 (u64)(-left) & x86_pmu.event_mask);
e4abb5d4 1611
cdd6c482 1612 perf_event_update_userpage(event);
194002b2 1613
e4abb5d4 1614 return ret;
2f18d1e8
IM		 1615}
1616
1617static inline void
cdd6c482 1618intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
2f18d1e8
IM		 1619{
1620 int idx = __idx - X86_PMC_IDX_FIXED;
1621 u64 ctrl_val, bits, mask;
1622 int err;
1623
1624 /*
0475f9ea
PM		 1625 * Enable IRQ generation (0x8),
1626 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1627 * if requested:
2f18d1e8 1628 */
0475f9ea
PM		 1629 bits = 0x8ULL;
1630 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1631 bits |= 0x2;
2f18d1e8
IM		 1632 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1633 bits |= 0x1;
1634 bits <<= (idx * 4);
1635 mask = 0xfULL << (idx * 4);
1636
1637 rdmsrl(hwc->config_base, ctrl_val);
1638 ctrl_val &= ~mask;
1639 ctrl_val |= bits;
1640 err = checking_wrmsrl(hwc->config_base, ctrl_val);
7e2ae347
IM		 1641}
1642
cdd6c482 1643static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
11d1578f 1644{
cdd6c482 1645 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
984b838c 1646 u64 val;
11d1578f 1647
984b838c 1648 val = hwc->config;
11d1578f 1649 if (cpuc->enabled)
984b838c
PZ		 1650 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1651
1652 (void)checking_wrmsrl(hwc->config_base + idx, val);
11d1578f
VW		 1653}
1654
1655
cdd6c482 1656static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
7e2ae347 1657{
30dd568c 1658 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
cdd6c482 1659 if (!__get_cpu_var(cpu_hw_events).enabled)
30dd568c
MM		 1660 return;
1661
1662 intel_pmu_enable_bts(hwc->config);
1663 return;
1664 }
1665
7c90cc45
RR		 1666 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1667 intel_pmu_enable_fixed(hwc, idx);
1668 return;
1669 }
1670
cdd6c482 1671 x86_pmu_enable_event(hwc, idx);
7c90cc45
RR		 1672}
1673
cdd6c482 1674static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
7c90cc45 1675{
cdd6c482 1676 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
7c90cc45
RR		 1677
1678 if (cpuc->enabled)
cdd6c482 1679 x86_pmu_enable_event(hwc, idx);
241771ef
IM		 1680}
1681
b690081d 1682/*
1da53e02
SE		 1683 * activate a single event
1684 *
1685 * The event is added to the group of enabled events
1686 * but only if it can be scehduled with existing events.
1687 *
1688 * Called with PMU disabled. If successful and return value 1,
1689 * then guaranteed to call perf_enable() and hw_perf_enable()
fe9081cc
PZ		 1690 */
1691static int x86_pmu_enable(struct perf_event *event)
1692{
1693 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1da53e02
SE		 1694 struct hw_perf_event *hwc;
1695 int assign[X86_PMC_IDX_MAX];
1696 int n, n0, ret;
fe9081cc 1697
1da53e02 1698 hwc = &event->hw;
fe9081cc 1699
1da53e02
SE		 1700 n0 = cpuc->n_events;
1701 n = collect_events(cpuc, event, false);
1702 if (n < 0)
1703 return n;
53b441a5 1704
1da53e02
SE		 1705 ret = x86_schedule_events(cpuc, n, assign);
1706 if (ret)
1707 return ret;
1708 /*
1709 * copy new assignment, now we know it is possible
1710 * will be used by hw_perf_enable()
1711 */
1712 memcpy(cpuc->assign, assign, n*sizeof(int));
7e2ae347 1713
1da53e02
SE		 1714 cpuc->n_events = n;
1715 cpuc->n_added = n - n0;
95cdd2e7 1716
1da53e02
SE		 1717 if (hwc->idx != -1)
1718 x86_perf_event_set_period(event, hwc, hwc->idx);
194002b2 1719
95cdd2e7 1720 return 0;
241771ef
IM		 1721}
1722
cdd6c482 1723static void x86_pmu_unthrottle(struct perf_event *event)
a78ac325 1724{
cdd6c482
IM		 1725 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1726 struct hw_perf_event *hwc = &event->hw;
a78ac325
PZ		 1727
1728 if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
cdd6c482 1729 cpuc->events[hwc->idx] != event))
a78ac325
PZ		 1730 return;
1731
1732 x86_pmu.enable(hwc, hwc->idx);
1733}
1734
cdd6c482 1735void perf_event_print_debug(void)
241771ef 1736{
2f18d1e8 1737 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
cdd6c482 1738 struct cpu_hw_events *cpuc;
5bb9efe3 1739 unsigned long flags;
1e125676
IM		 1740 int cpu, idx;
1741
cdd6c482 1742 if (!x86_pmu.num_events)
1e125676 1743 return;
241771ef 1744
5bb9efe3 1745 local_irq_save(flags);
241771ef
IM		 1746
1747 cpu = smp_processor_id();
cdd6c482 1748 cpuc = &per_cpu(cpu_hw_events, cpu);
241771ef 1749
faa28ae0 1750 if (x86_pmu.version >= 2) {
a1ef58f4
JSR		 1751 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1752 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1753 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1754 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1755
1756 pr_info("\n");
1757 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1758 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1759 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1760 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
f87ad35d 1761 }
1da53e02 1762 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
241771ef 1763
cdd6c482 1764 for (idx = 0; idx < x86_pmu.num_events; idx++) {
4a06bd85
RR		 1765 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1766 rdmsrl(x86_pmu.perfctr + idx, pmc_count);
241771ef 1767
245b2e70 1768 prev_left = per_cpu(pmc_prev_left[idx], cpu);
241771ef 1769
a1ef58f4 1770 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
241771ef 1771 cpu, idx, pmc_ctrl);
a1ef58f4 1772 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
241771ef 1773 cpu, idx, pmc_count);
a1ef58f4 1774 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
ee06094f 1775 cpu, idx, prev_left);
241771ef 1776 }
cdd6c482 1777 for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
2f18d1e8
IM		 1778 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1779
a1ef58f4 1780 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
2f18d1e8
IM		 1781 cpu, idx, pmc_count);
1782 }
5bb9efe3 1783 local_irq_restore(flags);
241771ef
IM		 1784}
1785
cdd6c482 1786static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
30dd568c
MM		 1787{
1788 struct debug_store *ds = cpuc->ds;
1789 struct bts_record {
1790 u64 from;
1791 u64 to;
1792 u64 flags;
1793 };
cdd6c482 1794 struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
596da17f 1795 struct bts_record *at, *top;
5622f295
MM		 1796 struct perf_output_handle handle;
1797 struct perf_event_header header;
1798 struct perf_sample_data data;
1799 struct pt_regs regs;
30dd568c 1800
cdd6c482 1801 if (!event)
30dd568c
MM		 1802 return;
1803
1804 if (!ds)
1805 return;
1806
596da17f 1807 at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1808 top = (struct bts_record *)(unsigned long)ds->bts_index;
30dd568c 1809
5622f295
MM		 1810 if (top <= at)
1811 return;
1812
596da17f 1813 ds->bts_index = ds->bts_buffer_base;
1814
5622f295 1815
cdd6c482 1816 data.period = event->hw.last_period;
5622f295 1817 data.addr = 0;
5e855db5 1818 data.raw = NULL;
5622f295
MM		 1819 regs.ip = 0;
1820
1821 /*
1822 * Prepare a generic sample, i.e. fill in the invariant fields.
1823 * We will overwrite the from and to address before we output
1824 * the sample.
1825 */
cdd6c482 1826 perf_prepare_sample(&header, &data, event, &regs);
5622f295 1827
cdd6c482 1828 if (perf_output_begin(&handle, event,
5622f295
MM		 1829 header.size * (top - at), 1, 1))
1830 return;
1831
596da17f 1832 for (; at < top; at++) {
5622f295
MM		 1833 data.ip = at->from;
1834 data.addr = at->to;
30dd568c 1835
cdd6c482 1836 perf_output_sample(&handle, &header, &data, event);
30dd568c
MM		 1837 }
1838
5622f295 1839 perf_output_end(&handle);
30dd568c
MM		 1840
1841 /* There's new data available. */
cdd6c482
IM		 1842 event->hw.interrupts++;
1843 event->pending_kill = POLL_IN;
30dd568c
MM		 1844}
1845
cdd6c482 1846static void x86_pmu_disable(struct perf_event *event)
241771ef 1847{
cdd6c482
IM		 1848 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1849 struct hw_perf_event *hwc = &event->hw;
1da53e02 1850 int i, idx = hwc->idx;
241771ef 1851
09534238
RR		 1852 /*
1853 * Must be done before we disable, otherwise the nmi handler
1854 * could reenable again:
1855 */
43f6201a 1856 clear_bit(idx, cpuc->active_mask);
d4369891 1857 x86_pmu.disable(hwc, idx);
241771ef 1858
2f18d1e8
IM		 1859 /*
1860 * Make sure the cleared pointer becomes visible before we
cdd6c482 1861 * (potentially) free the event:
2f18d1e8 1862 */
527e26af 1863 barrier();
241771ef 1864
ee06094f 1865 /*
cdd6c482 1866 * Drain the remaining delta count out of a event
ee06094f
IM		 1867 * that we are disabling:
1868 */
cdd6c482 1869 x86_perf_event_update(event, hwc, idx);
30dd568c
MM		 1870
1871 /* Drain the remaining BTS records. */
5622f295
MM		 1872 if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1873 intel_pmu_drain_bts_buffer(cpuc);
30dd568c 1874
cdd6c482 1875 cpuc->events[idx] = NULL;
194002b2 1876
1da53e02
SE		 1877 for (i = 0; i < cpuc->n_events; i++) {
1878 if (event == cpuc->event_list[i]) {
1879
1880 if (x86_pmu.put_event_constraints)
1881 x86_pmu.put_event_constraints(cpuc, event);
1882
1883 while (++i < cpuc->n_events)
1884 cpuc->event_list[i-1] = cpuc->event_list[i];
1885
1886 --cpuc->n_events;
1887 }
1888 }
cdd6c482 1889 perf_event_update_userpage(event);
241771ef
IM		 1890}
1891
7e2ae347 1892/*
cdd6c482
IM		 1893 * Save and restart an expired event. Called by NMI contexts,
1894 * so it has to be careful about preempting normal event ops:
7e2ae347 1895 */
cdd6c482 1896static int intel_pmu_save_and_restart(struct perf_event *event)
241771ef 1897{
cdd6c482 1898 struct hw_perf_event *hwc = &event->hw;
241771ef 1899 int idx = hwc->idx;
e4abb5d4 1900 int ret;
241771ef 1901
cdd6c482
IM		 1902 x86_perf_event_update(event, hwc, idx);
1903 ret = x86_perf_event_set_period(event, hwc, idx);
7e2ae347 1904
cdd6c482
IM		 1905 if (event->state == PERF_EVENT_STATE_ACTIVE)
1906 intel_pmu_enable_event(hwc, idx);
e4abb5d4
PZ		 1907
1908 return ret;
241771ef
IM		 1909}
1910
aaba9801
IM		 1911static void intel_pmu_reset(void)
1912{
cdd6c482 1913 struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
aaba9801
IM		 1914 unsigned long flags;
1915 int idx;
1916
cdd6c482 1917 if (!x86_pmu.num_events)
aaba9801
IM		 1918 return;
1919
1920 local_irq_save(flags);
1921
1922 printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1923
cdd6c482 1924 for (idx = 0; idx < x86_pmu.num_events; idx++) {
aaba9801
IM		 1925 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1926 checking_wrmsrl(x86_pmu.perfctr + idx, 0ull);
1927 }
cdd6c482 1928 for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
aaba9801
IM		 1929 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1930 }
30dd568c
MM		 1931 if (ds)
1932 ds->bts_index = ds->bts_buffer_base;
aaba9801
IM		 1933
1934 local_irq_restore(flags);
1935}
1936
11d1578f
VW		 1937static int p6_pmu_handle_irq(struct pt_regs *regs)
1938{
1939 struct perf_sample_data data;
cdd6c482
IM		 1940 struct cpu_hw_events *cpuc;
1941 struct perf_event *event;
1942 struct hw_perf_event *hwc;
11d1578f
VW		 1943 int idx, handled = 0;
1944 u64 val;
1945
11d1578f 1946 data.addr = 0;
5e855db5 1947 data.raw = NULL;
11d1578f 1948
cdd6c482 1949 cpuc = &__get_cpu_var(cpu_hw_events);
11d1578f 1950
cdd6c482 1951 for (idx = 0; idx < x86_pmu.num_events; idx++) {
11d1578f
VW		 1952 if (!test_bit(idx, cpuc->active_mask))
1953 continue;
1954
cdd6c482
IM		 1955 event = cpuc->events[idx];
1956 hwc = &event->hw;
11d1578f 1957
cdd6c482
IM		 1958 val = x86_perf_event_update(event, hwc, idx);
1959 if (val & (1ULL << (x86_pmu.event_bits - 1)))
11d1578f
VW		 1960 continue;
1961
1962 /*
cdd6c482 1963 * event overflow
11d1578f
VW		 1964 */
1965 handled = 1;
cdd6c482 1966 data.period = event->hw.last_period;
11d1578f 1967
cdd6c482 1968 if (!x86_perf_event_set_period(event, hwc, idx))
11d1578f
VW		 1969 continue;
1970
cdd6c482
IM		 1971 if (perf_event_overflow(event, 1, &data, regs))
1972 p6_pmu_disable_event(hwc, idx);
11d1578f
VW		 1973 }
1974
1975 if (handled)
1976 inc_irq_stat(apic_perf_irqs);
1977
1978 return handled;
1979}
aaba9801 1980
241771ef
IM		 1981/*
1982 * This handler is triggered by the local APIC, so the APIC IRQ handling
1983 * rules apply:
1984 */
a3288106 1985static int intel_pmu_handle_irq(struct pt_regs *regs)
241771ef 1986{
df1a132b 1987 struct perf_sample_data data;
cdd6c482 1988 struct cpu_hw_events *cpuc;
11d1578f 1989 int bit, loops;
4b39fd96 1990 u64 ack, status;
9029a5e3 1991
df1a132b 1992 data.addr = 0;
5e855db5 1993 data.raw = NULL;
df1a132b 1994
cdd6c482 1995 cpuc = &__get_cpu_var(cpu_hw_events);
241771ef 1996
9e35ad38 1997 perf_disable();
5622f295 1998 intel_pmu_drain_bts_buffer(cpuc);
19d84dab 1999 status = intel_pmu_get_status();
9e35ad38
PZ		 2000 if (!status) {
2001 perf_enable();
2002 return 0;
2003 }
87b9cf46 2004
9029a5e3 2005 loops = 0;
241771ef 2006again:
9029a5e3 2007 if (++loops > 100) {
cdd6c482
IM		 2008 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
2009 perf_event_print_debug();
aaba9801
IM		 2010 intel_pmu_reset();
2011 perf_enable();
9029a5e3
IM		 2012 return 1;
2013 }
2014
d278c484 2015 inc_irq_stat(apic_perf_irqs);
241771ef 2016 ack = status;
2f18d1e8 2017 for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
cdd6c482 2018 struct perf_event *event = cpuc->events[bit];
241771ef
IM		 2019
2020 clear_bit(bit, (unsigned long *) &status);
43f6201a 2021 if (!test_bit(bit, cpuc->active_mask))
241771ef
IM		 2022 continue;
2023
cdd6c482 2024 if (!intel_pmu_save_and_restart(event))
e4abb5d4
PZ		 2025 continue;
2026
cdd6c482 2027 data.period = event->hw.last_period;
60f916de 2028
cdd6c482
IM		 2029 if (perf_event_overflow(event, 1, &data, regs))
2030 intel_pmu_disable_event(&event->hw, bit);
241771ef
IM		 2031 }
2032
dee5d906 2033 intel_pmu_ack_status(ack);
241771ef
IM		 2034
2035 /*
2036 * Repeat if there is more work to be done:
2037 */
19d84dab 2038 status = intel_pmu_get_status();
241771ef
IM		 2039 if (status)
2040 goto again;
b0f3f28e 2041
48e22d56 2042 perf_enable();
9e35ad38
PZ		 2043
2044 return 1;
1b023a96
MG		 2045}
2046
a3288106 2047static int amd_pmu_handle_irq(struct pt_regs *regs)
a29aa8a7 2048{
df1a132b 2049 struct perf_sample_data data;
cdd6c482
IM		 2050 struct cpu_hw_events *cpuc;
2051 struct perf_event *event;
2052 struct hw_perf_event *hwc;
11d1578f 2053 int idx, handled = 0;
9029a5e3
IM		 2054 u64 val;
2055
df1a132b 2056 data.addr = 0;
5e855db5 2057 data.raw = NULL;
df1a132b 2058
cdd6c482 2059 cpuc = &__get_cpu_var(cpu_hw_events);
962bf7a6 2060
cdd6c482 2061 for (idx = 0; idx < x86_pmu.num_events; idx++) {
43f6201a 2062 if (!test_bit(idx, cpuc->active_mask))
a29aa8a7 2063 continue;
962bf7a6 2064
cdd6c482
IM		 2065 event = cpuc->events[idx];
2066 hwc = &event->hw;
a4016a79 2067
cdd6c482
IM		 2068 val = x86_perf_event_update(event, hwc, idx);
2069 if (val & (1ULL << (x86_pmu.event_bits - 1)))
48e22d56 2070 continue;
962bf7a6 2071
9e350de3 2072 /*
cdd6c482 2073 * event overflow
9e350de3
PZ		 2074 */
2075 handled = 1;
cdd6c482 2076 data.period = event->hw.last_period;
9e350de3 2077
cdd6c482 2078 if (!x86_perf_event_set_period(event, hwc, idx))
e4abb5d4
PZ		 2079 continue;
2080
cdd6c482
IM		 2081 if (perf_event_overflow(event, 1, &data, regs))
2082 amd_pmu_disable_event(hwc, idx);
a29aa8a7 2083 }
962bf7a6 2084
9e350de3
PZ		 2085 if (handled)
2086 inc_irq_stat(apic_perf_irqs);
2087
a29aa8a7
RR		 2088 return handled;
2089}
39d81eab 2090
b6276f35
PZ		 2091void smp_perf_pending_interrupt(struct pt_regs *regs)
2092{
2093 irq_enter();
2094 ack_APIC_irq();
2095 inc_irq_stat(apic_pending_irqs);
cdd6c482 2096 perf_event_do_pending();
b6276f35
PZ		 2097 irq_exit();
2098}
2099
cdd6c482 2100void set_perf_event_pending(void)
b6276f35 2101{
04da8a43 2102#ifdef CONFIG_X86_LOCAL_APIC
7d428966
PZ		 2103 if (!x86_pmu.apic || !x86_pmu_initialized())
2104 return;
2105
b6276f35 2106 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
04da8a43 2107#endif
b6276f35
PZ		 2108}
2109
cdd6c482 2110void perf_events_lapic_init(void)
241771ef 2111{
04da8a43
IM		 2112#ifdef CONFIG_X86_LOCAL_APIC
2113 if (!x86_pmu.apic || !x86_pmu_initialized())
241771ef 2114 return;
85cf9dba 2115
241771ef 2116 /*
c323d95f 2117 * Always use NMI for PMU
241771ef 2118 */
c323d95f 2119 apic_write(APIC_LVTPC, APIC_DM_NMI);
04da8a43 2120#endif
241771ef
IM		 2121}
2122
2123static int __kprobes
cdd6c482 2124perf_event_nmi_handler(struct notifier_block *self,
241771ef
IM		 2125 unsigned long cmd, void *__args)
2126{
2127 struct die_args *args = __args;
2128 struct pt_regs *regs;
b0f3f28e 2129
cdd6c482 2130 if (!atomic_read(&active_events))
63a809a2
PZ		 2131 return NOTIFY_DONE;
2132
b0f3f28e
PZ		 2133 switch (cmd) {
2134 case DIE_NMI:
2135 case DIE_NMI_IPI:
2136 break;
241771ef 2137
b0f3f28e 2138 default:
241771ef 2139 return NOTIFY_DONE;
b0f3f28e 2140 }
241771ef
IM		 2141
2142 regs = args->regs;
2143
04da8a43 2144#ifdef CONFIG_X86_LOCAL_APIC
241771ef 2145 apic_write(APIC_LVTPC, APIC_DM_NMI);
04da8a43 2146#endif
a4016a79
PZ		 2147 /*
2148 * Can't rely on the handled return value to say it was our NMI, two
cdd6c482 2149 * events could trigger 'simultaneously' raising two back-to-back NMIs.
a4016a79
PZ		 2150 *
2151 * If the first NMI handles both, the latter will be empty and daze
2152 * the CPU.
2153 */
a3288106 2154 x86_pmu.handle_irq(regs);
241771ef 2155
a4016a79 2156 return NOTIFY_STOP;
241771ef
IM		 2157}
2158
63b14649
PZ		 2159static struct event_constraint unconstrained;
2160
c91e0f5d
PZ		 2161static struct event_constraint bts_constraint =
2162 EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
1da53e02 2163
63b14649
PZ		 2164static struct event_constraint *
2165intel_special_constraints(struct perf_event *event)
1da53e02
SE		 2166{
2167 unsigned int hw_event;
2168
2169 hw_event = event->hw.config & INTEL_ARCH_EVENT_MASK;
2170
2171 if (unlikely((hw_event ==
2172 x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
2173 (event->hw.sample_period == 1))) {
2174
63b14649 2175 return &bts_constraint;
1da53e02 2176 }
63b14649 2177 return NULL;
1da53e02
SE		 2178}
2179
63b14649
PZ		 2180static struct event_constraint *
2181intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1da53e02 2182{
63b14649 2183 struct event_constraint *c;
1da53e02 2184
63b14649
PZ		 2185 c = intel_special_constraints(event);
2186 if (c)
2187 return c;
1da53e02
SE		 2188
2189 if (x86_pmu.event_constraints) {
2190 for_each_event_constraint(c, x86_pmu.event_constraints) {
63b14649
PZ		 2191 if ((event->hw.config & c->cmask) == c->code)
2192 return c;
1da53e02
SE		 2193 }
2194 }
63b14649
PZ		 2195
2196 return &unconstrained;
1da53e02
SE		 2197}
2198
63b14649
PZ		 2199static struct event_constraint *
2200amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1da53e02 2201{
63b14649 2202 return &unconstrained;
1da53e02
SE		 2203}
2204
2205static int x86_event_sched_in(struct perf_event *event,
2206 struct perf_cpu_context *cpuctx, int cpu)
2207{
2208 int ret = 0;
2209
2210 event->state = PERF_EVENT_STATE_ACTIVE;
2211 event->oncpu = cpu;
2212 event->tstamp_running += event->ctx->time - event->tstamp_stopped;
2213
2214 if (!is_x86_event(event))
2215 ret = event->pmu->enable(event);
2216
2217 if (!ret && !is_software_event(event))
2218 cpuctx->active_oncpu++;
2219
2220 if (!ret && event->attr.exclusive)
2221 cpuctx->exclusive = 1;
2222
2223 return ret;
2224}
2225
2226static void x86_event_sched_out(struct perf_event *event,
2227 struct perf_cpu_context *cpuctx, int cpu)
2228{
2229 event->state = PERF_EVENT_STATE_INACTIVE;
2230 event->oncpu = -1;
2231
2232 if (!is_x86_event(event))
2233 event->pmu->disable(event);
2234
2235 event->tstamp_running -= event->ctx->time - event->tstamp_stopped;
2236
2237 if (!is_software_event(event))
2238 cpuctx->active_oncpu--;
2239
2240 if (event->attr.exclusive || !cpuctx->active_oncpu)
2241 cpuctx->exclusive = 0;
2242}
2243
2244/*
2245 * Called to enable a whole group of events.
2246 * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
2247 * Assumes the caller has disabled interrupts and has
2248 * frozen the PMU with hw_perf_save_disable.
2249 *
2250 * called with PMU disabled. If successful and return value 1,
2251 * then guaranteed to call perf_enable() and hw_perf_enable()
2252 */
2253int hw_perf_group_sched_in(struct perf_event *leader,
2254 struct perf_cpu_context *cpuctx,
2255 struct perf_event_context *ctx, int cpu)
2256{
2257 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
2258 struct perf_event *sub;
2259 int assign[X86_PMC_IDX_MAX];
2260 int n0, n1, ret;
2261
2262 /* n0 = total number of events */
2263 n0 = collect_events(cpuc, leader, true);
2264 if (n0 < 0)
2265 return n0;
2266
2267 ret = x86_schedule_events(cpuc, n0, assign);
2268 if (ret)
2269 return ret;
2270
2271 ret = x86_event_sched_in(leader, cpuctx, cpu);
2272 if (ret)
2273 return ret;
2274
2275 n1 = 1;
2276 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
8113070d 2277 if (sub->state > PERF_EVENT_STATE_OFF) {
1da53e02
SE		 2278 ret = x86_event_sched_in(sub, cpuctx, cpu);
2279 if (ret)
2280 goto undo;
2281 ++n1;
2282 }
2283 }
2284 /*
2285 * copy new assignment, now we know it is possible
2286 * will be used by hw_perf_enable()
2287 */
2288 memcpy(cpuc->assign, assign, n0*sizeof(int));
2289
2290 cpuc->n_events = n0;
2291 cpuc->n_added = n1;
2292 ctx->nr_active += n1;
2293
2294 /*
2295 * 1 means successful and events are active
2296 * This is not quite true because we defer
2297 * actual activation until hw_perf_enable() but
2298 * this way we* ensure caller won't try to enable
2299 * individual events
2300 */
2301 return 1;
2302undo:
2303 x86_event_sched_out(leader, cpuctx, cpu);
2304 n0 = 1;
2305 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2306 if (sub->state == PERF_EVENT_STATE_ACTIVE) {
2307 x86_event_sched_out(sub, cpuctx, cpu);
2308 if (++n0 == n1)
2309 break;
2310 }
2311 }
2312 return ret;
2313}
2314
cdd6c482
IM		 2315static __read_mostly struct notifier_block perf_event_nmi_notifier = {
2316 .notifier_call = perf_event_nmi_handler,
5b75af0a
MG		 2317 .next = NULL,
2318 .priority = 1
241771ef
IM		 2319};
2320
db48cccc 2321static __initconst struct x86_pmu p6_pmu = {
11d1578f
VW		 2322 .name = "p6",
2323 .handle_irq = p6_pmu_handle_irq,
2324 .disable_all = p6_pmu_disable_all,
2325 .enable_all = p6_pmu_enable_all,
cdd6c482
IM		 2326 .enable = p6_pmu_enable_event,
2327 .disable = p6_pmu_disable_event,
11d1578f
VW		 2328 .eventsel = MSR_P6_EVNTSEL0,
2329 .perfctr = MSR_P6_PERFCTR0,
2330 .event_map = p6_pmu_event_map,
2331 .raw_event = p6_pmu_raw_event,
2332 .max_events = ARRAY_SIZE(p6_perfmon_event_map),
04da8a43 2333 .apic = 1,
11d1578f
VW		 2334 .max_period = (1ULL << 31) - 1,
2335 .version = 0,
cdd6c482 2336 .num_events = 2,
11d1578f 2337 /*
cdd6c482 2338 * Events have 40 bits implemented. However they are designed such
11d1578f 2339 * that bits [32-39] are sign extensions of bit 31. As such the
cdd6c482 2340 * effective width of a event for P6-like PMU is 32 bits only.
11d1578f
VW		 2341 *
2342 * See IA-32 Intel Architecture Software developer manual Vol 3B
2343 */
cdd6c482
IM		 2344 .event_bits = 32,
2345 .event_mask = (1ULL << 32) - 1,
1da53e02
SE		 2346 .get_event_constraints = intel_get_event_constraints,
2347 .event_constraints = intel_p6_event_constraints
11d1578f
VW		 2348};
2349
db48cccc 2350static __initconst struct x86_pmu intel_pmu = {
faa28ae0 2351 .name = "Intel",
39d81eab 2352 .handle_irq = intel_pmu_handle_irq,
9e35ad38
PZ		 2353 .disable_all = intel_pmu_disable_all,
2354 .enable_all = intel_pmu_enable_all,
cdd6c482
IM		 2355 .enable = intel_pmu_enable_event,
2356 .disable = intel_pmu_disable_event,
b56a3802
JSR		 2357 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
2358 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
5f4ec28f
RR		 2359 .event_map = intel_pmu_event_map,
2360 .raw_event = intel_pmu_raw_event,
b56a3802 2361 .max_events = ARRAY_SIZE(intel_perfmon_event_map),
04da8a43 2362 .apic = 1,
c619b8ff
RR		 2363 /*
2364 * Intel PMCs cannot be accessed sanely above 32 bit width,
2365 * so we install an artificial 1<<31 period regardless of
cdd6c482 2366 * the generic event period:
c619b8ff
RR		 2367 */
2368 .max_period = (1ULL << 31) - 1,
30dd568c
MM		 2369 .enable_bts = intel_pmu_enable_bts,
2370 .disable_bts = intel_pmu_disable_bts,
1da53e02 2371 .get_event_constraints = intel_get_event_constraints
b56a3802
JSR		 2372};
2373
db48cccc 2374static __initconst struct x86_pmu amd_pmu = {
faa28ae0 2375 .name = "AMD",
39d81eab 2376 .handle_irq = amd_pmu_handle_irq,
9e35ad38
PZ		 2377 .disable_all = amd_pmu_disable_all,
2378 .enable_all = amd_pmu_enable_all,
cdd6c482
IM		 2379 .enable = amd_pmu_enable_event,
2380 .disable = amd_pmu_disable_event,
f87ad35d
JSR		 2381 .eventsel = MSR_K7_EVNTSEL0,
2382 .perfctr = MSR_K7_PERFCTR0,
5f4ec28f
RR		 2383 .event_map = amd_pmu_event_map,
2384 .raw_event = amd_pmu_raw_event,
f87ad35d 2385 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
cdd6c482
IM		 2386 .num_events = 4,
2387 .event_bits = 48,
2388 .event_mask = (1ULL << 48) - 1,
04da8a43 2389 .apic = 1,
c619b8ff
RR		 2390 /* use highest bit to detect overflow */
2391 .max_period = (1ULL << 47) - 1,
1da53e02 2392 .get_event_constraints = amd_get_event_constraints
f87ad35d
JSR		 2393};
2394
db48cccc 2395static __init int p6_pmu_init(void)
11d1578f 2396{
11d1578f
VW		 2397 switch (boot_cpu_data.x86_model) {
2398 case 1:
2399 case 3: /* Pentium Pro */
2400 case 5:
2401 case 6: /* Pentium II */
2402 case 7:
2403 case 8:
2404 case 11: /* Pentium III */
11d1578f
VW		 2405 case 9:
2406 case 13:
f1c6a581
DQ		 2407 /* Pentium M */
2408 break;
11d1578f
VW		 2409 default:
2410 pr_cont("unsupported p6 CPU model %d ",
2411 boot_cpu_data.x86_model);
2412 return -ENODEV;
2413 }
2414
04da8a43
IM		 2415 x86_pmu = p6_pmu;
2416
11d1578f
VW		 2417 return 0;
2418}
2419
db48cccc 2420static __init int intel_pmu_init(void)
241771ef 2421{
7bb497bd 2422 union cpuid10_edx edx;
241771ef 2423 union cpuid10_eax eax;
703e937c 2424 unsigned int unused;
7bb497bd 2425 unsigned int ebx;
faa28ae0 2426 int version;
241771ef 2427
11d1578f
VW		 2428 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2429 /* check for P6 processor family */
2430 if (boot_cpu_data.x86 == 6) {
2431 return p6_pmu_init();
2432 } else {
72eae04d 2433 return -ENODEV;
11d1578f
VW		 2434 }
2435 }
da1a776b 2436
241771ef
IM		 2437 /*
2438 * Check whether the Architectural PerfMon supports
dfc65094 2439 * Branch Misses Retired hw_event or not.
241771ef 2440 */
703e937c 2441 cpuid(10, &eax.full, &ebx, &unused, &edx.full);
241771ef 2442 if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
72eae04d 2443 return -ENODEV;
241771ef 2444
faa28ae0
RR		 2445 version = eax.split.version_id;
2446 if (version < 2)
72eae04d 2447 return -ENODEV;
7bb497bd 2448
1123e3ad
IM		 2449 x86_pmu = intel_pmu;
2450 x86_pmu.version = version;
cdd6c482
IM		 2451 x86_pmu.num_events = eax.split.num_events;
2452 x86_pmu.event_bits = eax.split.bit_width;
2453 x86_pmu.event_mask = (1ULL << eax.split.bit_width) - 1;
066d7dea
IM		 2454
2455 /*
cdd6c482
IM		 2456 * Quirk: v2 perfmon does not report fixed-purpose events, so
2457 * assume at least 3 events:
066d7dea 2458 */
cdd6c482 2459 x86_pmu.num_events_fixed = max((int)edx.split.num_events_fixed, 3);
b56a3802 2460
8326f44d 2461 /*
1123e3ad 2462 * Install the hw-cache-events table:
8326f44d
IM		 2463 */
2464 switch (boot_cpu_data.x86_model) {
dc81081b
YW		 2465 case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2466 case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2467 case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2468 case 29: /* six-core 45 nm xeon "Dunnington" */
8326f44d 2469 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
820a6442 2470 sizeof(hw_cache_event_ids));
8326f44d 2471
1da53e02 2472 x86_pmu.event_constraints = intel_core_event_constraints;
1123e3ad 2473 pr_cont("Core2 events, ");
8326f44d 2474 break;
8326f44d
IM		 2475 case 26:
2476 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
820a6442 2477 sizeof(hw_cache_event_ids));
8326f44d 2478
1da53e02 2479 x86_pmu.event_constraints = intel_nehalem_event_constraints;
1123e3ad 2480 pr_cont("Nehalem/Corei7 events, ");
8326f44d
IM		 2481 break;
2482 case 28:
2483 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
820a6442 2484 sizeof(hw_cache_event_ids));
8326f44d 2485
1da53e02 2486 x86_pmu.event_constraints = intel_gen_event_constraints;
1123e3ad 2487 pr_cont("Atom events, ");
8326f44d 2488 break;
1da53e02
SE		 2489 default:
2490 /*
2491 * default constraints for v2 and up
2492 */
2493 x86_pmu.event_constraints = intel_gen_event_constraints;
2494 pr_cont("generic architected perfmon, ");
8326f44d 2495 }
72eae04d 2496 return 0;
b56a3802
JSR		 2497}
2498
db48cccc 2499static __init int amd_pmu_init(void)
f87ad35d 2500{
4d2be126
JSR		 2501 /* Performance-monitoring supported from K7 and later: */
2502 if (boot_cpu_data.x86 < 6)
2503 return -ENODEV;
2504
4a06bd85 2505 x86_pmu = amd_pmu;
f86748e9 2506
f4db43a3
JSR		 2507 /* Events are common for all AMDs */
2508 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2509 sizeof(hw_cache_event_ids));
f86748e9 2510
72eae04d 2511 return 0;
f87ad35d
JSR		 2512}
2513
12558038
CG		 2514static void __init pmu_check_apic(void)
2515{
2516 if (cpu_has_apic)
2517 return;
2518
2519 x86_pmu.apic = 0;
2520 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2521 pr_info("no hardware sampling interrupt available.\n");
2522}
2523
cdd6c482 2524void __init init_hw_perf_events(void)
b56a3802 2525{
72eae04d
RR		 2526 int err;
2527
cdd6c482 2528 pr_info("Performance Events: ");
1123e3ad 2529
b56a3802
JSR		 2530 switch (boot_cpu_data.x86_vendor) {
2531 case X86_VENDOR_INTEL:
72eae04d 2532 err = intel_pmu_init();
b56a3802 2533 break;
f87ad35d 2534 case X86_VENDOR_AMD:
72eae04d 2535 err = amd_pmu_init();
f87ad35d 2536 break;
4138960a
RR		 2537 default:
2538 return;
b56a3802 2539 }
1123e3ad 2540 if (err != 0) {
cdd6c482 2541 pr_cont("no PMU driver, software events only.\n");
b56a3802 2542 return;
1123e3ad 2543 }
b56a3802 2544
12558038
CG		 2545 pmu_check_apic();
2546
1123e3ad 2547 pr_cont("%s PMU driver.\n", x86_pmu.name);
faa28ae0 2548
cdd6c482
IM		 2549 if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2550 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2551 x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2552 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
241771ef 2553 }
cdd6c482
IM		 2554 perf_event_mask = (1 << x86_pmu.num_events) - 1;
2555 perf_max_events = x86_pmu.num_events;
241771ef 2556
cdd6c482
IM		 2557 if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2558 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2559 x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2560 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
703e937c 2561 }
862a1a5f 2562
cdd6c482
IM		 2563 perf_event_mask |=
2564 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2565 x86_pmu.intel_ctrl = perf_event_mask;
241771ef 2566
cdd6c482
IM		 2567 perf_events_lapic_init();
2568 register_die_notifier(&perf_event_nmi_notifier);
1123e3ad 2569
63b14649
PZ		 2570 unconstrained = (struct event_constraint)
2571 EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_events) - 1, 0);
2572
57c0c15b
IM		 2573 pr_info("... version: %d\n", x86_pmu.version);
2574 pr_info("... bit width: %d\n", x86_pmu.event_bits);
2575 pr_info("... generic registers: %d\n", x86_pmu.num_events);
2576 pr_info("... value mask: %016Lx\n", x86_pmu.event_mask);
2577 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
2578 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_events_fixed);
2579 pr_info("... event mask: %016Lx\n", perf_event_mask);
241771ef 2580}
621a01ea 2581
cdd6c482 2582static inline void x86_pmu_read(struct perf_event *event)
ee06094f 2583{
cdd6c482 2584 x86_perf_event_update(event, &event->hw, event->hw.idx);
ee06094f
IM		 2585}
2586
4aeb0b42
RR		 2587static const struct pmu pmu = {
2588 .enable = x86_pmu_enable,
2589 .disable = x86_pmu_disable,
2590 .read = x86_pmu_read,
a78ac325 2591 .unthrottle = x86_pmu_unthrottle,
621a01ea
IM		 2592};
2593
1da53e02
SE		 2594/*
2595 * validate a single event group
2596 *
2597 * validation include:
2598 * - check events are compatible which each other
2599 * - events do not compete for the same counter
2600 * - number of events <= number of counters
2601 *
2602 * validation ensures the group can be loaded onto the
2603 * PMU if it was the only group available.
2604 */
fe9081cc
PZ		 2605static int validate_group(struct perf_event *event)
2606{
1da53e02 2607 struct perf_event *leader = event->group_leader;
502568d5
PZ		 2608 struct cpu_hw_events *fake_cpuc;
2609 int ret, n;
fe9081cc 2610
502568d5
PZ		 2611 ret = -ENOMEM;
2612 fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO);
2613 if (!fake_cpuc)
2614 goto out;
fe9081cc 2615
1da53e02
SE		 2616 /*
2617 * the event is not yet connected with its
2618 * siblings therefore we must first collect
2619 * existing siblings, then add the new event
2620 * before we can simulate the scheduling
2621 */
502568d5
PZ		 2622 ret = -ENOSPC;
2623 n = collect_events(fake_cpuc, leader, true);
1da53e02 2624 if (n < 0)
502568d5 2625 goto out_free;
fe9081cc 2626
502568d5
PZ		 2627 fake_cpuc->n_events = n;
2628 n = collect_events(fake_cpuc, event, false);
1da53e02 2629 if (n < 0)
502568d5 2630 goto out_free;
fe9081cc 2631
502568d5 2632 fake_cpuc->n_events = n;
1da53e02 2633
502568d5
PZ		 2634 ret = x86_schedule_events(fake_cpuc, n, NULL);
2635
2636out_free:
2637 kfree(fake_cpuc);
2638out:
2639 return ret;
fe9081cc
PZ		 2640}
2641
cdd6c482 2642const struct pmu *hw_perf_event_init(struct perf_event *event)
621a01ea 2643{
8113070d 2644 const struct pmu *tmp;
621a01ea
IM		 2645 int err;
2646
cdd6c482 2647 err = __hw_perf_event_init(event);
fe9081cc 2648 if (!err) {
8113070d
SE		 2649 /*
2650 * we temporarily connect event to its pmu
2651 * such that validate_group() can classify
2652 * it as an x86 event using is_x86_event()
2653 */
2654 tmp = event->pmu;
2655 event->pmu = &pmu;
2656
fe9081cc
PZ		 2657 if (event->group_leader != event)
2658 err = validate_group(event);
8113070d
SE		 2659
2660 event->pmu = tmp;
fe9081cc 2661 }
a1792cda 2662 if (err) {
cdd6c482
IM		 2663 if (event->destroy)
2664 event->destroy(event);
9ea98e19 2665 return ERR_PTR(err);
a1792cda 2666 }
621a01ea 2667
4aeb0b42 2668 return &pmu;
621a01ea 2669}
d7d59fb3
PZ		 2670
2671/*
2672 * callchain support
2673 */
2674
2675static inline
f9188e02 2676void callchain_store(struct perf_callchain_entry *entry, u64 ip)
d7d59fb3 2677{
f9188e02 2678 if (entry->nr < PERF_MAX_STACK_DEPTH)
d7d59fb3
PZ		 2679 entry->ip[entry->nr++] = ip;
2680}
2681
245b2e70
TH		 2682static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2683static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
d7d59fb3
PZ		 2684
2685
2686static void
2687backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2688{
2689 /* Ignore warnings */
2690}
2691
2692static void backtrace_warning(void *data, char *msg)
2693{
2694 /* Ignore warnings */
2695}
2696
2697static int backtrace_stack(void *data, char *name)
2698{
038e836e 2699 return 0;
d7d59fb3
PZ		 2700}
2701
2702static void backtrace_address(void *data, unsigned long addr, int reliable)
2703{
2704 struct perf_callchain_entry *entry = data;
2705
2706 if (reliable)
2707 callchain_store(entry, addr);
2708}
2709
2710static const struct stacktrace_ops backtrace_ops = {
2711 .warning = backtrace_warning,
2712 .warning_symbol = backtrace_warning_symbol,
2713 .stack = backtrace_stack,
2714 .address = backtrace_address,
06d65bda 2715 .walk_stack = print_context_stack_bp,
d7d59fb3
PZ		 2716};
2717
038e836e
IM		 2718#include "../dumpstack.h"
2719
d7d59fb3
PZ		 2720static void
2721perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2722{
f9188e02 2723 callchain_store(entry, PERF_CONTEXT_KERNEL);
038e836e 2724 callchain_store(entry, regs->ip);
d7d59fb3 2725
48b5ba9c 2726 dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
d7d59fb3
PZ		 2727}
2728
74193ef0
PZ		 2729/*
2730 * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2731 */
2732static unsigned long
2733copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
d7d59fb3 2734{
74193ef0
PZ		 2735 unsigned long offset, addr = (unsigned long)from;
2736 int type = in_nmi() ? KM_NMI : KM_IRQ0;
2737 unsigned long size, len = 0;
2738 struct page *page;
2739 void *map;
d7d59fb3
PZ		 2740 int ret;
2741
74193ef0
PZ		 2742 do {
2743 ret = __get_user_pages_fast(addr, 1, 0, &page);
2744 if (!ret)
2745 break;
d7d59fb3 2746
74193ef0
PZ		 2747 offset = addr & (PAGE_SIZE - 1);
2748 size = min(PAGE_SIZE - offset, n - len);
d7d59fb3 2749
74193ef0
PZ		 2750 map = kmap_atomic(page, type);
2751 memcpy(to, map+offset, size);
2752 kunmap_atomic(map, type);
2753 put_page(page);
2754
2755 len += size;
2756 to += size;
2757 addr += size;
2758
2759 } while (len < n);
2760
2761 return len;
2762}
2763
2764static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2765{
2766 unsigned long bytes;
2767
2768 bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2769
2770 return bytes == sizeof(*frame);
d7d59fb3
PZ		 2771}
2772
2773static void
2774perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2775{
2776 struct stack_frame frame;
2777 const void __user *fp;
2778
5a6cec3a
IM		 2779 if (!user_mode(regs))
2780 regs = task_pt_regs(current);
2781
74193ef0 2782 fp = (void __user *)regs->bp;
d7d59fb3 2783
f9188e02 2784 callchain_store(entry, PERF_CONTEXT_USER);
d7d59fb3
PZ		 2785 callchain_store(entry, regs->ip);
2786
f9188e02 2787 while (entry->nr < PERF_MAX_STACK_DEPTH) {
038e836e 2788 frame.next_frame = NULL;
d7d59fb3
PZ		 2789 frame.return_address = 0;
2790
2791 if (!copy_stack_frame(fp, &frame))
2792 break;
2793
5a6cec3a 2794 if ((unsigned long)fp < regs->sp)
d7d59fb3
PZ		 2795 break;
2796
2797 callchain_store(entry, frame.return_address);
038e836e 2798 fp = frame.next_frame;
d7d59fb3
PZ		 2799 }
2800}
2801
2802static void
2803perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2804{
2805 int is_user;
2806
2807 if (!regs)
2808 return;
2809
2810 is_user = user_mode(regs);
2811
d7d59fb3
PZ		 2812 if (is_user && current->state != TASK_RUNNING)
2813 return;
2814
2815 if (!is_user)
2816 perf_callchain_kernel(regs, entry);
2817
2818 if (current->mm)
2819 perf_callchain_user(regs, entry);
2820}
2821
2822struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2823{
2824 struct perf_callchain_entry *entry;
2825
2826 if (in_nmi())
245b2e70 2827 entry = &__get_cpu_var(pmc_nmi_entry);
d7d59fb3 2828 else
245b2e70 2829 entry = &__get_cpu_var(pmc_irq_entry);
d7d59fb3
PZ		 2830
2831 entry->nr = 0;
2832
2833 perf_do_callchain(regs, entry);
2834
2835 return entry;
2836}
30dd568c 2837
cdd6c482 2838void hw_perf_event_setup_online(int cpu)
30dd568c
MM		 2839{
2840 init_debug_store_on_cpu(cpu);
2841}
Linux 6.x block layer and io_uring tree(s)