projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf_counter: Fix stack corruption in perf_read_hw
[linux-block.git] / arch / x86 / kernel / cpu / perf_counter.c
CommitLineData
241771ef
IM		 1/*
2 * Performance counter x86 architecture code
3 *
98144511
IM		 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
241771ef
IM		 9 *
10 * For licencing details see kernel-base/COPYING
11 */
12
13#include <linux/perf_counter.h>
14#include <linux/capability.h>
15#include <linux/notifier.h>
16#include <linux/hardirq.h>
17#include <linux/kprobes.h>
4ac13294 18#include <linux/module.h>
241771ef
IM		 19#include <linux/kdebug.h>
20#include <linux/sched.h>
d7d59fb3 21#include <linux/uaccess.h>
241771ef 22
241771ef 23#include <asm/apic.h>
d7d59fb3 24#include <asm/stacktrace.h>
4e935e47 25#include <asm/nmi.h>
241771ef 26
862a1a5f 27static u64 perf_counter_mask __read_mostly;
703e937c 28
241771ef 29struct cpu_hw_counters {
862a1a5f 30 struct perf_counter *counters[X86_PMC_IDX_MAX];
43f6201a
RR		 31 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
32 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
4b39fd96 33 unsigned long interrupts;
b0f3f28e 34 int enabled;
241771ef
IM		 35};
36
37/*
5f4ec28f 38 * struct x86_pmu - generic x86 pmu
241771ef 39 */
5f4ec28f 40struct x86_pmu {
faa28ae0
RR		 41 const char *name;
42 int version;
a3288106 43 int (*handle_irq)(struct pt_regs *);
9e35ad38
PZ		 44 void (*disable_all)(void);
45 void (*enable_all)(void);
7c90cc45 46 void (*enable)(struct hw_perf_counter *, int);
d4369891 47 void (*disable)(struct hw_perf_counter *, int);
169e41eb
JSR		 48 unsigned eventsel;
49 unsigned perfctr;
b0f3f28e
PZ		 50 u64 (*event_map)(int);
51 u64 (*raw_event)(u64);
169e41eb 52 int max_events;
0933e5c6
RR		 53 int num_counters;
54 int num_counters_fixed;
55 int counter_bits;
56 u64 counter_mask;
c619b8ff 57 u64 max_period;
9e35ad38 58 u64 intel_ctrl;
b56a3802
JSR		 59};
60
4a06bd85 61static struct x86_pmu x86_pmu __read_mostly;
b56a3802 62
b0f3f28e
PZ		 63static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
64 .enabled = 1,
65};
241771ef 66
b56a3802
JSR		 67/*
68 * Intel PerfMon v3. Used on Core2 and later.
69 */
b0f3f28e 70static const u64 intel_perfmon_event_map[] =
241771ef 71{
f4dbfa8f
PZ		 72 [PERF_COUNT_HW_CPU_CYCLES] = 0x003c,
73 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
74 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e,
75 [PERF_COUNT_HW_CACHE_MISSES] = 0x412e,
76 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
77 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
78 [PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
241771ef
IM		 79};
80
5f4ec28f 81static u64 intel_pmu_event_map(int event)
b56a3802
JSR		 82{
83 return intel_perfmon_event_map[event];
84}
241771ef 85
8326f44d
IM		 86/*
87 * Generalized hw caching related event table, filled
88 * in on a per model basis. A value of 0 means
89 * 'not supported', -1 means 'event makes no sense on
90 * this CPU', any other value means the raw event
91 * ID.
92 */
93
94#define C(x) PERF_COUNT_HW_CACHE_##x
95
96static u64 __read_mostly hw_cache_event_ids
97 [PERF_COUNT_HW_CACHE_MAX]
98 [PERF_COUNT_HW_CACHE_OP_MAX]
99 [PERF_COUNT_HW_CACHE_RESULT_MAX];
100
101static const u64 nehalem_hw_cache_event_ids
102 [PERF_COUNT_HW_CACHE_MAX]
103 [PERF_COUNT_HW_CACHE_OP_MAX]
104 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
105{
106 [ C(L1D) ] = {
107 [ C(OP_READ) ] = {
108 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
109 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
110 },
111 [ C(OP_WRITE) ] = {
112 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
113 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
114 },
115 [ C(OP_PREFETCH) ] = {
116 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */
117 [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */
118 },
119 },
120 [ C(L1I ) ] = {
121 [ C(OP_READ) ] = {
fecc8ac8 122 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
8326f44d
IM		 123 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
124 },
125 [ C(OP_WRITE) ] = {
126 [ C(RESULT_ACCESS) ] = -1,
127 [ C(RESULT_MISS) ] = -1,
128 },
129 [ C(OP_PREFETCH) ] = {
130 [ C(RESULT_ACCESS) ] = 0x0,
131 [ C(RESULT_MISS) ] = 0x0,
132 },
133 },
8be6e8f3 134 [ C(LL ) ] = {
8326f44d
IM		 135 [ C(OP_READ) ] = {
136 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS */
137 [ C(RESULT_MISS) ] = 0x0224, /* L2_RQSTS.LD_MISS */
138 },
139 [ C(OP_WRITE) ] = {
140 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS */
141 [ C(RESULT_MISS) ] = 0x0824, /* L2_RQSTS.RFO_MISS */
142 },
143 [ C(OP_PREFETCH) ] = {
8be6e8f3
PZ		 144 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference */
145 [ C(RESULT_MISS) ] = 0x412e, /* LLC Misses */
8326f44d
IM		 146 },
147 },
148 [ C(DTLB) ] = {
149 [ C(OP_READ) ] = {
150 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
151 [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */
152 },
153 [ C(OP_WRITE) ] = {
154 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
155 [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */
156 },
157 [ C(OP_PREFETCH) ] = {
158 [ C(RESULT_ACCESS) ] = 0x0,
159 [ C(RESULT_MISS) ] = 0x0,
160 },
161 },
162 [ C(ITLB) ] = {
163 [ C(OP_READ) ] = {
164 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */
fecc8ac8 165 [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */
8326f44d
IM		 166 },
167 [ C(OP_WRITE) ] = {
168 [ C(RESULT_ACCESS) ] = -1,
169 [ C(RESULT_MISS) ] = -1,
170 },
171 [ C(OP_PREFETCH) ] = {
172 [ C(RESULT_ACCESS) ] = -1,
173 [ C(RESULT_MISS) ] = -1,
174 },
175 },
176 [ C(BPU ) ] = {
177 [ C(OP_READ) ] = {
178 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
179 [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */
180 },
181 [ C(OP_WRITE) ] = {
182 [ C(RESULT_ACCESS) ] = -1,
183 [ C(RESULT_MISS) ] = -1,
184 },
185 [ C(OP_PREFETCH) ] = {
186 [ C(RESULT_ACCESS) ] = -1,
187 [ C(RESULT_MISS) ] = -1,
188 },
189 },
190};
191
192static const u64 core2_hw_cache_event_ids
193 [PERF_COUNT_HW_CACHE_MAX]
194 [PERF_COUNT_HW_CACHE_OP_MAX]
195 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
196{
0312af84
TG		 197 [ C(L1D) ] = {
198 [ C(OP_READ) ] = {
199 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */
200 [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */
201 },
202 [ C(OP_WRITE) ] = {
203 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */
204 [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */
205 },
206 [ C(OP_PREFETCH) ] = {
207 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */
208 [ C(RESULT_MISS) ] = 0,
209 },
210 },
211 [ C(L1I ) ] = {
212 [ C(OP_READ) ] = {
213 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */
214 [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */
215 },
216 [ C(OP_WRITE) ] = {
217 [ C(RESULT_ACCESS) ] = -1,
218 [ C(RESULT_MISS) ] = -1,
219 },
220 [ C(OP_PREFETCH) ] = {
221 [ C(RESULT_ACCESS) ] = 0,
222 [ C(RESULT_MISS) ] = 0,
223 },
224 },
8be6e8f3 225 [ C(LL ) ] = {
0312af84
TG		 226 [ C(OP_READ) ] = {
227 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
228 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
229 },
230 [ C(OP_WRITE) ] = {
231 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
232 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
233 },
234 [ C(OP_PREFETCH) ] = {
235 [ C(RESULT_ACCESS) ] = 0,
236 [ C(RESULT_MISS) ] = 0,
237 },
238 },
239 [ C(DTLB) ] = {
240 [ C(OP_READ) ] = {
241 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */
242 [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */
243 },
244 [ C(OP_WRITE) ] = {
245 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */
246 [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */
247 },
248 [ C(OP_PREFETCH) ] = {
249 [ C(RESULT_ACCESS) ] = 0,
250 [ C(RESULT_MISS) ] = 0,
251 },
252 },
253 [ C(ITLB) ] = {
254 [ C(OP_READ) ] = {
255 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
256 [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */
257 },
258 [ C(OP_WRITE) ] = {
259 [ C(RESULT_ACCESS) ] = -1,
260 [ C(RESULT_MISS) ] = -1,
261 },
262 [ C(OP_PREFETCH) ] = {
263 [ C(RESULT_ACCESS) ] = -1,
264 [ C(RESULT_MISS) ] = -1,
265 },
266 },
267 [ C(BPU ) ] = {
268 [ C(OP_READ) ] = {
269 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
270 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
271 },
272 [ C(OP_WRITE) ] = {
273 [ C(RESULT_ACCESS) ] = -1,
274 [ C(RESULT_MISS) ] = -1,
275 },
276 [ C(OP_PREFETCH) ] = {
277 [ C(RESULT_ACCESS) ] = -1,
278 [ C(RESULT_MISS) ] = -1,
279 },
280 },
8326f44d
IM		 281};
282
283static const u64 atom_hw_cache_event_ids
284 [PERF_COUNT_HW_CACHE_MAX]
285 [PERF_COUNT_HW_CACHE_OP_MAX]
286 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
287{
ad689220
TG		 288 [ C(L1D) ] = {
289 [ C(OP_READ) ] = {
290 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */
291 [ C(RESULT_MISS) ] = 0,
292 },
293 [ C(OP_WRITE) ] = {
fecc8ac8 294 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */
ad689220
TG		 295 [ C(RESULT_MISS) ] = 0,
296 },
297 [ C(OP_PREFETCH) ] = {
298 [ C(RESULT_ACCESS) ] = 0x0,
299 [ C(RESULT_MISS) ] = 0,
300 },
301 },
302 [ C(L1I ) ] = {
303 [ C(OP_READ) ] = {
fecc8ac8
YW		 304 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */
305 [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */
ad689220
TG		 306 },
307 [ C(OP_WRITE) ] = {
308 [ C(RESULT_ACCESS) ] = -1,
309 [ C(RESULT_MISS) ] = -1,
310 },
311 [ C(OP_PREFETCH) ] = {
312 [ C(RESULT_ACCESS) ] = 0,
313 [ C(RESULT_MISS) ] = 0,
314 },
315 },
8be6e8f3 316 [ C(LL ) ] = {
ad689220
TG		 317 [ C(OP_READ) ] = {
318 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */
319 [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */
320 },
321 [ C(OP_WRITE) ] = {
322 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */
323 [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */
324 },
325 [ C(OP_PREFETCH) ] = {
326 [ C(RESULT_ACCESS) ] = 0,
327 [ C(RESULT_MISS) ] = 0,
328 },
329 },
330 [ C(DTLB) ] = {
331 [ C(OP_READ) ] = {
fecc8ac8 332 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */
ad689220
TG		 333 [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */
334 },
335 [ C(OP_WRITE) ] = {
fecc8ac8 336 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */
ad689220
TG		 337 [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */
338 },
339 [ C(OP_PREFETCH) ] = {
340 [ C(RESULT_ACCESS) ] = 0,
341 [ C(RESULT_MISS) ] = 0,
342 },
343 },
344 [ C(ITLB) ] = {
345 [ C(OP_READ) ] = {
346 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
347 [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */
348 },
349 [ C(OP_WRITE) ] = {
350 [ C(RESULT_ACCESS) ] = -1,
351 [ C(RESULT_MISS) ] = -1,
352 },
353 [ C(OP_PREFETCH) ] = {
354 [ C(RESULT_ACCESS) ] = -1,
355 [ C(RESULT_MISS) ] = -1,
356 },
357 },
358 [ C(BPU ) ] = {
359 [ C(OP_READ) ] = {
360 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
361 [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
362 },
363 [ C(OP_WRITE) ] = {
364 [ C(RESULT_ACCESS) ] = -1,
365 [ C(RESULT_MISS) ] = -1,
366 },
367 [ C(OP_PREFETCH) ] = {
368 [ C(RESULT_ACCESS) ] = -1,
369 [ C(RESULT_MISS) ] = -1,
370 },
371 },
8326f44d
IM		 372};
373
5f4ec28f 374static u64 intel_pmu_raw_event(u64 event)
b0f3f28e 375{
82bae4f8
PZ		 376#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
377#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
ff99be57
PZ		 378#define CORE_EVNTSEL_EDGE_MASK 0x00040000ULL
379#define CORE_EVNTSEL_INV_MASK 0x00800000ULL
82bae4f8 380#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL
b0f3f28e 381
128f048f 382#define CORE_EVNTSEL_MASK \
b0f3f28e
PZ		 383 (CORE_EVNTSEL_EVENT_MASK | \
384 CORE_EVNTSEL_UNIT_MASK | \
ff99be57
PZ		 385 CORE_EVNTSEL_EDGE_MASK | \
386 CORE_EVNTSEL_INV_MASK | \
b0f3f28e
PZ		 387 CORE_EVNTSEL_COUNTER_MASK)
388
389 return event & CORE_EVNTSEL_MASK;
390}
391
f86748e9
TG		 392static const u64 amd_0f_hw_cache_event_ids
393 [PERF_COUNT_HW_CACHE_MAX]
394 [PERF_COUNT_HW_CACHE_OP_MAX]
395 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
396{
397 [ C(L1D) ] = {
398 [ C(OP_READ) ] = {
399 [ C(RESULT_ACCESS) ] = 0,
400 [ C(RESULT_MISS) ] = 0,
401 },
402 [ C(OP_WRITE) ] = {
403 [ C(RESULT_ACCESS) ] = 0,
404 [ C(RESULT_MISS) ] = 0,
405 },
406 [ C(OP_PREFETCH) ] = {
407 [ C(RESULT_ACCESS) ] = 0,
408 [ C(RESULT_MISS) ] = 0,
409 },
410 },
411 [ C(L1I ) ] = {
412 [ C(OP_READ) ] = {
413 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches */
414 [ C(RESULT_MISS) ] = 0x0081, /* Instruction cache misses */
415 },
416 [ C(OP_WRITE) ] = {
417 [ C(RESULT_ACCESS) ] = -1,
418 [ C(RESULT_MISS) ] = -1,
419 },
420 [ C(OP_PREFETCH) ] = {
421 [ C(RESULT_ACCESS) ] = 0,
422 [ C(RESULT_MISS) ] = 0,
423 },
424 },
8be6e8f3 425 [ C(LL ) ] = {
f86748e9
TG		 426 [ C(OP_READ) ] = {
427 [ C(RESULT_ACCESS) ] = 0,
428 [ C(RESULT_MISS) ] = 0,
429 },
430 [ C(OP_WRITE) ] = {
431 [ C(RESULT_ACCESS) ] = 0,
432 [ C(RESULT_MISS) ] = 0,
433 },
434 [ C(OP_PREFETCH) ] = {
435 [ C(RESULT_ACCESS) ] = 0,
436 [ C(RESULT_MISS) ] = 0,
437 },
438 },
439 [ C(DTLB) ] = {
440 [ C(OP_READ) ] = {
441 [ C(RESULT_ACCESS) ] = 0,
442 [ C(RESULT_MISS) ] = 0,
443 },
444 [ C(OP_WRITE) ] = {
445 [ C(RESULT_ACCESS) ] = 0,
446 [ C(RESULT_MISS) ] = 0,
447 },
448 [ C(OP_PREFETCH) ] = {
449 [ C(RESULT_ACCESS) ] = 0,
450 [ C(RESULT_MISS) ] = 0,
451 },
452 },
453 [ C(ITLB) ] = {
454 [ C(OP_READ) ] = {
455 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes */
456 [ C(RESULT_MISS) ] = 0x0085, /* Instr. fetch ITLB misses */
457 },
458 [ C(OP_WRITE) ] = {
459 [ C(RESULT_ACCESS) ] = -1,
460 [ C(RESULT_MISS) ] = -1,
461 },
462 [ C(OP_PREFETCH) ] = {
463 [ C(RESULT_ACCESS) ] = -1,
464 [ C(RESULT_MISS) ] = -1,
465 },
466 },
467 [ C(BPU ) ] = {
468 [ C(OP_READ) ] = {
469 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr. */
470 [ C(RESULT_MISS) ] = 0x00c3, /* Retired Mispredicted BI */
471 },
472 [ C(OP_WRITE) ] = {
473 [ C(RESULT_ACCESS) ] = -1,
474 [ C(RESULT_MISS) ] = -1,
475 },
476 [ C(OP_PREFETCH) ] = {
477 [ C(RESULT_ACCESS) ] = -1,
478 [ C(RESULT_MISS) ] = -1,
479 },
480 },
481};
482
f87ad35d
JSR		 483/*
484 * AMD Performance Monitor K7 and later.
485 */
b0f3f28e 486static const u64 amd_perfmon_event_map[] =
f87ad35d 487{
f4dbfa8f
PZ		 488 [PERF_COUNT_HW_CPU_CYCLES] = 0x0076,
489 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
490 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0080,
491 [PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
492 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4,
493 [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5,
f87ad35d
JSR		 494};
495
5f4ec28f 496static u64 amd_pmu_event_map(int event)
f87ad35d
JSR		 497{
498 return amd_perfmon_event_map[event];
499}
500
5f4ec28f 501static u64 amd_pmu_raw_event(u64 event)
b0f3f28e 502{
82bae4f8
PZ		 503#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
504#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
ff99be57
PZ		 505#define K7_EVNTSEL_EDGE_MASK 0x000040000ULL
506#define K7_EVNTSEL_INV_MASK 0x000800000ULL
82bae4f8 507#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
b0f3f28e
PZ		 508
509#define K7_EVNTSEL_MASK \
510 (K7_EVNTSEL_EVENT_MASK | \
511 K7_EVNTSEL_UNIT_MASK | \
ff99be57
PZ		 512 K7_EVNTSEL_EDGE_MASK | \
513 K7_EVNTSEL_INV_MASK | \
b0f3f28e
PZ		 514 K7_EVNTSEL_COUNTER_MASK)
515
516 return event & K7_EVNTSEL_MASK;
517}
518
ee06094f
IM		 519/*
520 * Propagate counter elapsed time into the generic counter.
521 * Can only be executed on the CPU where the counter is active.
522 * Returns the delta events processed.
523 */
4b7bfd0d 524static u64
ee06094f
IM		 525x86_perf_counter_update(struct perf_counter *counter,
526 struct hw_perf_counter *hwc, int idx)
527{
ec3232bd
PZ		 528 int shift = 64 - x86_pmu.counter_bits;
529 u64 prev_raw_count, new_raw_count;
530 s64 delta;
ee06094f 531
ee06094f
IM		 532 /*
533 * Careful: an NMI might modify the previous counter value.
534 *
535 * Our tactic to handle this is to first atomically read and
536 * exchange a new raw count - then add that new-prev delta
537 * count to the generic counter atomically:
538 */
539again:
540 prev_raw_count = atomic64_read(&hwc->prev_count);
541 rdmsrl(hwc->counter_base + idx, new_raw_count);
542
543 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
544 new_raw_count) != prev_raw_count)
545 goto again;
546
547 /*
548 * Now we have the new raw value and have updated the prev
549 * timestamp already. We can now calculate the elapsed delta
550 * (counter-)time and add that to the generic counter.
551 *
552 * Careful, not all hw sign-extends above the physical width
ec3232bd 553 * of the count.
ee06094f 554 */
ec3232bd
PZ		 555 delta = (new_raw_count << shift) - (prev_raw_count << shift);
556 delta >>= shift;
ee06094f
IM		 557
558 atomic64_add(delta, &counter->count);
559 atomic64_sub(delta, &hwc->period_left);
4b7bfd0d
RR		 560
561 return new_raw_count;
ee06094f
IM		 562}
563
ba77813a 564static atomic_t active_counters;
4e935e47
PZ		 565static DEFINE_MUTEX(pmc_reserve_mutex);
566
567static bool reserve_pmc_hardware(void)
568{
569 int i;
570
571 if (nmi_watchdog == NMI_LOCAL_APIC)
572 disable_lapic_nmi_watchdog();
573
0933e5c6 574 for (i = 0; i < x86_pmu.num_counters; i++) {
4a06bd85 575 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
4e935e47
PZ		 576 goto perfctr_fail;
577 }
578
0933e5c6 579 for (i = 0; i < x86_pmu.num_counters; i++) {
4a06bd85 580 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
4e935e47
PZ		 581 goto eventsel_fail;
582 }
583
584 return true;
585
586eventsel_fail:
587 for (i--; i >= 0; i--)
4a06bd85 588 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47 589
0933e5c6 590 i = x86_pmu.num_counters;
4e935e47
PZ		 591
592perfctr_fail:
593 for (i--; i >= 0; i--)
4a06bd85 594 release_perfctr_nmi(x86_pmu.perfctr + i);
4e935e47
PZ		 595
596 if (nmi_watchdog == NMI_LOCAL_APIC)
597 enable_lapic_nmi_watchdog();
598
599 return false;
600}
601
602static void release_pmc_hardware(void)
603{
604 int i;
605
0933e5c6 606 for (i = 0; i < x86_pmu.num_counters; i++) {
4a06bd85
RR		 607 release_perfctr_nmi(x86_pmu.perfctr + i);
608 release_evntsel_nmi(x86_pmu.eventsel + i);
4e935e47
PZ		 609 }
610
611 if (nmi_watchdog == NMI_LOCAL_APIC)
612 enable_lapic_nmi_watchdog();
613}
614
615static void hw_perf_counter_destroy(struct perf_counter *counter)
616{
ba77813a 617 if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
4e935e47
PZ		 618 release_pmc_hardware();
619 mutex_unlock(&pmc_reserve_mutex);
620 }
621}
622
85cf9dba
RR		 623static inline int x86_pmu_initialized(void)
624{
625 return x86_pmu.handle_irq != NULL;
626}
627
8326f44d
IM		 628static inline int
629set_ext_hw_attr(struct hw_perf_counter *hwc, struct perf_counter_attr *attr)
630{
631 unsigned int cache_type, cache_op, cache_result;
632 u64 config, val;
633
634 config = attr->config;
635
636 cache_type = (config >> 0) & 0xff;
637 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
638 return -EINVAL;
639
640 cache_op = (config >> 8) & 0xff;
641 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
642 return -EINVAL;
643
644 cache_result = (config >> 16) & 0xff;
645 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
646 return -EINVAL;
647
648 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
649
650 if (val == 0)
651 return -ENOENT;
652
653 if (val == -1)
654 return -EINVAL;
655
656 hwc->config |= val;
657
658 return 0;
659}
660
241771ef 661/*
0d48696f 662 * Setup the hardware configuration for a given attr_type
241771ef 663 */
621a01ea 664static int __hw_perf_counter_init(struct perf_counter *counter)
241771ef 665{
0d48696f 666 struct perf_counter_attr *attr = &counter->attr;
241771ef 667 struct hw_perf_counter *hwc = &counter->hw;
4e935e47 668 int err;
241771ef 669
85cf9dba
RR		 670 if (!x86_pmu_initialized())
671 return -ENODEV;
241771ef 672
4e935e47 673 err = 0;
ba77813a 674 if (!atomic_inc_not_zero(&active_counters)) {
4e935e47 675 mutex_lock(&pmc_reserve_mutex);
ba77813a 676 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
4e935e47
PZ		 677 err = -EBUSY;
678 else
ba77813a 679 atomic_inc(&active_counters);
4e935e47
PZ		 680 mutex_unlock(&pmc_reserve_mutex);
681 }
682 if (err)
683 return err;
684
241771ef 685 /*
0475f9ea 686 * Generate PMC IRQs:
241771ef
IM		 687 * (keep 'enabled' bit clear for now)
688 */
0475f9ea 689 hwc->config = ARCH_PERFMON_EVENTSEL_INT;
241771ef
IM		 690
691 /*
0475f9ea 692 * Count user and OS events unless requested not to.
241771ef 693 */
0d48696f 694 if (!attr->exclude_user)
0475f9ea 695 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
0d48696f 696 if (!attr->exclude_kernel)
241771ef 697 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
0475f9ea 698
bd2b5b12 699 if (!hwc->sample_period) {
b23f3325 700 hwc->sample_period = x86_pmu.max_period;
9e350de3 701 hwc->last_period = hwc->sample_period;
bd2b5b12
PZ		 702 atomic64_set(&hwc->period_left, hwc->sample_period);
703 }
d2517a49 704
8326f44d 705 counter->destroy = hw_perf_counter_destroy;
241771ef
IM		 706
707 /*
dfa7c899 708 * Raw event type provide the config in the event structure
241771ef 709 */
a21ca2ca
IM		 710 if (attr->type == PERF_TYPE_RAW) {
711 hwc->config |= x86_pmu.raw_event(attr->config);
8326f44d 712 return 0;
241771ef 713 }
241771ef 714
8326f44d
IM		 715 if (attr->type == PERF_TYPE_HW_CACHE)
716 return set_ext_hw_attr(hwc, attr);
717
718 if (attr->config >= x86_pmu.max_events)
719 return -EINVAL;
720 /*
721 * The generic map:
722 */
723 hwc->config |= x86_pmu.event_map(attr->config);
4e935e47 724
241771ef
IM		 725 return 0;
726}
727
9e35ad38 728static void intel_pmu_disable_all(void)
4ac13294 729{
862a1a5f 730 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
241771ef 731}
b56a3802 732
9e35ad38 733static void amd_pmu_disable_all(void)
f87ad35d 734{
b0f3f28e 735 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
9e35ad38
PZ		 736 int idx;
737
738 if (!cpuc->enabled)
739 return;
b0f3f28e 740
b0f3f28e 741 cpuc->enabled = 0;
60b3df9c
PZ		 742 /*
743 * ensure we write the disable before we start disabling the
5f4ec28f
RR		 744 * counters proper, so that amd_pmu_enable_counter() does the
745 * right thing.
60b3df9c 746 */
b0f3f28e 747 barrier();
f87ad35d 748
0933e5c6 749 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
b0f3f28e
PZ		 750 u64 val;
751
43f6201a 752 if (!test_bit(idx, cpuc->active_mask))
4295ee62 753 continue;
f87ad35d 754 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
4295ee62
RR		 755 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
756 continue;
757 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
758 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d 759 }
f87ad35d
JSR		 760}
761
9e35ad38 762void hw_perf_disable(void)
b56a3802 763{
85cf9dba 764 if (!x86_pmu_initialized())
9e35ad38
PZ		 765 return;
766 return x86_pmu.disable_all();
b56a3802 767}
241771ef 768
9e35ad38 769static void intel_pmu_enable_all(void)
b56a3802 770{
9e35ad38 771 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
b56a3802
JSR		 772}
773
9e35ad38 774static void amd_pmu_enable_all(void)
f87ad35d 775{
b0f3f28e 776 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
f87ad35d
JSR		 777 int idx;
778
9e35ad38 779 if (cpuc->enabled)
b0f3f28e
PZ		 780 return;
781
9e35ad38
PZ		 782 cpuc->enabled = 1;
783 barrier();
784
0933e5c6 785 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
4295ee62 786 u64 val;
b0f3f28e 787
43f6201a 788 if (!test_bit(idx, cpuc->active_mask))
4295ee62
RR		 789 continue;
790 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
791 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
792 continue;
793 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
794 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
f87ad35d
JSR		 795 }
796}
797
9e35ad38 798void hw_perf_enable(void)
ee06094f 799{
85cf9dba 800 if (!x86_pmu_initialized())
2b9ff0db 801 return;
9e35ad38 802 x86_pmu.enable_all();
ee06094f 803}
ee06094f 804
19d84dab 805static inline u64 intel_pmu_get_status(void)
b0f3f28e
PZ		 806{
807 u64 status;
808
b7f8859a 809 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
b0f3f28e 810
b7f8859a 811 return status;
b0f3f28e
PZ		 812}
813
dee5d906 814static inline void intel_pmu_ack_status(u64 ack)
b0f3f28e
PZ		 815{
816 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
817}
818
7c90cc45 819static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
b0f3f28e 820{
7c90cc45 821 int err;
7c90cc45
RR		 822 err = checking_wrmsrl(hwc->config_base + idx,
823 hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
b0f3f28e
PZ		 824}
825
d4369891 826static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
b0f3f28e 827{
d4369891 828 int err;
d4369891
RR		 829 err = checking_wrmsrl(hwc->config_base + idx,
830 hwc->config);
b0f3f28e
PZ		 831}
832
2f18d1e8 833static inline void
d4369891 834intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
2f18d1e8
IM		 835{
836 int idx = __idx - X86_PMC_IDX_FIXED;
837 u64 ctrl_val, mask;
838 int err;
839
840 mask = 0xfULL << (idx * 4);
841
842 rdmsrl(hwc->config_base, ctrl_val);
843 ctrl_val &= ~mask;
844 err = checking_wrmsrl(hwc->config_base, ctrl_val);
845}
846
7e2ae347 847static inline void
d4369891 848intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
7e2ae347 849{
d4369891
RR		 850 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
851 intel_pmu_disable_fixed(hwc, idx);
852 return;
853 }
854
855 x86_pmu_disable_counter(hwc, idx);
856}
857
858static inline void
859amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
860{
861 x86_pmu_disable_counter(hwc, idx);
7e2ae347
IM		 862}
863
2f18d1e8 864static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
241771ef 865
ee06094f
IM		 866/*
867 * Set the next IRQ period, based on the hwc->period_left value.
868 * To be called with the counter disabled in hw:
869 */
e4abb5d4 870static int
26816c28 871x86_perf_counter_set_period(struct perf_counter *counter,
ee06094f 872 struct hw_perf_counter *hwc, int idx)
241771ef 873{
2f18d1e8 874 s64 left = atomic64_read(&hwc->period_left);
e4abb5d4
PZ		 875 s64 period = hwc->sample_period;
876 int err, ret = 0;
ee06094f 877
ee06094f
IM		 878 /*
879 * If we are way outside a reasoable range then just skip forward:
880 */
881 if (unlikely(left <= -period)) {
882 left = period;
883 atomic64_set(&hwc->period_left, left);
9e350de3 884 hwc->last_period = period;
e4abb5d4 885 ret = 1;
ee06094f
IM		 886 }
887
888 if (unlikely(left <= 0)) {
889 left += period;
890 atomic64_set(&hwc->period_left, left);
9e350de3 891 hwc->last_period = period;
e4abb5d4 892 ret = 1;
ee06094f 893 }
1c80f4b5
IM		 894 /*
895 * Quirk: certain CPUs dont like it if just 1 event is left:
896 */
897 if (unlikely(left < 2))
898 left = 2;
241771ef 899
e4abb5d4
PZ		 900 if (left > x86_pmu.max_period)
901 left = x86_pmu.max_period;
902
ee06094f
IM		 903 per_cpu(prev_left[idx], smp_processor_id()) = left;
904
905 /*
906 * The hw counter starts counting from this counter offset,
907 * mark it to be able to extra future deltas:
908 */
2f18d1e8 909 atomic64_set(&hwc->prev_count, (u64)-left);
ee06094f 910
2f18d1e8 911 err = checking_wrmsrl(hwc->counter_base + idx,
0933e5c6 912 (u64)(-left) & x86_pmu.counter_mask);
e4abb5d4
PZ		 913
914 return ret;
2f18d1e8
IM		 915}
916
917static inline void
7c90cc45 918intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
2f18d1e8
IM		 919{
920 int idx = __idx - X86_PMC_IDX_FIXED;
921 u64 ctrl_val, bits, mask;
922 int err;
923
924 /*
0475f9ea
PM		 925 * Enable IRQ generation (0x8),
926 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
927 * if requested:
2f18d1e8 928 */
0475f9ea
PM		 929 bits = 0x8ULL;
930 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
931 bits |= 0x2;
2f18d1e8
IM		 932 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
933 bits |= 0x1;
934 bits <<= (idx * 4);
935 mask = 0xfULL << (idx * 4);
936
937 rdmsrl(hwc->config_base, ctrl_val);
938 ctrl_val &= ~mask;
939 ctrl_val |= bits;
940 err = checking_wrmsrl(hwc->config_base, ctrl_val);
7e2ae347
IM		 941}
942
7c90cc45 943static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
7e2ae347 944{
7c90cc45
RR		 945 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
946 intel_pmu_enable_fixed(hwc, idx);
947 return;
948 }
949
950 x86_pmu_enable_counter(hwc, idx);
951}
952
953static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
954{
955 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
956
957 if (cpuc->enabled)
958 x86_pmu_enable_counter(hwc, idx);
2b583d8b 959 else
d4369891 960 x86_pmu_disable_counter(hwc, idx);
241771ef
IM		 961}
962
2f18d1e8
IM		 963static int
964fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
862a1a5f 965{
2f18d1e8
IM		 966 unsigned int event;
967
ef7b3e09 968 if (!x86_pmu.num_counters_fixed)
f87ad35d
JSR		 969 return -1;
970
dff5da6d
YW		 971 /*
972 * Quirk, IA32_FIXED_CTRs do not work on current Atom processors:
973 */
974 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
975 boot_cpu_data.x86_model == 28)
976 return -1;
977
2f18d1e8
IM		 978 event = hwc->config & ARCH_PERFMON_EVENT_MASK;
979
f4dbfa8f 980 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
2f18d1e8 981 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
f4dbfa8f 982 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
2f18d1e8 983 return X86_PMC_IDX_FIXED_CPU_CYCLES;
f4dbfa8f 984 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
2f18d1e8
IM		 985 return X86_PMC_IDX_FIXED_BUS_CYCLES;
986
862a1a5f
IM		 987 return -1;
988}
989
ee06094f
IM		 990/*
991 * Find a PMC slot for the freshly enabled / scheduled in counter:
992 */
4aeb0b42 993static int x86_pmu_enable(struct perf_counter *counter)
241771ef
IM		 994{
995 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
996 struct hw_perf_counter *hwc = &counter->hw;
2f18d1e8 997 int idx;
241771ef 998
2f18d1e8
IM		 999 idx = fixed_mode_idx(counter, hwc);
1000 if (idx >= 0) {
1001 /*
1002 * Try to get the fixed counter, if that is already taken
1003 * then try to get a generic counter:
1004 */
43f6201a 1005 if (test_and_set_bit(idx, cpuc->used_mask))
2f18d1e8 1006 goto try_generic;
0dff86aa 1007
2f18d1e8
IM		 1008 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1009 /*
1010 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
1011 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1012 */
1013 hwc->counter_base =
1014 MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
241771ef 1015 hwc->idx = idx;
2f18d1e8
IM		 1016 } else {
1017 idx = hwc->idx;
1018 /* Try to get the previous generic counter again */
43f6201a 1019 if (test_and_set_bit(idx, cpuc->used_mask)) {
2f18d1e8 1020try_generic:
43f6201a 1021 idx = find_first_zero_bit(cpuc->used_mask,
0933e5c6
RR		 1022 x86_pmu.num_counters);
1023 if (idx == x86_pmu.num_counters)
2f18d1e8
IM		 1024 return -EAGAIN;
1025
43f6201a 1026 set_bit(idx, cpuc->used_mask);
2f18d1e8
IM		 1027 hwc->idx = idx;
1028 }
4a06bd85
RR		 1029 hwc->config_base = x86_pmu.eventsel;
1030 hwc->counter_base = x86_pmu.perfctr;
241771ef
IM		 1031 }
1032
c323d95f 1033 perf_counters_lapic_init();
53b441a5 1034
d4369891 1035 x86_pmu.disable(hwc, idx);
241771ef 1036
862a1a5f 1037 cpuc->counters[idx] = counter;
43f6201a 1038 set_bit(idx, cpuc->active_mask);
7e2ae347 1039
26816c28 1040 x86_perf_counter_set_period(counter, hwc, idx);
7c90cc45 1041 x86_pmu.enable(hwc, idx);
95cdd2e7
IM		 1042
1043 return 0;
241771ef
IM		 1044}
1045
a78ac325
PZ		 1046static void x86_pmu_unthrottle(struct perf_counter *counter)
1047{
1048 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1049 struct hw_perf_counter *hwc = &counter->hw;
1050
1051 if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1052 cpuc->counters[hwc->idx] != counter))
1053 return;
1054
1055 x86_pmu.enable(hwc, hwc->idx);
1056}
1057
241771ef
IM		 1058void perf_counter_print_debug(void)
1059{
2f18d1e8 1060 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
0dff86aa 1061 struct cpu_hw_counters *cpuc;
5bb9efe3 1062 unsigned long flags;
1e125676
IM		 1063 int cpu, idx;
1064
0933e5c6 1065 if (!x86_pmu.num_counters)
1e125676 1066 return;
241771ef 1067
5bb9efe3 1068 local_irq_save(flags);
241771ef
IM		 1069
1070 cpu = smp_processor_id();
0dff86aa 1071 cpuc = &per_cpu(cpu_hw_counters, cpu);
241771ef 1072
faa28ae0 1073 if (x86_pmu.version >= 2) {
a1ef58f4
JSR		 1074 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1075 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1076 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1077 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1078
1079 pr_info("\n");
1080 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1081 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1082 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1083 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
f87ad35d 1084 }
43f6201a 1085 pr_info("CPU#%d: used: %016llx\n", cpu, *(u64 *)cpuc->used_mask);
241771ef 1086
0933e5c6 1087 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
4a06bd85
RR		 1088 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1089 rdmsrl(x86_pmu.perfctr + idx, pmc_count);
241771ef 1090
ee06094f 1091 prev_left = per_cpu(prev_left[idx], cpu);
241771ef 1092
a1ef58f4 1093 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
241771ef 1094 cpu, idx, pmc_ctrl);
a1ef58f4 1095 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
241771ef 1096 cpu, idx, pmc_count);
a1ef58f4 1097 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
ee06094f 1098 cpu, idx, prev_left);
241771ef 1099 }
0933e5c6 1100 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
2f18d1e8
IM		 1101 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1102
a1ef58f4 1103 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
2f18d1e8
IM		 1104 cpu, idx, pmc_count);
1105 }
5bb9efe3 1106 local_irq_restore(flags);
241771ef
IM		 1107}
1108
4aeb0b42 1109static void x86_pmu_disable(struct perf_counter *counter)
241771ef
IM		 1110{
1111 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
1112 struct hw_perf_counter *hwc = &counter->hw;
6f00cada 1113 int idx = hwc->idx;
241771ef 1114
09534238
RR		 1115 /*
1116 * Must be done before we disable, otherwise the nmi handler
1117 * could reenable again:
1118 */
43f6201a 1119 clear_bit(idx, cpuc->active_mask);
d4369891 1120 x86_pmu.disable(hwc, idx);
241771ef 1121
2f18d1e8
IM		 1122 /*
1123 * Make sure the cleared pointer becomes visible before we
1124 * (potentially) free the counter:
1125 */
527e26af 1126 barrier();
241771ef 1127
ee06094f
IM		 1128 /*
1129 * Drain the remaining delta count out of a counter
1130 * that we are disabling:
1131 */
1132 x86_perf_counter_update(counter, hwc, idx);
09534238 1133 cpuc->counters[idx] = NULL;
43f6201a 1134 clear_bit(idx, cpuc->used_mask);
241771ef
IM		 1135}
1136
7e2ae347 1137/*
ee06094f
IM		 1138 * Save and restart an expired counter. Called by NMI contexts,
1139 * so it has to be careful about preempting normal counter ops:
7e2ae347 1140 */
e4abb5d4 1141static int intel_pmu_save_and_restart(struct perf_counter *counter)
241771ef
IM		 1142{
1143 struct hw_perf_counter *hwc = &counter->hw;
1144 int idx = hwc->idx;
e4abb5d4 1145 int ret;
241771ef 1146
ee06094f 1147 x86_perf_counter_update(counter, hwc, idx);
e4abb5d4 1148 ret = x86_perf_counter_set_period(counter, hwc, idx);
7e2ae347 1149
2f18d1e8 1150 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
7c90cc45 1151 intel_pmu_enable_counter(hwc, idx);
e4abb5d4
PZ		 1152
1153 return ret;
241771ef
IM		 1154}
1155
aaba9801
IM		 1156static void intel_pmu_reset(void)
1157{
1158 unsigned long flags;
1159 int idx;
1160
1161 if (!x86_pmu.num_counters)
1162 return;
1163
1164 local_irq_save(flags);
1165
1166 printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1167
1168 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1169 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1170 checking_wrmsrl(x86_pmu.perfctr + idx, 0ull);
1171 }
1172 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1173 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1174 }
1175
1176 local_irq_restore(flags);
1177}
1178
1179
241771ef
IM		 1180/*
1181 * This handler is triggered by the local APIC, so the APIC IRQ handling
1182 * rules apply:
1183 */
a3288106 1184static int intel_pmu_handle_irq(struct pt_regs *regs)
241771ef 1185{
df1a132b 1186 struct perf_sample_data data;
9029a5e3 1187 struct cpu_hw_counters *cpuc;
9029a5e3 1188 int bit, cpu, loops;
4b39fd96 1189 u64 ack, status;
9029a5e3 1190
df1a132b
PZ		 1191 data.regs = regs;
1192 data.addr = 0;
1193
9029a5e3
IM		 1194 cpu = smp_processor_id();
1195 cpuc = &per_cpu(cpu_hw_counters, cpu);
241771ef 1196
9e35ad38 1197 perf_disable();
19d84dab 1198 status = intel_pmu_get_status();
9e35ad38
PZ		 1199 if (!status) {
1200 perf_enable();
1201 return 0;
1202 }
87b9cf46 1203
9029a5e3 1204 loops = 0;
241771ef 1205again:
9029a5e3
IM		 1206 if (++loops > 100) {
1207 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
34adc806 1208 perf_counter_print_debug();
aaba9801
IM		 1209 intel_pmu_reset();
1210 perf_enable();
9029a5e3
IM		 1211 return 1;
1212 }
1213
d278c484 1214 inc_irq_stat(apic_perf_irqs);
241771ef 1215 ack = status;
2f18d1e8 1216 for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
862a1a5f 1217 struct perf_counter *counter = cpuc->counters[bit];
241771ef
IM		 1218
1219 clear_bit(bit, (unsigned long *) &status);
43f6201a 1220 if (!test_bit(bit, cpuc->active_mask))
241771ef
IM		 1221 continue;
1222
e4abb5d4
PZ		 1223 if (!intel_pmu_save_and_restart(counter))
1224 continue;
1225
df1a132b 1226 if (perf_counter_overflow(counter, 1, &data))
d4369891 1227 intel_pmu_disable_counter(&counter->hw, bit);
241771ef
IM		 1228 }
1229
dee5d906 1230 intel_pmu_ack_status(ack);
241771ef
IM		 1231
1232 /*
1233 * Repeat if there is more work to be done:
1234 */
19d84dab 1235 status = intel_pmu_get_status();
241771ef
IM		 1236 if (status)
1237 goto again;
b0f3f28e 1238
48e22d56 1239 perf_enable();
9e35ad38
PZ		 1240
1241 return 1;
1b023a96
MG		 1242}
1243
a3288106 1244static int amd_pmu_handle_irq(struct pt_regs *regs)
a29aa8a7 1245{
df1a132b 1246 struct perf_sample_data data;
9029a5e3 1247 struct cpu_hw_counters *cpuc;
a29aa8a7
RR		 1248 struct perf_counter *counter;
1249 struct hw_perf_counter *hwc;
df1a132b 1250 int cpu, idx, handled = 0;
9029a5e3
IM		 1251 u64 val;
1252
df1a132b
PZ		 1253 data.regs = regs;
1254 data.addr = 0;
1255
9029a5e3
IM		 1256 cpu = smp_processor_id();
1257 cpuc = &per_cpu(cpu_hw_counters, cpu);
962bf7a6 1258
a29aa8a7 1259 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
43f6201a 1260 if (!test_bit(idx, cpuc->active_mask))
a29aa8a7 1261 continue;
962bf7a6 1262
a29aa8a7
RR		 1263 counter = cpuc->counters[idx];
1264 hwc = &counter->hw;
a4016a79 1265
4b7bfd0d 1266 val = x86_perf_counter_update(counter, hwc, idx);
a29aa8a7 1267 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
48e22d56 1268 continue;
962bf7a6 1269
9e350de3
PZ		 1270 /*
1271 * counter overflow
1272 */
1273 handled = 1;
1274 data.period = counter->hw.last_period;
1275
e4abb5d4
PZ		 1276 if (!x86_perf_counter_set_period(counter, hwc, idx))
1277 continue;
1278
df1a132b 1279 if (perf_counter_overflow(counter, 1, &data))
a29aa8a7 1280 amd_pmu_disable_counter(hwc, idx);
a29aa8a7 1281 }
962bf7a6 1282
9e350de3
PZ		 1283 if (handled)
1284 inc_irq_stat(apic_perf_irqs);
1285
a29aa8a7
RR		 1286 return handled;
1287}
39d81eab 1288
b6276f35
PZ		 1289void smp_perf_pending_interrupt(struct pt_regs *regs)
1290{
1291 irq_enter();
1292 ack_APIC_irq();
1293 inc_irq_stat(apic_pending_irqs);
1294 perf_counter_do_pending();
1295 irq_exit();
1296}
1297
1298void set_perf_counter_pending(void)
1299{
1300 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1301}
1302
c323d95f 1303void perf_counters_lapic_init(void)
241771ef 1304{
85cf9dba 1305 if (!x86_pmu_initialized())
241771ef 1306 return;
85cf9dba 1307
241771ef 1308 /*
c323d95f 1309 * Always use NMI for PMU
241771ef 1310 */
c323d95f 1311 apic_write(APIC_LVTPC, APIC_DM_NMI);
241771ef
IM		 1312}
1313
1314static int __kprobes
1315perf_counter_nmi_handler(struct notifier_block *self,
1316 unsigned long cmd, void *__args)
1317{
1318 struct die_args *args = __args;
1319 struct pt_regs *regs;
b0f3f28e 1320
ba77813a 1321 if (!atomic_read(&active_counters))
63a809a2
PZ		 1322 return NOTIFY_DONE;
1323
b0f3f28e
PZ		 1324 switch (cmd) {
1325 case DIE_NMI:
1326 case DIE_NMI_IPI:
1327 break;
241771ef 1328
b0f3f28e 1329 default:
241771ef 1330 return NOTIFY_DONE;
b0f3f28e 1331 }
241771ef
IM		 1332
1333 regs = args->regs;
1334
1335 apic_write(APIC_LVTPC, APIC_DM_NMI);
a4016a79
PZ		 1336 /*
1337 * Can't rely on the handled return value to say it was our NMI, two
1338 * counters could trigger 'simultaneously' raising two back-to-back NMIs.
1339 *
1340 * If the first NMI handles both, the latter will be empty and daze
1341 * the CPU.
1342 */
a3288106 1343 x86_pmu.handle_irq(regs);
241771ef 1344
a4016a79 1345 return NOTIFY_STOP;
241771ef
IM		 1346}
1347
1348static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
5b75af0a
MG		 1349 .notifier_call = perf_counter_nmi_handler,
1350 .next = NULL,
1351 .priority = 1
241771ef
IM		 1352};
1353
5f4ec28f 1354static struct x86_pmu intel_pmu = {
faa28ae0 1355 .name = "Intel",
39d81eab 1356 .handle_irq = intel_pmu_handle_irq,
9e35ad38
PZ		 1357 .disable_all = intel_pmu_disable_all,
1358 .enable_all = intel_pmu_enable_all,
5f4ec28f
RR		 1359 .enable = intel_pmu_enable_counter,
1360 .disable = intel_pmu_disable_counter,
b56a3802
JSR		 1361 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
1362 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
5f4ec28f
RR		 1363 .event_map = intel_pmu_event_map,
1364 .raw_event = intel_pmu_raw_event,
b56a3802 1365 .max_events = ARRAY_SIZE(intel_perfmon_event_map),
c619b8ff
RR		 1366 /*
1367 * Intel PMCs cannot be accessed sanely above 32 bit width,
1368 * so we install an artificial 1<<31 period regardless of
1369 * the generic counter period:
1370 */
1371 .max_period = (1ULL << 31) - 1,
b56a3802
JSR		 1372};
1373
5f4ec28f 1374static struct x86_pmu amd_pmu = {
faa28ae0 1375 .name = "AMD",
39d81eab 1376 .handle_irq = amd_pmu_handle_irq,
9e35ad38
PZ		 1377 .disable_all = amd_pmu_disable_all,
1378 .enable_all = amd_pmu_enable_all,
5f4ec28f
RR		 1379 .enable = amd_pmu_enable_counter,
1380 .disable = amd_pmu_disable_counter,
f87ad35d
JSR		 1381 .eventsel = MSR_K7_EVNTSEL0,
1382 .perfctr = MSR_K7_PERFCTR0,
5f4ec28f
RR		 1383 .event_map = amd_pmu_event_map,
1384 .raw_event = amd_pmu_raw_event,
f87ad35d 1385 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
0933e5c6
RR		 1386 .num_counters = 4,
1387 .counter_bits = 48,
1388 .counter_mask = (1ULL << 48) - 1,
c619b8ff
RR		 1389 /* use highest bit to detect overflow */
1390 .max_period = (1ULL << 47) - 1,
f87ad35d
JSR		 1391};
1392
72eae04d 1393static int intel_pmu_init(void)
241771ef 1394{
7bb497bd 1395 union cpuid10_edx edx;
241771ef 1396 union cpuid10_eax eax;
703e937c 1397 unsigned int unused;
7bb497bd 1398 unsigned int ebx;
faa28ae0 1399 int version;
241771ef 1400
da1a776b 1401 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
72eae04d 1402 return -ENODEV;
da1a776b 1403
241771ef
IM		 1404 /*
1405 * Check whether the Architectural PerfMon supports
1406 * Branch Misses Retired Event or not.
1407 */
703e937c 1408 cpuid(10, &eax.full, &ebx, &unused, &edx.full);
241771ef 1409 if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
72eae04d 1410 return -ENODEV;
241771ef 1411
faa28ae0
RR		 1412 version = eax.split.version_id;
1413 if (version < 2)
72eae04d 1414 return -ENODEV;
7bb497bd 1415
1123e3ad
IM		 1416 x86_pmu = intel_pmu;
1417 x86_pmu.version = version;
1418 x86_pmu.num_counters = eax.split.num_counters;
1419 x86_pmu.counter_bits = eax.split.bit_width;
1420 x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
066d7dea
IM		 1421
1422 /*
1423 * Quirk: v2 perfmon does not report fixed-purpose counters, so
1424 * assume at least 3 counters:
1425 */
1123e3ad 1426 x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
b56a3802 1427
9e35ad38
PZ		 1428 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1429
8326f44d 1430 /*
1123e3ad 1431 * Install the hw-cache-events table:
8326f44d
IM		 1432 */
1433 switch (boot_cpu_data.x86_model) {
dc81081b
YW		 1434 case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
1435 case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
1436 case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
1437 case 29: /* six-core 45 nm xeon "Dunnington" */
8326f44d 1438 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
820a6442 1439 sizeof(hw_cache_event_ids));
8326f44d 1440
1123e3ad 1441 pr_cont("Core2 events, ");
8326f44d
IM		 1442 break;
1443 default:
1444 case 26:
1445 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
820a6442 1446 sizeof(hw_cache_event_ids));
8326f44d 1447
1123e3ad 1448 pr_cont("Nehalem/Corei7 events, ");
8326f44d
IM		 1449 break;
1450 case 28:
1451 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
820a6442 1452 sizeof(hw_cache_event_ids));
8326f44d 1453
1123e3ad 1454 pr_cont("Atom events, ");
8326f44d
IM		 1455 break;
1456 }
72eae04d 1457 return 0;
b56a3802
JSR		 1458}
1459
72eae04d 1460static int amd_pmu_init(void)
f87ad35d 1461{
4a06bd85 1462 x86_pmu = amd_pmu;
f86748e9
TG		 1463
1464 switch (boot_cpu_data.x86) {
1465 case 0x0f:
1466 case 0x10:
1467 case 0x11:
1468 memcpy(hw_cache_event_ids, amd_0f_hw_cache_event_ids,
1469 sizeof(hw_cache_event_ids));
1470
1471 pr_cont("AMD Family 0f/10/11 events, ");
1472 break;
1473 }
72eae04d 1474 return 0;
f87ad35d
JSR		 1475}
1476
b56a3802
JSR		 1477void __init init_hw_perf_counters(void)
1478{
72eae04d
RR		 1479 int err;
1480
1123e3ad
IM		 1481 pr_info("Performance Counters: ");
1482
b56a3802
JSR		 1483 switch (boot_cpu_data.x86_vendor) {
1484 case X86_VENDOR_INTEL:
72eae04d 1485 err = intel_pmu_init();
b56a3802 1486 break;
f87ad35d 1487 case X86_VENDOR_AMD:
72eae04d 1488 err = amd_pmu_init();
f87ad35d 1489 break;
4138960a
RR		 1490 default:
1491 return;
b56a3802 1492 }
1123e3ad
IM		 1493 if (err != 0) {
1494 pr_cont("no PMU driver, software counters only.\n");
b56a3802 1495 return;
1123e3ad 1496 }
b56a3802 1497
1123e3ad 1498 pr_cont("%s PMU driver.\n", x86_pmu.name);
faa28ae0 1499
0933e5c6
RR		 1500 if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1501 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
241771ef 1502 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
0933e5c6 1503 x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
241771ef 1504 }
0933e5c6
RR		 1505 perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1506 perf_max_counters = x86_pmu.num_counters;
241771ef 1507
0933e5c6
RR		 1508 if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1509 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
703e937c 1510 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
0933e5c6 1511 x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
703e937c 1512 }
862a1a5f 1513
0933e5c6
RR		 1514 perf_counter_mask |=
1515 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
241771ef 1516
c323d95f 1517 perf_counters_lapic_init();
241771ef 1518 register_die_notifier(&perf_counter_nmi_notifier);
1123e3ad
IM		 1519
1520 pr_info("... version: %d\n", x86_pmu.version);
1521 pr_info("... bit width: %d\n", x86_pmu.counter_bits);
1522 pr_info("... generic counters: %d\n", x86_pmu.num_counters);
1523 pr_info("... value mask: %016Lx\n", x86_pmu.counter_mask);
1524 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
1525 pr_info("... fixed-purpose counters: %d\n", x86_pmu.num_counters_fixed);
1526 pr_info("... counter mask: %016Lx\n", perf_counter_mask);
241771ef 1527}
621a01ea 1528
bb775fc2 1529static inline void x86_pmu_read(struct perf_counter *counter)
ee06094f
IM		 1530{
1531 x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1532}
1533
4aeb0b42
RR		 1534static const struct pmu pmu = {
1535 .enable = x86_pmu_enable,
1536 .disable = x86_pmu_disable,
1537 .read = x86_pmu_read,
a78ac325 1538 .unthrottle = x86_pmu_unthrottle,
621a01ea
IM		 1539};
1540
4aeb0b42 1541const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
621a01ea
IM		 1542{
1543 int err;
1544
1545 err = __hw_perf_counter_init(counter);
1546 if (err)
9ea98e19 1547 return ERR_PTR(err);
621a01ea 1548
4aeb0b42 1549 return &pmu;
621a01ea 1550}
d7d59fb3
PZ		 1551
1552/*
1553 * callchain support
1554 */
1555
1556static inline
1557void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1558{
1559 if (entry->nr < MAX_STACK_DEPTH)
1560 entry->ip[entry->nr++] = ip;
1561}
1562
1563static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1564static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1565
1566
1567static void
1568backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1569{
1570 /* Ignore warnings */
1571}
1572
1573static void backtrace_warning(void *data, char *msg)
1574{
1575 /* Ignore warnings */
1576}
1577
1578static int backtrace_stack(void *data, char *name)
1579{
1580 /* Don't bother with IRQ stacks for now */
1581 return -1;
1582}
1583
1584static void backtrace_address(void *data, unsigned long addr, int reliable)
1585{
1586 struct perf_callchain_entry *entry = data;
1587
1588 if (reliable)
1589 callchain_store(entry, addr);
1590}
1591
1592static const struct stacktrace_ops backtrace_ops = {
1593 .warning = backtrace_warning,
1594 .warning_symbol = backtrace_warning_symbol,
1595 .stack = backtrace_stack,
1596 .address = backtrace_address,
1597};
1598
1599static void
1600perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1601{
1602 unsigned long bp;
1603 char *stack;
5872bdb8 1604 int nr = entry->nr;
d7d59fb3
PZ		 1605
1606 callchain_store(entry, instruction_pointer(regs));
1607
1608 stack = ((char *)regs + sizeof(struct pt_regs));
1609#ifdef CONFIG_FRAME_POINTER
1610 bp = frame_pointer(regs);
1611#else
1612 bp = 0;
1613#endif
1614
1615 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
5872bdb8
PZ		 1616
1617 entry->kernel = entry->nr - nr;
d7d59fb3
PZ		 1618}
1619
1620
1621struct stack_frame {
1622 const void __user *next_fp;
1623 unsigned long return_address;
1624};
1625
1626static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1627{
1628 int ret;
1629
1630 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1631 return 0;
1632
1633 ret = 1;
1634 pagefault_disable();
1635 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1636 ret = 0;
1637 pagefault_enable();
1638
1639 return ret;
1640}
1641
1642static void
1643perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1644{
1645 struct stack_frame frame;
1646 const void __user *fp;
5872bdb8 1647 int nr = entry->nr;
d7d59fb3
PZ		 1648
1649 regs = (struct pt_regs *)current->thread.sp0 - 1;
1650 fp = (void __user *)regs->bp;
1651
1652 callchain_store(entry, regs->ip);
1653
1654 while (entry->nr < MAX_STACK_DEPTH) {
1655 frame.next_fp = NULL;
1656 frame.return_address = 0;
1657
1658 if (!copy_stack_frame(fp, &frame))
1659 break;
1660
1661 if ((unsigned long)fp < user_stack_pointer(regs))
1662 break;
1663
1664 callchain_store(entry, frame.return_address);
1665 fp = frame.next_fp;
1666 }
5872bdb8
PZ		 1667
1668 entry->user = entry->nr - nr;
d7d59fb3
PZ		 1669}
1670
1671static void
1672perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1673{
1674 int is_user;
1675
1676 if (!regs)
1677 return;
1678
1679 is_user = user_mode(regs);
1680
1681 if (!current || current->pid == 0)
1682 return;
1683
1684 if (is_user && current->state != TASK_RUNNING)
1685 return;
1686
1687 if (!is_user)
1688 perf_callchain_kernel(regs, entry);
1689
1690 if (current->mm)
1691 perf_callchain_user(regs, entry);
1692}
1693
1694struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1695{
1696 struct perf_callchain_entry *entry;
1697
1698 if (in_nmi())
1699 entry = &__get_cpu_var(nmi_entry);
1700 else
1701 entry = &__get_cpu_var(irq_entry);
1702
1703 entry->nr = 0;
5872bdb8
PZ		 1704 entry->hv = 0;
1705 entry->kernel = 0;
1706 entry->user = 0;
d7d59fb3
PZ		 1707
1708 perf_do_callchain(regs, entry);
1709
1710 return entry;
1711}
Linux 6.x block layer and io_uring tree(s)