projects / linux-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
perf_counter, x86: rework counter disable functions
[linux-block.git] / arch / x86 / kernel / cpu / perf_counter.c
... / ...
CommitLineData
1/*
2 * Performance counter x86 architecture code
3 *
4 * Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2009 Jaswinder Singh Rajput
7 * Copyright(C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 *
9 * For licencing details see kernel-base/COPYING
10 */
11
12#include <linux/perf_counter.h>
13#include <linux/capability.h>
14#include <linux/notifier.h>
15#include <linux/hardirq.h>
16#include <linux/kprobes.h>
17#include <linux/module.h>
18#include <linux/kdebug.h>
19#include <linux/sched.h>
20#include <linux/uaccess.h>
21
22#include <asm/apic.h>
23#include <asm/stacktrace.h>
24#include <asm/nmi.h>
25
26static bool perf_counters_initialized __read_mostly;
27static u64 perf_counter_mask __read_mostly;
28
29struct cpu_hw_counters {
30 struct perf_counter *counters[X86_PMC_IDX_MAX];
31 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
32 unsigned long active[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
33 unsigned long interrupts;
34 u64 throttle_ctrl;
35 int enabled;
36};
37
38/*
39 * struct x86_pmu - generic x86 pmu
40 */
41struct x86_pmu {
42 const char *name;
43 int version;
44 int (*handle_irq)(struct pt_regs *, int);
45 u64 (*save_disable_all)(void);
46 void (*restore_all)(u64);
47 void (*enable)(struct hw_perf_counter *, int);
48 void (*disable)(struct hw_perf_counter *, int);
49 unsigned eventsel;
50 unsigned perfctr;
51 u64 (*event_map)(int);
52 u64 (*raw_event)(u64);
53 int max_events;
54 int num_counters;
55 int num_counters_fixed;
56 int counter_bits;
57 u64 counter_mask;
58};
59
60static struct x86_pmu x86_pmu __read_mostly;
61
62static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
63 .enabled = 1,
64};
65
66/*
67 * Intel PerfMon v3. Used on Core2 and later.
68 */
69static const u64 intel_perfmon_event_map[] =
70{
71 [PERF_COUNT_CPU_CYCLES] = 0x003c,
72 [PERF_COUNT_INSTRUCTIONS] = 0x00c0,
73 [PERF_COUNT_CACHE_REFERENCES] = 0x4f2e,
74 [PERF_COUNT_CACHE_MISSES] = 0x412e,
75 [PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
76 [PERF_COUNT_BRANCH_MISSES] = 0x00c5,
77 [PERF_COUNT_BUS_CYCLES] = 0x013c,
78};
79
80static u64 intel_pmu_event_map(int event)
81{
82 return intel_perfmon_event_map[event];
83}
84
85static u64 intel_pmu_raw_event(u64 event)
86{
87#define CORE_EVNTSEL_EVENT_MASK 0x000000FFULL
88#define CORE_EVNTSEL_UNIT_MASK 0x0000FF00ULL
89#define CORE_EVNTSEL_COUNTER_MASK 0xFF000000ULL
90
91#define CORE_EVNTSEL_MASK \
92 (CORE_EVNTSEL_EVENT_MASK | \
93 CORE_EVNTSEL_UNIT_MASK | \
94 CORE_EVNTSEL_COUNTER_MASK)
95
96 return event & CORE_EVNTSEL_MASK;
97}
98
99/*
100 * AMD Performance Monitor K7 and later.
101 */
102static const u64 amd_perfmon_event_map[] =
103{
104 [PERF_COUNT_CPU_CYCLES] = 0x0076,
105 [PERF_COUNT_INSTRUCTIONS] = 0x00c0,
106 [PERF_COUNT_CACHE_REFERENCES] = 0x0080,
107 [PERF_COUNT_CACHE_MISSES] = 0x0081,
108 [PERF_COUNT_BRANCH_INSTRUCTIONS] = 0x00c4,
109 [PERF_COUNT_BRANCH_MISSES] = 0x00c5,
110};
111
112static u64 amd_pmu_event_map(int event)
113{
114 return amd_perfmon_event_map[event];
115}
116
117static u64 amd_pmu_raw_event(u64 event)
118{
119#define K7_EVNTSEL_EVENT_MASK 0x7000000FFULL
120#define K7_EVNTSEL_UNIT_MASK 0x00000FF00ULL
121#define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
122
123#define K7_EVNTSEL_MASK \
124 (K7_EVNTSEL_EVENT_MASK | \
125 K7_EVNTSEL_UNIT_MASK | \
126 K7_EVNTSEL_COUNTER_MASK)
127
128 return event & K7_EVNTSEL_MASK;
129}
130
131/*
132 * Propagate counter elapsed time into the generic counter.
133 * Can only be executed on the CPU where the counter is active.
134 * Returns the delta events processed.
135 */
136static void
137x86_perf_counter_update(struct perf_counter *counter,
138 struct hw_perf_counter *hwc, int idx)
139{
140 u64 prev_raw_count, new_raw_count, delta;
141
142 /*
143 * Careful: an NMI might modify the previous counter value.
144 *
145 * Our tactic to handle this is to first atomically read and
146 * exchange a new raw count - then add that new-prev delta
147 * count to the generic counter atomically:
148 */
149again:
150 prev_raw_count = atomic64_read(&hwc->prev_count);
151 rdmsrl(hwc->counter_base + idx, new_raw_count);
152
153 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
154 new_raw_count) != prev_raw_count)
155 goto again;
156
157 /*
158 * Now we have the new raw value and have updated the prev
159 * timestamp already. We can now calculate the elapsed delta
160 * (counter-)time and add that to the generic counter.
161 *
162 * Careful, not all hw sign-extends above the physical width
163 * of the count, so we do that by clipping the delta to 32 bits:
164 */
165 delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
166
167 atomic64_add(delta, &counter->count);
168 atomic64_sub(delta, &hwc->period_left);
169}
170
171static atomic_t num_counters;
172static DEFINE_MUTEX(pmc_reserve_mutex);
173
174static bool reserve_pmc_hardware(void)
175{
176 int i;
177
178 if (nmi_watchdog == NMI_LOCAL_APIC)
179 disable_lapic_nmi_watchdog();
180
181 for (i = 0; i < x86_pmu.num_counters; i++) {
182 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
183 goto perfctr_fail;
184 }
185
186 for (i = 0; i < x86_pmu.num_counters; i++) {
187 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
188 goto eventsel_fail;
189 }
190
191 return true;
192
193eventsel_fail:
194 for (i--; i >= 0; i--)
195 release_evntsel_nmi(x86_pmu.eventsel + i);
196
197 i = x86_pmu.num_counters;
198
199perfctr_fail:
200 for (i--; i >= 0; i--)
201 release_perfctr_nmi(x86_pmu.perfctr + i);
202
203 if (nmi_watchdog == NMI_LOCAL_APIC)
204 enable_lapic_nmi_watchdog();
205
206 return false;
207}
208
209static void release_pmc_hardware(void)
210{
211 int i;
212
213 for (i = 0; i < x86_pmu.num_counters; i++) {
214 release_perfctr_nmi(x86_pmu.perfctr + i);
215 release_evntsel_nmi(x86_pmu.eventsel + i);
216 }
217
218 if (nmi_watchdog == NMI_LOCAL_APIC)
219 enable_lapic_nmi_watchdog();
220}
221
222static void hw_perf_counter_destroy(struct perf_counter *counter)
223{
224 if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
225 release_pmc_hardware();
226 mutex_unlock(&pmc_reserve_mutex);
227 }
228}
229
230/*
231 * Setup the hardware configuration for a given hw_event_type
232 */
233static int __hw_perf_counter_init(struct perf_counter *counter)
234{
235 struct perf_counter_hw_event *hw_event = &counter->hw_event;
236 struct hw_perf_counter *hwc = &counter->hw;
237 int err;
238
239 /* disable temporarily */
240 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
241 return -ENOSYS;
242
243 if (unlikely(!perf_counters_initialized))
244 return -EINVAL;
245
246 err = 0;
247 if (atomic_inc_not_zero(&num_counters)) {
248 mutex_lock(&pmc_reserve_mutex);
249 if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
250 err = -EBUSY;
251 else
252 atomic_inc(&num_counters);
253 mutex_unlock(&pmc_reserve_mutex);
254 }
255 if (err)
256 return err;
257
258 /*
259 * Generate PMC IRQs:
260 * (keep 'enabled' bit clear for now)
261 */
262 hwc->config = ARCH_PERFMON_EVENTSEL_INT;
263
264 /*
265 * Count user and OS events unless requested not to.
266 */
267 if (!hw_event->exclude_user)
268 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
269 if (!hw_event->exclude_kernel)
270 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
271
272 /*
273 * If privileged enough, allow NMI events:
274 */
275 hwc->nmi = 0;
276 if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
277 hwc->nmi = 1;
278
279 hwc->irq_period = hw_event->irq_period;
280 /*
281 * Intel PMCs cannot be accessed sanely above 32 bit width,
282 * so we install an artificial 1<<31 period regardless of
283 * the generic counter period:
284 */
285 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
286 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
287 hwc->irq_period = 0x7FFFFFFF;
288
289 atomic64_set(&hwc->period_left, hwc->irq_period);
290
291 /*
292 * Raw event type provide the config in the event structure
293 */
294 if (perf_event_raw(hw_event)) {
295 hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
296 } else {
297 if (perf_event_id(hw_event) >= x86_pmu.max_events)
298 return -EINVAL;
299 /*
300 * The generic map:
301 */
302 hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
303 }
304
305 counter->destroy = hw_perf_counter_destroy;
306
307 return 0;
308}
309
310static u64 intel_pmu_save_disable_all(void)
311{
312 u64 ctrl;
313
314 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
315 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
316
317 return ctrl;
318}
319
320static u64 amd_pmu_save_disable_all(void)
321{
322 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
323 int enabled, idx;
324
325 enabled = cpuc->enabled;
326 cpuc->enabled = 0;
327 /*
328 * ensure we write the disable before we start disabling the
329 * counters proper, so that amd_pmu_enable_counter() does the
330 * right thing.
331 */
332 barrier();
333
334 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
335 u64 val;
336
337 if (!test_bit(idx, cpuc->active))
338 continue;
339 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
340 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
341 continue;
342 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
343 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
344 }
345
346 return enabled;
347}
348
349u64 hw_perf_save_disable(void)
350{
351 if (unlikely(!perf_counters_initialized))
352 return 0;
353
354 return x86_pmu.save_disable_all();
355}
356/*
357 * Exported because of ACPI idle
358 */
359EXPORT_SYMBOL_GPL(hw_perf_save_disable);
360
361static void intel_pmu_restore_all(u64 ctrl)
362{
363 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
364}
365
366static void amd_pmu_restore_all(u64 ctrl)
367{
368 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
369 int idx;
370
371 cpuc->enabled = ctrl;
372 barrier();
373 if (!ctrl)
374 return;
375
376 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
377 u64 val;
378
379 if (!test_bit(idx, cpuc->active))
380 continue;
381 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
382 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
383 continue;
384 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
385 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
386 }
387}
388
389void hw_perf_restore(u64 ctrl)
390{
391 if (unlikely(!perf_counters_initialized))
392 return;
393
394 x86_pmu.restore_all(ctrl);
395}
396/*
397 * Exported because of ACPI idle
398 */
399EXPORT_SYMBOL_GPL(hw_perf_restore);
400
401static inline u64 intel_pmu_get_status(u64 mask)
402{
403 u64 status;
404
405 if (unlikely(!perf_counters_initialized))
406 return 0;
407 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
408
409 return status;
410}
411
412static inline void intel_pmu_ack_status(u64 ack)
413{
414 wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
415}
416
417static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
418{
419 int err;
420
421 if (unlikely(!perf_counters_initialized))
422 return;
423
424 err = checking_wrmsrl(hwc->config_base + idx,
425 hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
426}
427
428static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
429{
430 int err;
431
432 if (unlikely(!perf_counters_initialized))
433 return;
434
435 err = checking_wrmsrl(hwc->config_base + idx,
436 hwc->config);
437}
438
439static inline void
440intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
441{
442 int idx = __idx - X86_PMC_IDX_FIXED;
443 u64 ctrl_val, mask;
444 int err;
445
446 mask = 0xfULL << (idx * 4);
447
448 rdmsrl(hwc->config_base, ctrl_val);
449 ctrl_val &= ~mask;
450 err = checking_wrmsrl(hwc->config_base, ctrl_val);
451}
452
453static inline void
454intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
455{
456 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
457 intel_pmu_disable_fixed(hwc, idx);
458 return;
459 }
460
461 x86_pmu_disable_counter(hwc, idx);
462}
463
464static inline void
465amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
466{
467 x86_pmu_disable_counter(hwc, idx);
468}
469
470static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
471
472/*
473 * Set the next IRQ period, based on the hwc->period_left value.
474 * To be called with the counter disabled in hw:
475 */
476static void
477x86_perf_counter_set_period(struct perf_counter *counter,
478 struct hw_perf_counter *hwc, int idx)
479{
480 s64 left = atomic64_read(&hwc->period_left);
481 s64 period = hwc->irq_period;
482 int err;
483
484 /*
485 * If we are way outside a reasoable range then just skip forward:
486 */
487 if (unlikely(left <= -period)) {
488 left = period;
489 atomic64_set(&hwc->period_left, left);
490 }
491
492 if (unlikely(left <= 0)) {
493 left += period;
494 atomic64_set(&hwc->period_left, left);
495 }
496
497 per_cpu(prev_left[idx], smp_processor_id()) = left;
498
499 /*
500 * The hw counter starts counting from this counter offset,
501 * mark it to be able to extra future deltas:
502 */
503 atomic64_set(&hwc->prev_count, (u64)-left);
504
505 err = checking_wrmsrl(hwc->counter_base + idx,
506 (u64)(-left) & x86_pmu.counter_mask);
507}
508
509static inline void
510intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
511{
512 int idx = __idx - X86_PMC_IDX_FIXED;
513 u64 ctrl_val, bits, mask;
514 int err;
515
516 /*
517 * Enable IRQ generation (0x8),
518 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
519 * if requested:
520 */
521 bits = 0x8ULL;
522 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
523 bits |= 0x2;
524 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
525 bits |= 0x1;
526 bits <<= (idx * 4);
527 mask = 0xfULL << (idx * 4);
528
529 rdmsrl(hwc->config_base, ctrl_val);
530 ctrl_val &= ~mask;
531 ctrl_val |= bits;
532 err = checking_wrmsrl(hwc->config_base, ctrl_val);
533}
534
535static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
536{
537 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
538 intel_pmu_enable_fixed(hwc, idx);
539 return;
540 }
541
542 x86_pmu_enable_counter(hwc, idx);
543}
544
545static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
546{
547 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
548
549 if (cpuc->enabled)
550 x86_pmu_enable_counter(hwc, idx);
551 else
552 x86_pmu_disable_counter(hwc, idx);
553}
554
555static int
556fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
557{
558 unsigned int event;
559
560 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
561 return -1;
562
563 if (unlikely(hwc->nmi))
564 return -1;
565
566 event = hwc->config & ARCH_PERFMON_EVENT_MASK;
567
568 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
569 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
570 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
571 return X86_PMC_IDX_FIXED_CPU_CYCLES;
572 if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
573 return X86_PMC_IDX_FIXED_BUS_CYCLES;
574
575 return -1;
576}
577
578/*
579 * Find a PMC slot for the freshly enabled / scheduled in counter:
580 */
581static int x86_pmu_enable(struct perf_counter *counter)
582{
583 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
584 struct hw_perf_counter *hwc = &counter->hw;
585 int idx;
586
587 idx = fixed_mode_idx(counter, hwc);
588 if (idx >= 0) {
589 /*
590 * Try to get the fixed counter, if that is already taken
591 * then try to get a generic counter:
592 */
593 if (test_and_set_bit(idx, cpuc->used))
594 goto try_generic;
595
596 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
597 /*
598 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
599 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
600 */
601 hwc->counter_base =
602 MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
603 hwc->idx = idx;
604 } else {
605 idx = hwc->idx;
606 /* Try to get the previous generic counter again */
607 if (test_and_set_bit(idx, cpuc->used)) {
608try_generic:
609 idx = find_first_zero_bit(cpuc->used,
610 x86_pmu.num_counters);
611 if (idx == x86_pmu.num_counters)
612 return -EAGAIN;
613
614 set_bit(idx, cpuc->used);
615 hwc->idx = idx;
616 }
617 hwc->config_base = x86_pmu.eventsel;
618 hwc->counter_base = x86_pmu.perfctr;
619 }
620
621 perf_counters_lapic_init(hwc->nmi);
622
623 x86_pmu.disable(hwc, idx);
624
625 cpuc->counters[idx] = counter;
626 set_bit(idx, cpuc->active);
627
628 x86_perf_counter_set_period(counter, hwc, idx);
629 x86_pmu.enable(hwc, idx);
630
631 return 0;
632}
633
634void perf_counter_print_debug(void)
635{
636 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
637 struct cpu_hw_counters *cpuc;
638 int cpu, idx;
639
640 if (!x86_pmu.num_counters)
641 return;
642
643 local_irq_disable();
644
645 cpu = smp_processor_id();
646 cpuc = &per_cpu(cpu_hw_counters, cpu);
647
648 if (x86_pmu.version >= 2) {
649 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
650 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
651 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
652 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
653
654 pr_info("\n");
655 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
656 pr_info("CPU#%d: status: %016llx\n", cpu, status);
657 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
658 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
659 }
660 pr_info("CPU#%d: used: %016llx\n", cpu, *(u64 *)cpuc->used);
661
662 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
663 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
664 rdmsrl(x86_pmu.perfctr + idx, pmc_count);
665
666 prev_left = per_cpu(prev_left[idx], cpu);
667
668 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
669 cpu, idx, pmc_ctrl);
670 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
671 cpu, idx, pmc_count);
672 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
673 cpu, idx, prev_left);
674 }
675 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
676 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
677
678 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
679 cpu, idx, pmc_count);
680 }
681 local_irq_enable();
682}
683
684static void x86_pmu_disable(struct perf_counter *counter)
685{
686 struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
687 struct hw_perf_counter *hwc = &counter->hw;
688 int idx = hwc->idx;
689
690 /*
691 * Must be done before we disable, otherwise the nmi handler
692 * could reenable again:
693 */
694 clear_bit(idx, cpuc->active);
695 x86_pmu.disable(hwc, idx);
696
697 /*
698 * Make sure the cleared pointer becomes visible before we
699 * (potentially) free the counter:
700 */
701 barrier();
702
703 /*
704 * Drain the remaining delta count out of a counter
705 * that we are disabling:
706 */
707 x86_perf_counter_update(counter, hwc, idx);
708 cpuc->counters[idx] = NULL;
709 clear_bit(idx, cpuc->used);
710}
711
712/*
713 * Save and restart an expired counter. Called by NMI contexts,
714 * so it has to be careful about preempting normal counter ops:
715 */
716static void intel_pmu_save_and_restart(struct perf_counter *counter)
717{
718 struct hw_perf_counter *hwc = &counter->hw;
719 int idx = hwc->idx;
720
721 x86_perf_counter_update(counter, hwc, idx);
722 x86_perf_counter_set_period(counter, hwc, idx);
723
724 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
725 intel_pmu_enable_counter(hwc, idx);
726}
727
728/*
729 * Maximum interrupt frequency of 100KHz per CPU
730 */
731#define PERFMON_MAX_INTERRUPTS (100000/HZ)
732
733/*
734 * This handler is triggered by the local APIC, so the APIC IRQ handling
735 * rules apply:
736 */
737static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
738{
739 int bit, cpu = smp_processor_id();
740 u64 ack, status;
741 struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
742 int ret = 0;
743
744 cpuc->throttle_ctrl = intel_pmu_save_disable_all();
745
746 status = intel_pmu_get_status(cpuc->throttle_ctrl);
747 if (!status)
748 goto out;
749
750 ret = 1;
751again:
752 inc_irq_stat(apic_perf_irqs);
753 ack = status;
754 for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
755 struct perf_counter *counter = cpuc->counters[bit];
756
757 clear_bit(bit, (unsigned long *) &status);
758 if (!test_bit(bit, cpuc->active))
759 continue;
760
761 intel_pmu_save_and_restart(counter);
762 if (perf_counter_overflow(counter, nmi, regs, 0))
763 intel_pmu_disable_counter(&counter->hw, bit);
764 }
765
766 intel_pmu_ack_status(ack);
767
768 /*
769 * Repeat if there is more work to be done:
770 */
771 status = intel_pmu_get_status(cpuc->throttle_ctrl);
772 if (status)
773 goto again;
774out:
775 /*
776 * Restore - do not reenable when global enable is off or throttled:
777 */
778 if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
779 intel_pmu_restore_all(cpuc->throttle_ctrl);
780
781 return ret;
782}
783
784static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi) { return 0; }
785
786void perf_counter_unthrottle(void)
787{
788 struct cpu_hw_counters *cpuc;
789
790 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
791 return;
792
793 if (unlikely(!perf_counters_initialized))
794 return;
795
796 cpuc = &__get_cpu_var(cpu_hw_counters);
797 if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
798 if (printk_ratelimit())
799 printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
800 hw_perf_restore(cpuc->throttle_ctrl);
801 }
802 cpuc->interrupts = 0;
803}
804
805void smp_perf_counter_interrupt(struct pt_regs *regs)
806{
807 irq_enter();
808 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
809 ack_APIC_irq();
810 x86_pmu.handle_irq(regs, 0);
811 irq_exit();
812}
813
814void smp_perf_pending_interrupt(struct pt_regs *regs)
815{
816 irq_enter();
817 ack_APIC_irq();
818 inc_irq_stat(apic_pending_irqs);
819 perf_counter_do_pending();
820 irq_exit();
821}
822
823void set_perf_counter_pending(void)
824{
825 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
826}
827
828void perf_counters_lapic_init(int nmi)
829{
830 u32 apic_val;
831
832 if (!perf_counters_initialized)
833 return;
834 /*
835 * Enable the performance counter vector in the APIC LVT:
836 */
837 apic_val = apic_read(APIC_LVTERR);
838
839 apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
840 if (nmi)
841 apic_write(APIC_LVTPC, APIC_DM_NMI);
842 else
843 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
844 apic_write(APIC_LVTERR, apic_val);
845}
846
847static int __kprobes
848perf_counter_nmi_handler(struct notifier_block *self,
849 unsigned long cmd, void *__args)
850{
851 struct die_args *args = __args;
852 struct pt_regs *regs;
853 int ret;
854
855 switch (cmd) {
856 case DIE_NMI:
857 case DIE_NMI_IPI:
858 break;
859
860 default:
861 return NOTIFY_DONE;
862 }
863
864 regs = args->regs;
865
866 apic_write(APIC_LVTPC, APIC_DM_NMI);
867 ret = x86_pmu.handle_irq(regs, 1);
868
869 return ret ? NOTIFY_STOP : NOTIFY_OK;
870}
871
872static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
873 .notifier_call = perf_counter_nmi_handler,
874 .next = NULL,
875 .priority = 1
876};
877
878static struct x86_pmu intel_pmu = {
879 .name = "Intel",
880 .handle_irq = intel_pmu_handle_irq,
881 .save_disable_all = intel_pmu_save_disable_all,
882 .restore_all = intel_pmu_restore_all,
883 .enable = intel_pmu_enable_counter,
884 .disable = intel_pmu_disable_counter,
885 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
886 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
887 .event_map = intel_pmu_event_map,
888 .raw_event = intel_pmu_raw_event,
889 .max_events = ARRAY_SIZE(intel_perfmon_event_map),
890};
891
892static struct x86_pmu amd_pmu = {
893 .name = "AMD",
894 .handle_irq = amd_pmu_handle_irq,
895 .save_disable_all = amd_pmu_save_disable_all,
896 .restore_all = amd_pmu_restore_all,
897 .enable = amd_pmu_enable_counter,
898 .disable = amd_pmu_disable_counter,
899 .eventsel = MSR_K7_EVNTSEL0,
900 .perfctr = MSR_K7_PERFCTR0,
901 .event_map = amd_pmu_event_map,
902 .raw_event = amd_pmu_raw_event,
903 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
904 .num_counters = 4,
905 .counter_bits = 48,
906 .counter_mask = (1ULL << 48) - 1,
907};
908
909static int intel_pmu_init(void)
910{
911 union cpuid10_edx edx;
912 union cpuid10_eax eax;
913 unsigned int unused;
914 unsigned int ebx;
915 int version;
916
917 if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
918 return -ENODEV;
919
920 /*
921 * Check whether the Architectural PerfMon supports
922 * Branch Misses Retired Event or not.
923 */
924 cpuid(10, &eax.full, &ebx, &unused, &edx.full);
925 if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
926 return -ENODEV;
927
928 version = eax.split.version_id;
929 if (version < 2)
930 return -ENODEV;
931
932 x86_pmu = intel_pmu;
933 x86_pmu.version = version;
934 x86_pmu.num_counters = eax.split.num_counters;
935 x86_pmu.num_counters_fixed = edx.split.num_counters_fixed;
936 x86_pmu.counter_bits = eax.split.bit_width;
937 x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
938
939 return 0;
940}
941
942static int amd_pmu_init(void)
943{
944 x86_pmu = amd_pmu;
945 return 0;
946}
947
948void __init init_hw_perf_counters(void)
949{
950 int err;
951
952 switch (boot_cpu_data.x86_vendor) {
953 case X86_VENDOR_INTEL:
954 err = intel_pmu_init();
955 break;
956 case X86_VENDOR_AMD:
957 err = amd_pmu_init();
958 break;
959 default:
960 return;
961 }
962 if (err != 0)
963 return;
964
965 pr_info("%s Performance Monitoring support detected.\n", x86_pmu.name);
966 pr_info("... version: %d\n", x86_pmu.version);
967 pr_info("... bit width: %d\n", x86_pmu.counter_bits);
968
969 pr_info("... num counters: %d\n", x86_pmu.num_counters);
970 if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
971 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
972 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
973 x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
974 }
975 perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
976 perf_max_counters = x86_pmu.num_counters;
977
978 pr_info("... value mask: %016Lx\n", x86_pmu.counter_mask);
979
980 if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
981 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
982 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
983 x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
984 }
985 pr_info("... fixed counters: %d\n", x86_pmu.num_counters_fixed);
986
987 perf_counter_mask |=
988 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
989
990 pr_info("... counter mask: %016Lx\n", perf_counter_mask);
991 perf_counters_initialized = true;
992
993 perf_counters_lapic_init(0);
994 register_die_notifier(&perf_counter_nmi_notifier);
995}
996
997static inline void x86_pmu_read(struct perf_counter *counter)
998{
999 x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1000}
1001
1002static const struct pmu pmu = {
1003 .enable = x86_pmu_enable,
1004 .disable = x86_pmu_disable,
1005 .read = x86_pmu_read,
1006};
1007
1008const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1009{
1010 int err;
1011
1012 err = __hw_perf_counter_init(counter);
1013 if (err)
1014 return ERR_PTR(err);
1015
1016 return &pmu;
1017}
1018
1019/*
1020 * callchain support
1021 */
1022
1023static inline
1024void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1025{
1026 if (entry->nr < MAX_STACK_DEPTH)
1027 entry->ip[entry->nr++] = ip;
1028}
1029
1030static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1031static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1032
1033
1034static void
1035backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1036{
1037 /* Ignore warnings */
1038}
1039
1040static void backtrace_warning(void *data, char *msg)
1041{
1042 /* Ignore warnings */
1043}
1044
1045static int backtrace_stack(void *data, char *name)
1046{
1047 /* Don't bother with IRQ stacks for now */
1048 return -1;
1049}
1050
1051static void backtrace_address(void *data, unsigned long addr, int reliable)
1052{
1053 struct perf_callchain_entry *entry = data;
1054
1055 if (reliable)
1056 callchain_store(entry, addr);
1057}
1058
1059static const struct stacktrace_ops backtrace_ops = {
1060 .warning = backtrace_warning,
1061 .warning_symbol = backtrace_warning_symbol,
1062 .stack = backtrace_stack,
1063 .address = backtrace_address,
1064};
1065
1066static void
1067perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1068{
1069 unsigned long bp;
1070 char *stack;
1071 int nr = entry->nr;
1072
1073 callchain_store(entry, instruction_pointer(regs));
1074
1075 stack = ((char *)regs + sizeof(struct pt_regs));
1076#ifdef CONFIG_FRAME_POINTER
1077 bp = frame_pointer(regs);
1078#else
1079 bp = 0;
1080#endif
1081
1082 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1083
1084 entry->kernel = entry->nr - nr;
1085}
1086
1087
1088struct stack_frame {
1089 const void __user *next_fp;
1090 unsigned long return_address;
1091};
1092
1093static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1094{
1095 int ret;
1096
1097 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1098 return 0;
1099
1100 ret = 1;
1101 pagefault_disable();
1102 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1103 ret = 0;
1104 pagefault_enable();
1105
1106 return ret;
1107}
1108
1109static void
1110perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1111{
1112 struct stack_frame frame;
1113 const void __user *fp;
1114 int nr = entry->nr;
1115
1116 regs = (struct pt_regs *)current->thread.sp0 - 1;
1117 fp = (void __user *)regs->bp;
1118
1119 callchain_store(entry, regs->ip);
1120
1121 while (entry->nr < MAX_STACK_DEPTH) {
1122 frame.next_fp = NULL;
1123 frame.return_address = 0;
1124
1125 if (!copy_stack_frame(fp, &frame))
1126 break;
1127
1128 if ((unsigned long)fp < user_stack_pointer(regs))
1129 break;
1130
1131 callchain_store(entry, frame.return_address);
1132 fp = frame.next_fp;
1133 }
1134
1135 entry->user = entry->nr - nr;
1136}
1137
1138static void
1139perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1140{
1141 int is_user;
1142
1143 if (!regs)
1144 return;
1145
1146 is_user = user_mode(regs);
1147
1148 if (!current || current->pid == 0)
1149 return;
1150
1151 if (is_user && current->state != TASK_RUNNING)
1152 return;
1153
1154 if (!is_user)
1155 perf_callchain_kernel(regs, entry);
1156
1157 if (current->mm)
1158 perf_callchain_user(regs, entry);
1159}
1160
1161struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1162{
1163 struct perf_callchain_entry *entry;
1164
1165 if (in_nmi())
1166 entry = &__get_cpu_var(nmi_entry);
1167 else
1168 entry = &__get_cpu_var(irq_entry);
1169
1170 entry->nr = 0;
1171 entry->hv = 0;
1172 entry->kernel = 0;
1173 entry->user = 0;
1174
1175 perf_do_callchain(regs, entry);
1176
1177 return entry;
1178}
Linux 6.x block layer and io_uring tree(s)