1 /* Performance event support for sparc64.
3 * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
5 * This code is based almost entirely upon the x86 perf event
8 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
9 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
10 * Copyright (C) 2009 Jaswinder Singh Rajput
11 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
12 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
15 #include <linux/perf_event.h>
16 #include <linux/kprobes.h>
17 #include <linux/kernel.h>
18 #include <linux/kdebug.h>
19 #include <linux/mutex.h>
21 #include <asm/cpudata.h>
22 #include <asm/atomic.h>
26 /* Sparc64 chips have two performance counters, 32-bits each, with
27 * overflow interrupts generated on transition from 0xffffffff to 0.
28 * The counters are accessed in one go using a 64-bit register.
30 * Both counters are controlled using a single control register. The
31 * only way to stop all sampling is to clear all of the context (user,
32 * supervisor, hypervisor) sampling enable bits. But these bits apply
33 * to both counters, thus the two counters can't be enabled/disabled
36 * The control register has two event fields, one for each of the two
37 * counters. It's thus nearly impossible to have one counter going
38 * while keeping the other one stopped. Therefore it is possible to
39 * get overflow interrupts for counters not currently "in use" and
40 * that condition must be checked in the overflow interrupt handler.
42 * So we use a hack, in that we program inactive counters with the
43 * "sw_count0" and "sw_count1" events. These count how many times
44 * the instruction "sethi %hi(0xfc000), %g0" is executed. It's an
45 * unusual way to encode a NOP and therefore will not trigger in
49 #define MAX_HWEVENTS 2
50 #define MAX_PERIOD ((1UL << 32) - 1)
52 #define PIC_UPPER_INDEX 0
53 #define PIC_LOWER_INDEX 1
55 struct cpu_hw_events {
56 struct perf_event *events[MAX_HWEVENTS];
57 unsigned long used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
58 unsigned long active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
61 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
63 struct perf_event_map {
67 #define PIC_UPPER 0x01
68 #define PIC_LOWER 0x02
71 #define C(x) PERF_COUNT_HW_CACHE_##x
73 #define CACHE_OP_UNSUPPORTED 0xfffe
74 #define CACHE_OP_NONSENSE 0xffff
76 typedef struct perf_event_map cache_map_t
77 [PERF_COUNT_HW_CACHE_MAX]
78 [PERF_COUNT_HW_CACHE_OP_MAX]
79 [PERF_COUNT_HW_CACHE_RESULT_MAX];
82 const struct perf_event_map *(*event_map)(int);
83 const cache_map_t *cache_map;
94 static const struct perf_event_map ultra3i_perfmon_event_map[] = {
95 [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
96 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
97 [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
98 [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
101 static const struct perf_event_map *ultra3i_event_map(int event_id)
103 return &ultra3i_perfmon_event_map[event_id];
106 static const cache_map_t ultra3i_cache_map = {
109 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
110 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
113 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
114 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
117 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
118 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
123 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
124 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
127 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
128 [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE },
130 [ C(OP_PREFETCH) ] = {
131 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
132 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
137 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
138 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
141 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
142 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
145 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
146 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
151 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
152 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
155 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
156 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
158 [ C(OP_PREFETCH) ] = {
159 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
160 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
165 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
166 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
169 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
170 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
172 [ C(OP_PREFETCH) ] = {
173 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
174 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
179 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
180 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
183 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
184 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
186 [ C(OP_PREFETCH) ] = {
187 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
188 [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED },
193 static const struct sparc_pmu ultra3i_pmu = {
194 .event_map = ultra3i_event_map,
195 .cache_map = &ultra3i_cache_map,
196 .max_events = ARRAY_SIZE(ultra3i_perfmon_event_map),
204 static const struct perf_event_map niagara2_perfmon_event_map[] = {
205 [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
206 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
207 [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
208 [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
209 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
210 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
213 static const struct perf_event_map *niagara2_event_map(int event_id)
215 return &niagara2_perfmon_event_map[event_id];
218 static const struct sparc_pmu niagara2_pmu = {
219 .event_map = niagara2_event_map,
220 .max_events = ARRAY_SIZE(niagara2_perfmon_event_map),
230 static const struct sparc_pmu *sparc_pmu __read_mostly;
232 static u64 event_encoding(u64 event_id, int idx)
234 if (idx == PIC_UPPER_INDEX)
235 event_id <<= sparc_pmu->upper_shift;
237 event_id <<= sparc_pmu->lower_shift;
241 static u64 mask_for_index(int idx)
243 return event_encoding(sparc_pmu->event_mask, idx);
246 static u64 nop_for_index(int idx)
248 return event_encoding(idx == PIC_UPPER_INDEX ?
249 sparc_pmu->upper_nop :
250 sparc_pmu->lower_nop, idx);
253 static inline void sparc_pmu_enable_event(struct hw_perf_event *hwc,
256 u64 val, mask = mask_for_index(idx);
258 val = pcr_ops->read();
259 pcr_ops->write((val & ~mask) | hwc->config);
262 static inline void sparc_pmu_disable_event(struct hw_perf_event *hwc,
265 u64 mask = mask_for_index(idx);
266 u64 nop = nop_for_index(idx);
267 u64 val = pcr_ops->read();
269 pcr_ops->write((val & ~mask) | nop);
272 void hw_perf_enable(void)
274 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
284 val = pcr_ops->read();
286 for (i = 0; i < MAX_HWEVENTS; i++) {
287 struct perf_event *cp = cpuc->events[i];
288 struct hw_perf_event *hwc;
293 val |= hwc->config_base;
299 void hw_perf_disable(void)
301 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
309 val = pcr_ops->read();
310 val &= ~(PCR_UTRACE | PCR_STRACE |
311 sparc_pmu->hv_bit | sparc_pmu->irq_bit);
315 static u32 read_pmc(int idx)
320 if (idx == PIC_UPPER_INDEX)
323 return val & 0xffffffff;
326 static void write_pmc(int idx, u64 val)
328 u64 shift, mask, pic;
331 if (idx == PIC_UPPER_INDEX)
334 mask = ((u64) 0xffffffff) << shift;
343 static int sparc_perf_event_set_period(struct perf_event *event,
344 struct hw_perf_event *hwc, int idx)
346 s64 left = atomic64_read(&hwc->period_left);
347 s64 period = hwc->sample_period;
350 if (unlikely(left <= -period)) {
352 atomic64_set(&hwc->period_left, left);
353 hwc->last_period = period;
357 if (unlikely(left <= 0)) {
359 atomic64_set(&hwc->period_left, left);
360 hwc->last_period = period;
363 if (left > MAX_PERIOD)
366 atomic64_set(&hwc->prev_count, (u64)-left);
368 write_pmc(idx, (u64)(-left) & 0xffffffff);
370 perf_event_update_userpage(event);
375 static int sparc_pmu_enable(struct perf_event *event)
377 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
378 struct hw_perf_event *hwc = &event->hw;
381 if (test_and_set_bit(idx, cpuc->used_mask))
384 sparc_pmu_disable_event(hwc, idx);
386 cpuc->events[idx] = event;
387 set_bit(idx, cpuc->active_mask);
389 sparc_perf_event_set_period(event, hwc, idx);
390 sparc_pmu_enable_event(hwc, idx);
391 perf_event_update_userpage(event);
395 static u64 sparc_perf_event_update(struct perf_event *event,
396 struct hw_perf_event *hwc, int idx)
399 u64 prev_raw_count, new_raw_count;
403 prev_raw_count = atomic64_read(&hwc->prev_count);
404 new_raw_count = read_pmc(idx);
406 if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
407 new_raw_count) != prev_raw_count)
410 delta = (new_raw_count << shift) - (prev_raw_count << shift);
413 atomic64_add(delta, &event->count);
414 atomic64_sub(delta, &hwc->period_left);
416 return new_raw_count;
419 static void sparc_pmu_disable(struct perf_event *event)
421 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
422 struct hw_perf_event *hwc = &event->hw;
425 clear_bit(idx, cpuc->active_mask);
426 sparc_pmu_disable_event(hwc, idx);
430 sparc_perf_event_update(event, hwc, idx);
431 cpuc->events[idx] = NULL;
432 clear_bit(idx, cpuc->used_mask);
434 perf_event_update_userpage(event);
437 static void sparc_pmu_read(struct perf_event *event)
439 struct hw_perf_event *hwc = &event->hw;
440 sparc_perf_event_update(event, hwc, hwc->idx);
443 static void sparc_pmu_unthrottle(struct perf_event *event)
445 struct hw_perf_event *hwc = &event->hw;
446 sparc_pmu_enable_event(hwc, hwc->idx);
449 static atomic_t active_events = ATOMIC_INIT(0);
450 static DEFINE_MUTEX(pmc_grab_mutex);
452 void perf_event_grab_pmc(void)
454 if (atomic_inc_not_zero(&active_events))
457 mutex_lock(&pmc_grab_mutex);
458 if (atomic_read(&active_events) == 0) {
459 if (atomic_read(&nmi_active) > 0) {
460 on_each_cpu(stop_nmi_watchdog, NULL, 1);
461 BUG_ON(atomic_read(&nmi_active) != 0);
463 atomic_inc(&active_events);
465 mutex_unlock(&pmc_grab_mutex);
468 void perf_event_release_pmc(void)
470 if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
471 if (atomic_read(&nmi_active) == 0)
472 on_each_cpu(start_nmi_watchdog, NULL, 1);
473 mutex_unlock(&pmc_grab_mutex);
477 static const struct perf_event_map *sparc_map_cache_event(u64 config)
479 unsigned int cache_type, cache_op, cache_result;
480 const struct perf_event_map *pmap;
482 if (!sparc_pmu->cache_map)
483 return ERR_PTR(-ENOENT);
485 cache_type = (config >> 0) & 0xff;
486 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
487 return ERR_PTR(-EINVAL);
489 cache_op = (config >> 8) & 0xff;
490 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
491 return ERR_PTR(-EINVAL);
493 cache_result = (config >> 16) & 0xff;
494 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
495 return ERR_PTR(-EINVAL);
497 pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
499 if (pmap->encoding == CACHE_OP_UNSUPPORTED)
500 return ERR_PTR(-ENOENT);
502 if (pmap->encoding == CACHE_OP_NONSENSE)
503 return ERR_PTR(-EINVAL);
508 static void hw_perf_event_destroy(struct perf_event *event)
510 perf_event_release_pmc();
513 static int __hw_perf_event_init(struct perf_event *event)
515 struct perf_event_attr *attr = &event->attr;
516 struct hw_perf_event *hwc = &event->hw;
517 const struct perf_event_map *pmap;
520 if (atomic_read(&nmi_active) < 0)
523 if (attr->type == PERF_TYPE_HARDWARE) {
524 if (attr->config >= sparc_pmu->max_events)
526 pmap = sparc_pmu->event_map(attr->config);
527 } else if (attr->type == PERF_TYPE_HW_CACHE) {
528 pmap = sparc_map_cache_event(attr->config);
530 return PTR_ERR(pmap);
534 perf_event_grab_pmc();
535 event->destroy = hw_perf_event_destroy;
537 /* We save the enable bits in the config_base. So to
538 * turn off sampling just write 'config', and to enable
539 * things write 'config | config_base'.
541 hwc->config_base = sparc_pmu->irq_bit;
542 if (!attr->exclude_user)
543 hwc->config_base |= PCR_UTRACE;
544 if (!attr->exclude_kernel)
545 hwc->config_base |= PCR_STRACE;
546 if (!attr->exclude_hv)
547 hwc->config_base |= sparc_pmu->hv_bit;
549 if (!hwc->sample_period) {
550 hwc->sample_period = MAX_PERIOD;
551 hwc->last_period = hwc->sample_period;
552 atomic64_set(&hwc->period_left, hwc->sample_period);
555 enc = pmap->encoding;
556 if (pmap->pic_mask & PIC_UPPER) {
557 hwc->idx = PIC_UPPER_INDEX;
558 enc <<= sparc_pmu->upper_shift;
560 hwc->idx = PIC_LOWER_INDEX;
561 enc <<= sparc_pmu->lower_shift;
568 static const struct pmu pmu = {
569 .enable = sparc_pmu_enable,
570 .disable = sparc_pmu_disable,
571 .read = sparc_pmu_read,
572 .unthrottle = sparc_pmu_unthrottle,
575 const struct pmu *hw_perf_event_init(struct perf_event *event)
577 int err = __hw_perf_event_init(event);
584 void perf_event_print_debug(void)
593 local_irq_save(flags);
595 cpu = smp_processor_id();
597 pcr = pcr_ops->read();
601 pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
604 local_irq_restore(flags);
607 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
608 unsigned long cmd, void *__args)
610 struct die_args *args = __args;
611 struct perf_sample_data data;
612 struct cpu_hw_events *cpuc;
613 struct pt_regs *regs;
616 if (!atomic_read(&active_events))
631 cpuc = &__get_cpu_var(cpu_hw_events);
632 for (idx = 0; idx < MAX_HWEVENTS; idx++) {
633 struct perf_event *event = cpuc->events[idx];
634 struct hw_perf_event *hwc;
637 if (!test_bit(idx, cpuc->active_mask))
640 val = sparc_perf_event_update(event, hwc, idx);
641 if (val & (1ULL << 31))
644 data.period = event->hw.last_period;
645 if (!sparc_perf_event_set_period(event, hwc, idx))
648 if (perf_event_overflow(event, 1, &data, regs))
649 sparc_pmu_disable_event(hwc, idx);
655 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
656 .notifier_call = perf_event_nmi_handler,
659 static bool __init supported_pmu(void)
661 if (!strcmp(sparc_pmu_type, "ultra3i")) {
662 sparc_pmu = &ultra3i_pmu;
665 if (!strcmp(sparc_pmu_type, "niagara2")) {
666 sparc_pmu = &niagara2_pmu;
672 void __init init_hw_perf_events(void)
674 pr_info("Performance events: ");
676 if (!supported_pmu()) {
677 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
681 pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
683 /* All sparc64 PMUs currently have 2 events. But this simple
684 * driver only supports one active event at a time.
688 register_die_notifier(&perf_event_nmi_notifier);