projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86: Add Intel Processor Trace (INTEL_PT) cpu feature detection
[linux-2.6-block.git] / arch / x86 / kernel / cpu / perf_event.h
CommitLineData
de0428a7
KW		 1/*
2 * Performance events x86 architecture header
3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
15#include <linux/perf_event.h>
16
1c2ac3fd
PZ		 17#if 0
18#undef wrmsrl
19#define wrmsrl(msr, val) \
20do { \
21 unsigned int _msr = (msr); \
22 u64 _val = (val); \
23 trace_printk("wrmsrl(%x, %Lx)\n", (unsigned int)(_msr), \
24 (unsigned long long)(_val)); \
25 native_write_msr((_msr), (u32)(_val), (u32)(_val >> 32)); \
26} while (0)
27#endif
28
de0428a7
KW		 29/*
30 * | NHM/WSM | SNB |
31 * register -------------------------------
32 * | HT | no HT | HT | no HT |
33 *-----------------------------------------
34 * offcore | core | core | cpu | core |
35 * lbr_sel | core | core | cpu | core |
36 * ld_lat | cpu | core | cpu | core |
37 *-----------------------------------------
38 *
39 * Given that there is a small number of shared regs,
40 * we can pre-allocate their slot in the per-cpu
41 * per-core reg tables.
42 */
43enum extra_reg_type {
44 EXTRA_REG_NONE = -1, /* not used */
45
46 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
47 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
b36817e8 48 EXTRA_REG_LBR = 2, /* lbr_select */
f20093ee 49 EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
de0428a7
KW		 50
51 EXTRA_REG_MAX /* number of entries needed */
52};
53
54struct event_constraint {
55 union {
56 unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
57 u64 idxmsk64;
58 };
59 u64 code;
60 u64 cmask;
61 int weight;
bc1738f6 62 int overlap;
9fac2cf3 63 int flags;
de0428a7 64};
f20093ee 65/*
2f7f73a5 66 * struct hw_perf_event.flags flags
f20093ee
SE		 67 */
68#define PERF_X86_EVENT_PEBS_LDLAT 0x1 /* ld+ldlat data address sampling */
9ad64c0f 69#define PERF_X86_EVENT_PEBS_ST 0x2 /* st data address sampling */
86a04461 70#define PERF_X86_EVENT_PEBS_ST_HSW 0x4 /* haswell style datala, store */
2f7f73a5 71#define PERF_X86_EVENT_COMMITTED 0x8 /* event passed commit_txn */
86a04461
AK		 72#define PERF_X86_EVENT_PEBS_LD_HSW 0x10 /* haswell style datala, load */
73#define PERF_X86_EVENT_PEBS_NA_HSW 0x20 /* haswell style datala, unknown */
7911d3f7
AL		 74#define PERF_X86_EVENT_RDPMC_ALLOWED 0x40 /* grant rdpmc permission */
75
de0428a7
KW		 76
77struct amd_nb {
78 int nb_id; /* NorthBridge id */
79 int refcnt; /* reference count */
80 struct perf_event *owners[X86_PMC_IDX_MAX];
81 struct event_constraint event_constraints[X86_PMC_IDX_MAX];
82};
83
84/* The maximal number of PEBS events: */
70ab7003 85#define MAX_PEBS_EVENTS 8
de0428a7
KW		 86
87/*
88 * A debug store configuration.
89 *
90 * We only support architectures that use 64bit fields.
91 */
92struct debug_store {
93 u64 bts_buffer_base;
94 u64 bts_index;
95 u64 bts_absolute_maximum;
96 u64 bts_interrupt_threshold;
97 u64 pebs_buffer_base;
98 u64 pebs_index;
99 u64 pebs_absolute_maximum;
100 u64 pebs_interrupt_threshold;
101 u64 pebs_event_reset[MAX_PEBS_EVENTS];
102};
103
104/*
105 * Per register state.
106 */
107struct er_account {
108 raw_spinlock_t lock; /* per-core: protect structure */
109 u64 config; /* extra MSR config */
110 u64 reg; /* extra MSR number */
111 atomic_t ref; /* reference count */
112};
113
114/*
115 * Per core/cpu state
116 *
117 * Used to coordinate shared registers between HT threads or
118 * among events on a single PMU.
119 */
120struct intel_shared_regs {
121 struct er_account regs[EXTRA_REG_MAX];
122 int refcnt; /* per-core: #HT threads */
123 unsigned core_id; /* per-core: core id */
124};
125
126#define MAX_LBR_ENTRIES 16
127
128struct cpu_hw_events {
129 /*
130 * Generic x86 PMC bits
131 */
132 struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
133 unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
134 unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
135 int enabled;
136
c347a2f1
PZ		 137 int n_events; /* the # of events in the below arrays */
138 int n_added; /* the # last events in the below arrays;
139 they've never been enabled yet */
140 int n_txn; /* the # last events in the below arrays;
141 added in the current transaction */
de0428a7
KW		 142 int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
143 u64 tags[X86_PMC_IDX_MAX];
144 struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
145
146 unsigned int group_flag;
5a425294 147 int is_fake;
de0428a7
KW		 148
149 /*
150 * Intel DebugStore bits
151 */
152 struct debug_store *ds;
153 u64 pebs_enabled;
154
155 /*
156 * Intel LBR bits
157 */
158 int lbr_users;
159 void *lbr_context;
160 struct perf_branch_stack lbr_stack;
161 struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
b36817e8 162 struct er_account *lbr_sel;
3e702ff6 163 u64 br_sel;
de0428a7 164
144d31e6
GN		 165 /*
166 * Intel host/guest exclude bits
167 */
168 u64 intel_ctrl_guest_mask;
169 u64 intel_ctrl_host_mask;
170 struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
171
2b9e344d
PZ		 172 /*
173 * Intel checkpoint mask
174 */
175 u64 intel_cp_status;
176
de0428a7
KW		 177 /*
178 * manage shared (per-core, per-cpu) registers
179 * used on Intel NHM/WSM/SNB
180 */
181 struct intel_shared_regs *shared_regs;
182
183 /*
184 * AMD specific bits
185 */
1018faa6
JR		 186 struct amd_nb *amd_nb;
187 /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
188 u64 perf_ctr_virt_mask;
de0428a7
KW		 189
190 void *kfree_on_online;
191};
192
9fac2cf3 193#define __EVENT_CONSTRAINT(c, n, m, w, o, f) {\
de0428a7
KW		 194 { .idxmsk64 = (n) }, \
195 .code = (c), \
196 .cmask = (m), \
197 .weight = (w), \
bc1738f6 198 .overlap = (o), \
9fac2cf3 199 .flags = f, \
de0428a7
KW		 200}
201
202#define EVENT_CONSTRAINT(c, n, m) \
9fac2cf3 203 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
bc1738f6
RR		 204
205/*
206 * The overlap flag marks event constraints with overlapping counter
207 * masks. This is the case if the counter mask of such an event is not
208 * a subset of any other counter mask of a constraint with an equal or
209 * higher weight, e.g.:
210 *
211 * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
212 * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
213 * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
214 *
215 * The event scheduler may not select the correct counter in the first
216 * cycle because it needs to know which subsequent events will be
217 * scheduled. It may fail to schedule the events then. So we set the
218 * overlap flag for such constraints to give the scheduler a hint which
219 * events to select for counter rescheduling.
220 *
221 * Care must be taken as the rescheduling algorithm is O(n!) which
222 * will increase scheduling cycles for an over-commited system
223 * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
224 * and its counter masks must be kept at a minimum.
225 */
226#define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
9fac2cf3 227 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
de0428a7
KW		 228
229/*
230 * Constraint on the Event code.
231 */
232#define INTEL_EVENT_CONSTRAINT(c, n) \
233 EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
234
235/*
236 * Constraint on the Event code + UMask + fixed-mask
237 *
238 * filter mask to validate fixed counter events.
239 * the following filters disqualify for fixed counters:
240 * - inv
241 * - edge
242 * - cnt-mask
3a632cb2
AK		 243 * - in_tx
244 * - in_tx_checkpointed
de0428a7
KW		 245 * The other filters are supported by fixed counters.
246 * The any-thread option is supported starting with v3.
247 */
3a632cb2 248#define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
de0428a7 249#define FIXED_EVENT_CONSTRAINT(c, n) \
3a632cb2 250 EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
de0428a7
KW		 251
252/*
253 * Constraint on the Event code + UMask
254 */
255#define INTEL_UEVENT_CONSTRAINT(c, n) \
256 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
257
7550ddff
AK		 258/* Like UEVENT_CONSTRAINT, but match flags too */
259#define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \
260 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
261
f20093ee 262#define INTEL_PLD_CONSTRAINT(c, n) \
86a04461 263 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
f20093ee
SE		 264 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
265
9ad64c0f 266#define INTEL_PST_CONSTRAINT(c, n) \
86a04461 267 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
9ad64c0f
SE		 268 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
269
86a04461
AK		 270/* Event constraint, but match on all event flags too. */
271#define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
272 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
273
274/* Check only flags, but allow all event/umask */
275#define INTEL_ALL_EVENT_CONSTRAINT(code, n) \
276 EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
277
278/* Check flags and event code, and set the HSW store flag */
279#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
280 __EVENT_CONSTRAINT(code, n, \
281 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
282 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
283
284/* Check flags and event code, and set the HSW load flag */
285#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
286 __EVENT_CONSTRAINT(code, n, \
287 ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
288 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
289
290/* Check flags and event code/umask, and set the HSW store flag */
291#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
292 __EVENT_CONSTRAINT(code, n, \
293 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
f9134f36
AK		 294 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
295
86a04461
AK		 296/* Check flags and event code/umask, and set the HSW load flag */
297#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
298 __EVENT_CONSTRAINT(code, n, \
299 INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
300 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
301
302/* Check flags and event code/umask, and set the HSW N/A flag */
303#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
304 __EVENT_CONSTRAINT(code, n, \
305 INTEL_ARCH_EVENT_MASK|INTEL_ARCH_EVENT_MASK, \
306 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
307
308
cf30d52e
MD		 309/*
310 * We define the end marker as having a weight of -1
311 * to enable blacklisting of events using a counter bitmask
312 * of zero and thus a weight of zero.
313 * The end marker has a weight that cannot possibly be
314 * obtained from counting the bits in the bitmask.
315 */
316#define EVENT_CONSTRAINT_END { .weight = -1 }
de0428a7 317
cf30d52e
MD		 318/*
319 * Check for end marker with weight == -1
320 */
de0428a7 321#define for_each_event_constraint(e, c) \
cf30d52e 322 for ((e) = (c); (e)->weight != -1; (e)++)
de0428a7
KW		 323
324/*
325 * Extra registers for specific events.
326 *
327 * Some events need large masks and require external MSRs.
328 * Those extra MSRs end up being shared for all events on
329 * a PMU and sometimes between PMU of sibling HT threads.
330 * In either case, the kernel needs to handle conflicting
331 * accesses to those extra, shared, regs. The data structure
332 * to manage those registers is stored in cpu_hw_event.
333 */
334struct extra_reg {
335 unsigned int event;
336 unsigned int msr;
337 u64 config_mask;
338 u64 valid_mask;
339 int idx; /* per_xxx->regs[] reg index */
338b522c 340 bool extra_msr_access;
de0428a7
KW		 341};
342
343#define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
338b522c
KL		 344 .event = (e), \
345 .msr = (ms), \
346 .config_mask = (m), \
347 .valid_mask = (vm), \
348 .idx = EXTRA_REG_##i, \
349 .extra_msr_access = true, \
de0428a7
KW		 350 }
351
352#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
353 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
354
f20093ee
SE		 355#define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
356 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
357 ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
358
359#define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
360 INTEL_UEVENT_EXTRA_REG(c, \
361 MSR_PEBS_LD_LAT_THRESHOLD, \
362 0xffff, \
363 LDLAT)
364
de0428a7
KW		 365#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
366
367union perf_capabilities {
368 struct {
369 u64 lbr_format:6;
370 u64 pebs_trap:1;
371 u64 pebs_arch_reg:1;
372 u64 pebs_format:4;
373 u64 smm_freeze:1;
069e0c3c
AK		 374 /*
375 * PMU supports separate counter range for writing
376 * values > 32bit.
377 */
378 u64 full_width_write:1;
de0428a7
KW		 379 };
380 u64 capabilities;
381};
382
c1d6f42f
PZ		 383struct x86_pmu_quirk {
384 struct x86_pmu_quirk *next;
385 void (*func)(void);
386};
387
f9b4eeb8
PZ		 388union x86_pmu_config {
389 struct {
390 u64 event:8,
391 umask:8,
392 usr:1,
393 os:1,
394 edge:1,
395 pc:1,
396 interrupt:1,
397 __reserved1:1,
398 en:1,
399 inv:1,
400 cmask:8,
401 event2:4,
402 __reserved2:4,
403 go:1,
404 ho:1;
405 } bits;
406 u64 value;
407};
408
409#define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
410
de0428a7
KW		 411/*
412 * struct x86_pmu - generic x86 pmu
413 */
414struct x86_pmu {
415 /*
416 * Generic x86 PMC bits
417 */
418 const char *name;
419 int version;
420 int (*handle_irq)(struct pt_regs *);
421 void (*disable_all)(void);
422 void (*enable_all)(int added);
423 void (*enable)(struct perf_event *);
424 void (*disable)(struct perf_event *);
425 int (*hw_config)(struct perf_event *event);
426 int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
427 unsigned eventsel;
428 unsigned perfctr;
4c1fd17a 429 int (*addr_offset)(int index, bool eventsel);
0fbdad07 430 int (*rdpmc_index)(int index);
de0428a7
KW		 431 u64 (*event_map)(int);
432 int max_events;
433 int num_counters;
434 int num_counters_fixed;
435 int cntval_bits;
436 u64 cntval_mask;
ffb871bc
GN		 437 union {
438 unsigned long events_maskl;
439 unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
440 };
441 int events_mask_len;
de0428a7
KW		 442 int apic;
443 u64 max_period;
444 struct event_constraint *
445 (*get_event_constraints)(struct cpu_hw_events *cpuc,
446 struct perf_event *event);
447
448 void (*put_event_constraints)(struct cpu_hw_events *cpuc,
449 struct perf_event *event);
450 struct event_constraint *event_constraints;
c1d6f42f 451 struct x86_pmu_quirk *quirks;
de0428a7 452 int perfctr_second_write;
72db5596 453 bool late_ack;
294fe0f5 454 unsigned (*limit_period)(struct perf_event *event, unsigned l);
de0428a7 455
0c9d42ed
PZ		 456 /*
457 * sysfs attrs
458 */
e97df763 459 int attr_rdpmc_broken;
0c9d42ed 460 int attr_rdpmc;
641cc938 461 struct attribute **format_attrs;
f20093ee 462 struct attribute **event_attrs;
0c9d42ed 463
a4747393 464 ssize_t (*events_sysfs_show)(char *page, u64 config);
1a6461b1 465 struct attribute **cpu_events;
a4747393 466
0c9d42ed
PZ		 467 /*
468 * CPU Hotplug hooks
469 */
de0428a7
KW		 470 int (*cpu_prepare)(int cpu);
471 void (*cpu_starting)(int cpu);
472 void (*cpu_dying)(int cpu);
473 void (*cpu_dead)(int cpu);
c93dc84c
PZ		 474
475 void (*check_microcode)(void);
ba532500
YZ		 476 void (*sched_task)(struct perf_event_context *ctx,
477 bool sched_in);
de0428a7
KW		 478
479 /*
480 * Intel Arch Perfmon v2+
481 */
482 u64 intel_ctrl;
483 union perf_capabilities intel_cap;
484
485 /*
486 * Intel DebugStore bits
487 */
597ed953 488 unsigned int bts :1,
3e0091e2
PZ		 489 bts_active :1,
490 pebs :1,
491 pebs_active :1,
492 pebs_broken :1;
de0428a7
KW		 493 int pebs_record_size;
494 void (*drain_pebs)(struct pt_regs *regs);
495 struct event_constraint *pebs_constraints;
0780c927 496 void (*pebs_aliases)(struct perf_event *event);
70ab7003 497 int max_pebs_events;
de0428a7
KW		 498
499 /*
500 * Intel LBR
501 */
502 unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */
503 int lbr_nr; /* hardware stack size */
b36817e8
SE		 504 u64 lbr_sel_mask; /* LBR_SELECT valid bits */
505 const int *lbr_sel_map; /* lbr_select mappings */
b7af41a1 506 bool lbr_double_abort; /* duplicated lbr aborts */
de0428a7
KW		 507
508 /*
509 * Extra registers for events
510 */
511 struct extra_reg *extra_regs;
512 unsigned int er_flags;
144d31e6
GN		 513
514 /*
515 * Intel host/guest support (KVM)
516 */
517 struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
de0428a7
KW		 518};
519
e18bf526
YZ		 520struct x86_perf_task_context {
521 u64 lbr_from[MAX_LBR_ENTRIES];
522 u64 lbr_to[MAX_LBR_ENTRIES];
523 int lbr_callstack_users;
524 int lbr_stack_state;
525};
526
c1d6f42f
PZ		 527#define x86_add_quirk(func_) \
528do { \
529 static struct x86_pmu_quirk __quirk __initdata = { \
530 .func = func_, \
531 }; \
532 __quirk.next = x86_pmu.quirks; \
533 x86_pmu.quirks = &__quirk; \
534} while (0)
535
de0428a7
KW		 536#define ERF_NO_HT_SHARING 1
537#define ERF_HAS_RSP_1 2
538
3a54aaa0
SE		 539#define EVENT_VAR(_id) event_attr_##_id
540#define EVENT_PTR(_id) &event_attr_##_id.attr.attr
541
542#define EVENT_ATTR(_name, _id) \
543static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
544 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
545 .id = PERF_COUNT_HW_##_id, \
546 .event_str = NULL, \
547};
548
549#define EVENT_ATTR_STR(_name, v, str) \
550static struct perf_pmu_events_attr event_attr_##v = { \
551 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
552 .id = 0, \
553 .event_str = str, \
554};
555
de0428a7
KW		 556extern struct x86_pmu x86_pmu __read_mostly;
557
e9d7f7cd
YZ		 558static inline bool x86_pmu_has_lbr_callstack(void)
559{
560 return x86_pmu.lbr_sel_map &&
561 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
562}
563
de0428a7
KW		 564DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
565
566int x86_perf_event_set_period(struct perf_event *event);
567
568/*
569 * Generalized hw caching related hw_event table, filled
570 * in on a per model basis. A value of 0 means
571 * 'not supported', -1 means 'hw_event makes no sense on
572 * this CPU', any other value means the raw hw_event
573 * ID.
574 */
575
576#define C(x) PERF_COUNT_HW_CACHE_##x
577
578extern u64 __read_mostly hw_cache_event_ids
579 [PERF_COUNT_HW_CACHE_MAX]
580 [PERF_COUNT_HW_CACHE_OP_MAX]
581 [PERF_COUNT_HW_CACHE_RESULT_MAX];
582extern u64 __read_mostly hw_cache_extra_regs
583 [PERF_COUNT_HW_CACHE_MAX]
584 [PERF_COUNT_HW_CACHE_OP_MAX]
585 [PERF_COUNT_HW_CACHE_RESULT_MAX];
586
587u64 x86_perf_event_update(struct perf_event *event);
588
de0428a7
KW		 589static inline unsigned int x86_pmu_config_addr(int index)
590{
4c1fd17a
JS		 591 return x86_pmu.eventsel + (x86_pmu.addr_offset ?
592 x86_pmu.addr_offset(index, true) : index);
de0428a7
KW		 593}
594
595static inline unsigned int x86_pmu_event_addr(int index)
596{
4c1fd17a
JS		 597 return x86_pmu.perfctr + (x86_pmu.addr_offset ?
598 x86_pmu.addr_offset(index, false) : index);
de0428a7
KW		 599}
600
0fbdad07
JS		 601static inline int x86_pmu_rdpmc_index(int index)
602{
603 return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
604}
605
de0428a7
KW		 606int x86_setup_perfctr(struct perf_event *event);
607
608int x86_pmu_hw_config(struct perf_event *event);
609
610void x86_pmu_disable_all(void);
611
612static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
613 u64 enable_mask)
614{
1018faa6
JR		 615 u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
616
de0428a7
KW		 617 if (hwc->extra_reg.reg)
618 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
1018faa6 619 wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
de0428a7
KW		 620}
621
622void x86_pmu_enable_all(int added);
623
43b45780 624int perf_assign_events(struct perf_event **events, int n,
4b4969b1 625 int wmin, int wmax, int *assign);
de0428a7
KW		 626int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
627
628void x86_pmu_stop(struct perf_event *event, int flags);
629
630static inline void x86_pmu_disable_event(struct perf_event *event)
631{
632 struct hw_perf_event *hwc = &event->hw;
633
634 wrmsrl(hwc->config_base, hwc->config);
635}
636
637void x86_pmu_enable_event(struct perf_event *event);
638
639int x86_pmu_handle_irq(struct pt_regs *regs);
640
641extern struct event_constraint emptyconstraint;
642
643extern struct event_constraint unconstrained;
644
3e702ff6
SE		 645static inline bool kernel_ip(unsigned long ip)
646{
647#ifdef CONFIG_X86_32
648 return ip > PAGE_OFFSET;
649#else
650 return (long)ip < 0;
651#endif
652}
653
d07bdfd3
PZ		 654/*
655 * Not all PMUs provide the right context information to place the reported IP
656 * into full context. Specifically segment registers are typically not
657 * supplied.
658 *
659 * Assuming the address is a linear address (it is for IBS), we fake the CS and
660 * vm86 mode using the known zero-based code segment and 'fix up' the registers
661 * to reflect this.
662 *
663 * Intel PEBS/LBR appear to typically provide the effective address, nothing
664 * much we can do about that but pray and treat it like a linear address.
665 */
666static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
667{
668 regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
669 if (regs->flags & X86_VM_MASK)
670 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
671 regs->ip = ip;
672}
673
0bf79d44 674ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
20550a43 675ssize_t intel_event_sysfs_show(char *page, u64 config);
43c032fe 676
de0428a7
KW		 677#ifdef CONFIG_CPU_SUP_AMD
678
679int amd_pmu_init(void);
680
681#else /* CONFIG_CPU_SUP_AMD */
682
683static inline int amd_pmu_init(void)
684{
685 return 0;
686}
687
688#endif /* CONFIG_CPU_SUP_AMD */
689
690#ifdef CONFIG_CPU_SUP_INTEL
691
692int intel_pmu_save_and_restart(struct perf_event *event);
693
694struct event_constraint *
695x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event);
696
697struct intel_shared_regs *allocate_shared_regs(int cpu);
698
699int intel_pmu_init(void);
700
701void init_debug_store_on_cpu(int cpu);
702
703void fini_debug_store_on_cpu(int cpu);
704
705void release_ds_buffers(void);
706
707void reserve_ds_buffers(void);
708
709extern struct event_constraint bts_constraint;
710
711void intel_pmu_enable_bts(u64 config);
712
713void intel_pmu_disable_bts(void);
714
715int intel_pmu_drain_bts_buffer(void);
716
717extern struct event_constraint intel_core2_pebs_event_constraints[];
718
719extern struct event_constraint intel_atom_pebs_event_constraints[];
720
1fa64180
YZ		 721extern struct event_constraint intel_slm_pebs_event_constraints[];
722
de0428a7
KW		 723extern struct event_constraint intel_nehalem_pebs_event_constraints[];
724
725extern struct event_constraint intel_westmere_pebs_event_constraints[];
726
727extern struct event_constraint intel_snb_pebs_event_constraints[];
728
20a36e39
SE		 729extern struct event_constraint intel_ivb_pebs_event_constraints[];
730
3044318f
AK		 731extern struct event_constraint intel_hsw_pebs_event_constraints[];
732
de0428a7
KW		 733struct event_constraint *intel_pebs_constraints(struct perf_event *event);
734
735void intel_pmu_pebs_enable(struct perf_event *event);
736
737void intel_pmu_pebs_disable(struct perf_event *event);
738
739void intel_pmu_pebs_enable_all(void);
740
741void intel_pmu_pebs_disable_all(void);
742
743void intel_ds_init(void);
744
2a0ad3b3
YZ		 745void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
746
de0428a7
KW		 747void intel_pmu_lbr_reset(void);
748
749void intel_pmu_lbr_enable(struct perf_event *event);
750
751void intel_pmu_lbr_disable(struct perf_event *event);
752
753void intel_pmu_lbr_enable_all(void);
754
755void intel_pmu_lbr_disable_all(void);
756
757void intel_pmu_lbr_read(void);
758
759void intel_pmu_lbr_init_core(void);
760
761void intel_pmu_lbr_init_nhm(void);
762
763void intel_pmu_lbr_init_atom(void);
764
c5cc2cd9
SE		 765void intel_pmu_lbr_init_snb(void);
766
e9d7f7cd
YZ		 767void intel_pmu_lbr_init_hsw(void);
768
60ce0fbd
SE		 769int intel_pmu_setup_lbr_filter(struct perf_event *event);
770
de0428a7
KW		 771int p4_pmu_init(void);
772
773int p6_pmu_init(void);
774
e717bf4e
VW		 775int knc_pmu_init(void);
776
f20093ee
SE		 777ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
778 char *page);
779
de0428a7
KW		 780#else /* CONFIG_CPU_SUP_INTEL */
781
782static inline void reserve_ds_buffers(void)
783{
784}
785
786static inline void release_ds_buffers(void)
787{
788}
789
790static inline int intel_pmu_init(void)
791{
792 return 0;
793}
794
795static inline struct intel_shared_regs *allocate_shared_regs(int cpu)
796{
797 return NULL;
798}
799
800#endif /* CONFIG_CPU_SUP_INTEL */
Linux 6.x block layer and io_uring tree(s)