projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bitmap: replace bitmap_{from,to}_u32array
[linux-2.6-block.git] / arch / arm64 / kernel / perf_event.c
CommitLineData
03089688
WD		 1/*
2 * PMU support
3 *
4 * Copyright (C) 2012 ARM Limited
5 * Author: Will Deacon <will.deacon@arm.com>
6 *
7 * This code is based heavily on the ARMv7 perf event code.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */
03089688 21
03089688 22#include <asm/irq_regs.h>
b8cfadfc 23#include <asm/perf_event.h>
bf2d4782 24#include <asm/sysreg.h>
d98ecdac 25#include <asm/virt.h>
03089688 26
dbee3a74 27#include <linux/acpi.h>
6475b2d8
MR		 28#include <linux/of.h>
29#include <linux/perf/arm_pmu.h>
30#include <linux/platform_device.h>
03089688
WD		 31
32/*
33 * ARMv8 PMUv3 Performance Events handling code.
b112c84a 34 * Common event types (some are defined in asm/perf_event.h).
03089688 35 */
03089688 36
90381cba 37/* At least one of the following is required. */
03598fdb
AK		 38#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x08
39#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x1B
03089688 40
90381cba 41/* Common architectural events. */
03598fdb
AK		 42#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x06
43#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x07
90381cba 44#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x09
03598fdb
AK		 45#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x0A
46#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x0B
47#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x0C
48#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x0D
49#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x0E
50#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x0F
51#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x1C
9e9caa6a
DR		 52#define ARMV8_PMUV3_PERFCTR_CHAIN 0x1E
53#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x21
90381cba
DR		 54
55/* Common microarchitectural events. */
03598fdb
AK		 56#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x01
57#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x02
58#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x05
90381cba 59#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x13
03598fdb
AK		 60#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x14
61#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x15
62#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x16
63#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x17
64#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x18
90381cba 65#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x19
03598fdb 66#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x1A
90381cba 67#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x1D
9e9caa6a
DR		 68#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x1F
69#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x20
70#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x22
71#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x23
72#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x24
73#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x25
74#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x26
75#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x27
76#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x28
77#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x29
78#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x2A
79#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x2B
80#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x2C
81#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x2D
03598fdb 82#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x2E
9e9caa6a 83#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x2F
03598fdb
AK		 84#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x30
85
86/* ARMv8 recommended implementation defined event types */
87#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD 0x40
88#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR 0x41
89#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD 0x42
90#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR 0x43
0893f745
AK		 91#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER 0x44
92#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER 0x45
93#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM 0x46
94#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN 0x47
95#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL 0x48
96
03598fdb
AK		 97#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD 0x4C
98#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR 0x4D
99#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD 0x4E
100#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR 0x4F
0893f745
AK		 101#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD 0x50
102#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR 0x51
103#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD 0x52
104#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR 0x53
105
106#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM 0x56
107#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN 0x57
108#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL 0x58
109
110#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD 0x5C
111#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR 0x5D
112#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD 0x5E
113#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR 0x5F
114
115#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD 0x60
116#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR 0x61
117#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED 0x62
118#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED 0x63
119#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL 0x64
120#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH 0x65
121
122#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD 0x66
123#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR 0x67
124#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC 0x68
125#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC 0x69
126#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC 0x6A
127
128#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC 0x6C
129#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC 0x6D
130#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC 0x6E
131#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC 0x6F
132#define ARMV8_IMPDEF_PERFCTR_LD_SPEC 0x70
133#define ARMV8_IMPDEF_PERFCTR_ST_SPEC 0x71
134#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC 0x72
135#define ARMV8_IMPDEF_PERFCTR_DP_SPEC 0x73
136#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC 0x74
137#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC 0x75
138#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC 0x76
139#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC 0x77
140#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC 0x78
141#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC 0x79
142#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC 0x7A
143
144#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC 0x7C
145#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC 0x7D
146#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC 0x7E
147
148#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF 0x81
149#define ARMV8_IMPDEF_PERFCTR_EXC_SVC 0x82
150#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT 0x83
151#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT 0x84
152
153#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ 0x86
154#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ 0x87
155#define ARMV8_IMPDEF_PERFCTR_EXC_SMC 0x88
156
157#define ARMV8_IMPDEF_PERFCTR_EXC_HVC 0x8A
158#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT 0x8B
159#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT 0x8C
160#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER 0x8D
161#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ 0x8E
162#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ 0x8F
163#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC 0x90
164#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC 0x91
165
166#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD 0xA0
167#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR 0xA1
168#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD 0xA2
169#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR 0xA3
170
171#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM 0xA6
172#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN 0xA7
173#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL 0xA8
5f140cce 174
ac82d127 175/* ARMv8 Cortex-A53 specific event types. */
03598fdb 176#define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2
ac82d127 177
d0aa2bff 178/* ARMv8 Cavium ThunderX specific event types. */
03598fdb
AK		 179#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST 0xE9
180#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS 0xEA
181#define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS 0xEB
182#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS 0xEC
183#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS 0xED
62a4dda9 184
03089688 185/* PMUv3 HW events mapping. */
236b9b91
JL		 186
187/*
188 * ARMv8 Architectural defined events, not all of these may
189 * be supported on any given implementation. Undefined events will
190 * be disabled at run-time.
191 */
03089688 192static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
ae2fb7ec 193 PERF_MAP_ALL_UNSUPPORTED,
03598fdb
AK		 194 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
195 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
196 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
197 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
236b9b91 198 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
03598fdb 199 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
236b9b91
JL		 200 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
201 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
202 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
03089688
WD		 203};
204
205static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
206 [PERF_COUNT_HW_CACHE_OP_MAX]
207 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
ae2fb7ec
MR		 208 PERF_CACHE_MAP_ALL_UNSUPPORTED,
209
03598fdb
AK		 210 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
211 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
212 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
213 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
ae2fb7ec 214
236b9b91
JL		 215 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
216 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
217
218 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
219 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB,
220
221 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
222 [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
223
03598fdb
AK		 224 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
225 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
226 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
227 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
03089688
WD		 228};
229
ac82d127
MR		 230static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
231 [PERF_COUNT_HW_CACHE_OP_MAX]
232 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
233 PERF_CACHE_MAP_ALL_UNSUPPORTED,
234
03598fdb 235 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,
ac82d127 236
5cf7fb26
JT		 237 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
238 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
ac82d127
MR		 239};
240
62a4dda9
MR		 241static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
242 [PERF_COUNT_HW_CACHE_OP_MAX]
243 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
244 PERF_CACHE_MAP_ALL_UNSUPPORTED,
245
03598fdb
AK		 246 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
247 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
248 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
249 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
62a4dda9 250
03598fdb
AK		 251 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
252 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
62a4dda9 253
5cf7fb26
JT		 254 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
255 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
62a4dda9
MR		 256};
257
5561b6c5
JT		 258static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
259 [PERF_COUNT_HW_CACHE_OP_MAX]
260 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
261 PERF_CACHE_MAP_ALL_UNSUPPORTED,
262
263 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
264 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
5561b6c5
JT		 265};
266
d0aa2bff
JG		 267static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
268 [PERF_COUNT_HW_CACHE_OP_MAX]
269 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
270 PERF_CACHE_MAP_ALL_UNSUPPORTED,
271
03598fdb
AK		 272 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
273 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
274 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
275 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST,
276 [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,
277 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,
278
03598fdb
AK		 279 [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,
280 [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,
281
282 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
283 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
284 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
285 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
62a4dda9
MR		 286};
287
201a72b2
AK		 288static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
289 [PERF_COUNT_HW_CACHE_OP_MAX]
290 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
291 PERF_CACHE_MAP_ALL_UNSUPPORTED,
292
293 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
294 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
295 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
296 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
297
201a72b2
AK		 298 [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
299 [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
300 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
301 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
302
201a72b2
AK		 303 [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
304 [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
305};
4b1a9e69
AK		 306
307static ssize_t
308armv8pmu_events_sysfs_show(struct device *dev,
309 struct device_attribute *attr, char *page)
310{
311 struct perf_pmu_events_attr *pmu_attr;
312
313 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
314
315 return sprintf(page, "event=0x%03llx\n", pmu_attr->id);
316}
317
9e9caa6a
DR		 318#define ARMV8_EVENT_ATTR_RESOLVE(m) #m
319#define ARMV8_EVENT_ATTR(name, config) \
4b1a9e69
AK		 320 PMU_EVENT_ATTR(name, armv8_event_attr_##name, \
321 config, armv8pmu_events_sysfs_show)
9e9caa6a 322
03598fdb
AK		 323ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR);
324ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL);
325ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL);
326ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL);
327ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE);
328ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL);
329ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED);
330ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED);
331ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED);
9e9caa6a 332ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN);
03598fdb
AK		 333ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN);
334ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED);
335ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED);
336ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED);
337ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED);
338ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED);
339ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED);
340ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES);
341ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED);
9e9caa6a 342ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS);
03598fdb
AK		 343ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE);
344ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB);
345ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE);
346ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL);
347ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB);
9e9caa6a 348ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS);
03598fdb
AK		 349ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR);
350ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC);
351ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED);
9e9caa6a 352ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES);
4ba2578f 353/* Don't expose the chain event in /sys, since it's useless in isolation */
9e9caa6a
DR		 354ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE);
355ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE);
356ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED);
357ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED);
358ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND);
359ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND);
360ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB);
361ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB);
362ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE);
363ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL);
364ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE);
365ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL);
366ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE);
367ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB);
368ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL);
03598fdb 369ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL);
9e9caa6a 370ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB);
03598fdb 371ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB);
9e9caa6a
DR		 372
373static struct attribute *armv8_pmuv3_event_attrs[] = {
374 &armv8_event_attr_sw_incr.attr.attr,
375 &armv8_event_attr_l1i_cache_refill.attr.attr,
376 &armv8_event_attr_l1i_tlb_refill.attr.attr,
377 &armv8_event_attr_l1d_cache_refill.attr.attr,
378 &armv8_event_attr_l1d_cache.attr.attr,
379 &armv8_event_attr_l1d_tlb_refill.attr.attr,
380 &armv8_event_attr_ld_retired.attr.attr,
381 &armv8_event_attr_st_retired.attr.attr,
382 &armv8_event_attr_inst_retired.attr.attr,
383 &armv8_event_attr_exc_taken.attr.attr,
384 &armv8_event_attr_exc_return.attr.attr,
385 &armv8_event_attr_cid_write_retired.attr.attr,
386 &armv8_event_attr_pc_write_retired.attr.attr,
387 &armv8_event_attr_br_immed_retired.attr.attr,
388 &armv8_event_attr_br_return_retired.attr.attr,
389 &armv8_event_attr_unaligned_ldst_retired.attr.attr,
390 &armv8_event_attr_br_mis_pred.attr.attr,
391 &armv8_event_attr_cpu_cycles.attr.attr,
392 &armv8_event_attr_br_pred.attr.attr,
393 &armv8_event_attr_mem_access.attr.attr,
394 &armv8_event_attr_l1i_cache.attr.attr,
395 &armv8_event_attr_l1d_cache_wb.attr.attr,
396 &armv8_event_attr_l2d_cache.attr.attr,
397 &armv8_event_attr_l2d_cache_refill.attr.attr,
398 &armv8_event_attr_l2d_cache_wb.attr.attr,
399 &armv8_event_attr_bus_access.attr.attr,
400 &armv8_event_attr_memory_error.attr.attr,
401 &armv8_event_attr_inst_spec.attr.attr,
402 &armv8_event_attr_ttbr_write_retired.attr.attr,
403 &armv8_event_attr_bus_cycles.attr.attr,
9e9caa6a
DR		 404 &armv8_event_attr_l1d_cache_allocate.attr.attr,
405 &armv8_event_attr_l2d_cache_allocate.attr.attr,
406 &armv8_event_attr_br_retired.attr.attr,
407 &armv8_event_attr_br_mis_pred_retired.attr.attr,
408 &armv8_event_attr_stall_frontend.attr.attr,
409 &armv8_event_attr_stall_backend.attr.attr,
410 &armv8_event_attr_l1d_tlb.attr.attr,
411 &armv8_event_attr_l1i_tlb.attr.attr,
412 &armv8_event_attr_l2i_cache.attr.attr,
413 &armv8_event_attr_l2i_cache_refill.attr.attr,
414 &armv8_event_attr_l3d_cache_allocate.attr.attr,
415 &armv8_event_attr_l3d_cache_refill.attr.attr,
416 &armv8_event_attr_l3d_cache.attr.attr,
417 &armv8_event_attr_l3d_cache_wb.attr.attr,
418 &armv8_event_attr_l2d_tlb_refill.attr.attr,
03598fdb 419 &armv8_event_attr_l2i_tlb_refill.attr.attr,
9e9caa6a 420 &armv8_event_attr_l2d_tlb.attr.attr,
03598fdb 421 &armv8_event_attr_l2i_tlb.attr.attr,
57d74123 422 NULL,
9e9caa6a
DR		 423};
424
4b1a9e69
AK		 425static umode_t
426armv8pmu_event_attr_is_visible(struct kobject *kobj,
427 struct attribute *attr, int unused)
428{
429 struct device *dev = kobj_to_dev(kobj);
430 struct pmu *pmu = dev_get_drvdata(dev);
431 struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
432 struct perf_pmu_events_attr *pmu_attr;
433
434 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
435
436 if (test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
437 return attr->mode;
438
439 return 0;
440}
441
9e9caa6a
DR		 442static struct attribute_group armv8_pmuv3_events_attr_group = {
443 .name = "events",
444 .attrs = armv8_pmuv3_event_attrs,
4b1a9e69 445 .is_visible = armv8pmu_event_attr_is_visible,
9e9caa6a
DR		 446};
447
fe7296e1 448PMU_FORMAT_ATTR(event, "config:0-15");
57d74123
WD		 449
450static struct attribute *armv8_pmuv3_format_attrs[] = {
451 &format_attr_event.attr,
452 NULL,
453};
454
455static struct attribute_group armv8_pmuv3_format_attr_group = {
456 .name = "format",
457 .attrs = armv8_pmuv3_format_attrs,
458};
459
03089688
WD		 460/*
461 * Perf Events' indices
462 */
463#define ARMV8_IDX_CYCLE_COUNTER 0
464#define ARMV8_IDX_COUNTER0 1
6475b2d8
MR		 465#define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
466 (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
03089688 467
03089688
WD		 468/*
469 * ARMv8 low level PMU access
470 */
471
472/*
473 * Perf Event to low level counters mapping
474 */
475#define ARMV8_IDX_TO_COUNTER(x) \
b8cfadfc 476 (((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
03089688
WD		 477
478static inline u32 armv8pmu_pmcr_read(void)
479{
bf2d4782 480 return read_sysreg(pmcr_el0);
03089688
WD		 481}
482
483static inline void armv8pmu_pmcr_write(u32 val)
484{
b8cfadfc 485 val &= ARMV8_PMU_PMCR_MASK;
03089688 486 isb();
bf2d4782 487 write_sysreg(val, pmcr_el0);
03089688
WD		 488}
489
490static inline int armv8pmu_has_overflowed(u32 pmovsr)
491{
b8cfadfc 492 return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
03089688
WD		 493}
494
6475b2d8 495static inline int armv8pmu_counter_valid(struct arm_pmu *cpu_pmu, int idx)
03089688 496{
6475b2d8
MR		 497 return idx >= ARMV8_IDX_CYCLE_COUNTER &&
498 idx <= ARMV8_IDX_COUNTER_LAST(cpu_pmu);
03089688
WD		 499}
500
501static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
502{
6475b2d8 503 return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
03089688
WD		 504}
505
506static inline int armv8pmu_select_counter(int idx)
507{
6475b2d8 508 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
bf2d4782 509 write_sysreg(counter, pmselr_el0);
03089688
WD		 510 isb();
511
512 return idx;
513}
514
6475b2d8 515static inline u32 armv8pmu_read_counter(struct perf_event *event)
03089688 516{
6475b2d8
MR		 517 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
518 struct hw_perf_event *hwc = &event->hw;
519 int idx = hwc->idx;
03089688
WD		 520 u32 value = 0;
521
6475b2d8 522 if (!armv8pmu_counter_valid(cpu_pmu, idx))
03089688
WD		 523 pr_err("CPU%u reading wrong counter %d\n",
524 smp_processor_id(), idx);
525 else if (idx == ARMV8_IDX_CYCLE_COUNTER)
bf2d4782 526 value = read_sysreg(pmccntr_el0);
03089688 527 else if (armv8pmu_select_counter(idx) == idx)
bf2d4782 528 value = read_sysreg(pmxevcntr_el0);
03089688
WD		 529
530 return value;
531}
532
6475b2d8 533static inline void armv8pmu_write_counter(struct perf_event *event, u32 value)
03089688 534{
6475b2d8
MR		 535 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
536 struct hw_perf_event *hwc = &event->hw;
537 int idx = hwc->idx;
538
539 if (!armv8pmu_counter_valid(cpu_pmu, idx))
03089688
WD		 540 pr_err("CPU%u writing wrong counter %d\n",
541 smp_processor_id(), idx);
7175f059
JG		 542 else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
543 /*
544 * Set the upper 32bits as this is a 64bit counter but we only
545 * count using the lower 32bits and we want an interrupt when
546 * it overflows.
547 */
548 u64 value64 = 0xffffffff00000000ULL | value;
549
bf2d4782 550 write_sysreg(value64, pmccntr_el0);
7175f059 551 } else if (armv8pmu_select_counter(idx) == idx)
bf2d4782 552 write_sysreg(value, pmxevcntr_el0);
03089688
WD		 553}
554
555static inline void armv8pmu_write_evtype(int idx, u32 val)
556{
557 if (armv8pmu_select_counter(idx) == idx) {
b8cfadfc 558 val &= ARMV8_PMU_EVTYPE_MASK;
bf2d4782 559 write_sysreg(val, pmxevtyper_el0);
03089688
WD		 560 }
561}
562
563static inline int armv8pmu_enable_counter(int idx)
564{
6475b2d8 565 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
bf2d4782 566 write_sysreg(BIT(counter), pmcntenset_el0);
03089688
WD		 567 return idx;
568}
569
570static inline int armv8pmu_disable_counter(int idx)
571{
6475b2d8 572 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
bf2d4782 573 write_sysreg(BIT(counter), pmcntenclr_el0);
03089688
WD		 574 return idx;
575}
576
577static inline int armv8pmu_enable_intens(int idx)
578{
6475b2d8 579 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
bf2d4782 580 write_sysreg(BIT(counter), pmintenset_el1);
03089688
WD		 581 return idx;
582}
583
584static inline int armv8pmu_disable_intens(int idx)
585{
6475b2d8 586 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
bf2d4782 587 write_sysreg(BIT(counter), pmintenclr_el1);
03089688
WD		 588 isb();
589 /* Clear the overflow flag in case an interrupt is pending. */
bf2d4782 590 write_sysreg(BIT(counter), pmovsclr_el0);
03089688 591 isb();
6475b2d8 592
03089688
WD		 593 return idx;
594}
595
596static inline u32 armv8pmu_getreset_flags(void)
597{
598 u32 value;
599
600 /* Read */
bf2d4782 601 value = read_sysreg(pmovsclr_el0);
03089688
WD		 602
603 /* Write to clear flags */
b8cfadfc 604 value &= ARMV8_PMU_OVSR_MASK;
bf2d4782 605 write_sysreg(value, pmovsclr_el0);
03089688
WD		 606
607 return value;
608}
609
6475b2d8 610static void armv8pmu_enable_event(struct perf_event *event)
03089688
WD		 611{
612 unsigned long flags;
6475b2d8
MR		 613 struct hw_perf_event *hwc = &event->hw;
614 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
615 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
616 int idx = hwc->idx;
03089688
WD		 617
618 /*
619 * Enable counter and interrupt, and set the counter to count
620 * the event that we're interested in.
621 */
622 raw_spin_lock_irqsave(&events->pmu_lock, flags);
623
624 /*
625 * Disable counter
626 */
627 armv8pmu_disable_counter(idx);
628
629 /*
630 * Set event (if destined for PMNx counters).
631 */
632 armv8pmu_write_evtype(idx, hwc->config_base);
633
634 /*
635 * Enable interrupt for this counter
636 */
637 armv8pmu_enable_intens(idx);
638
639 /*
640 * Enable counter
641 */
642 armv8pmu_enable_counter(idx);
643
644 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
645}
646
6475b2d8 647static void armv8pmu_disable_event(struct perf_event *event)
03089688
WD		 648{
649 unsigned long flags;
6475b2d8
MR		 650 struct hw_perf_event *hwc = &event->hw;
651 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
652 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
653 int idx = hwc->idx;
03089688
WD		 654
655 /*
656 * Disable counter and interrupt
657 */
658 raw_spin_lock_irqsave(&events->pmu_lock, flags);
659
660 /*
661 * Disable counter
662 */
663 armv8pmu_disable_counter(idx);
664
665 /*
666 * Disable interrupt for this counter
667 */
668 armv8pmu_disable_intens(idx);
669
670 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
671}
672
673static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
674{
675 u32 pmovsr;
676 struct perf_sample_data data;
6475b2d8
MR		 677 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
678 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
03089688
WD		 679 struct pt_regs *regs;
680 int idx;
681
682 /*
683 * Get and reset the IRQ flags
684 */
685 pmovsr = armv8pmu_getreset_flags();
686
687 /*
688 * Did an overflow occur?
689 */
690 if (!armv8pmu_has_overflowed(pmovsr))
691 return IRQ_NONE;
692
693 /*
694 * Handle the counter(s) overflow(s)
695 */
696 regs = get_irq_regs();
697
03089688
WD		 698 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
699 struct perf_event *event = cpuc->events[idx];
700 struct hw_perf_event *hwc;
701
702 /* Ignore if we don't have an event. */
703 if (!event)
704 continue;
705
706 /*
707 * We have a single interrupt for all counters. Check that
708 * each counter has overflowed before we process it.
709 */
710 if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
711 continue;
712
713 hwc = &event->hw;
6475b2d8 714 armpmu_event_update(event);
03089688 715 perf_sample_data_init(&data, 0, hwc->last_period);
6475b2d8 716 if (!armpmu_event_set_period(event))
03089688
WD		 717 continue;
718
719 if (perf_event_overflow(event, &data, regs))
6475b2d8 720 cpu_pmu->disable(event);
03089688
WD		 721 }
722
723 /*
724 * Handle the pending perf events.
725 *
726 * Note: this call *must* be run with interrupts disabled. For
727 * platforms that can have the PMU interrupts raised as an NMI, this
728 * will not work.
729 */
730 irq_work_run();
731
732 return IRQ_HANDLED;
733}
734
6475b2d8 735static void armv8pmu_start(struct arm_pmu *cpu_pmu)
03089688
WD		 736{
737 unsigned long flags;
6475b2d8 738 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
03089688
WD		 739
740 raw_spin_lock_irqsave(&events->pmu_lock, flags);
741 /* Enable all counters */
b8cfadfc 742 armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
03089688
WD		 743 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
744}
745
6475b2d8 746static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
03089688
WD		 747{
748 unsigned long flags;
6475b2d8 749 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
03089688
WD		 750
751 raw_spin_lock_irqsave(&events->pmu_lock, flags);
752 /* Disable all counters */
b8cfadfc 753 armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
03089688
WD		 754 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
755}
756
757static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
6475b2d8 758 struct perf_event *event)
03089688
WD		 759{
760 int idx;
6475b2d8
MR		 761 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
762 struct hw_perf_event *hwc = &event->hw;
b8cfadfc 763 unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
03089688 764
1031a159 765 /* Always prefer to place a cycle counter into the cycle counter. */
03598fdb 766 if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
1031a159
PA		 767 if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
768 return ARMV8_IDX_CYCLE_COUNTER;
03089688
WD		 769 }
770
771 /*
1031a159 772 * Otherwise use events counters
03089688
WD		 773 */
774 for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
775 if (!test_and_set_bit(idx, cpuc->used_mask))
776 return idx;
777 }
778
779 /* The counters are all in use. */
780 return -EAGAIN;
781}
782
783/*
784 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
785 */
786static int armv8pmu_set_event_filter(struct hw_perf_event *event,
787 struct perf_event_attr *attr)
788{
789 unsigned long config_base = 0;
790
791 if (attr->exclude_idle)
792 return -EPERM;
78a19cfd
GK		 793
794 /*
795 * If we're running in hyp mode, then we *are* the hypervisor.
796 * Therefore we ignore exclude_hv in this configuration, since
797 * there's no hypervisor to sample anyway. This is consistent
798 * with other architectures (x86 and Power).
799 */
800 if (is_kernel_in_hyp_mode()) {
801 if (!attr->exclude_kernel)
802 config_base |= ARMV8_PMU_INCLUDE_EL2;
803 } else {
804 if (attr->exclude_kernel)
805 config_base |= ARMV8_PMU_EXCLUDE_EL1;
806 if (!attr->exclude_hv)
807 config_base |= ARMV8_PMU_INCLUDE_EL2;
808 }
03089688 809 if (attr->exclude_user)
b8cfadfc 810 config_base |= ARMV8_PMU_EXCLUDE_EL0;
03089688
WD		 811
812 /*
813 * Install the filter into config_base as this is used to
814 * construct the event type.
815 */
816 event->config_base = config_base;
817
818 return 0;
819}
820
821static void armv8pmu_reset(void *info)
822{
6475b2d8 823 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
03089688
WD		 824 u32 idx, nb_cnt = cpu_pmu->num_events;
825
826 /* The counter and interrupt enable registers are unknown at reset. */
6475b2d8
MR		 827 for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
828 armv8pmu_disable_counter(idx);
829 armv8pmu_disable_intens(idx);
830 }
03089688 831
7175f059
JG		 832 /*
833 * Initialize & Reset PMNC. Request overflow interrupt for
834 * 64 bit cycle counter but cheat in armv8pmu_write_counter().
835 */
b8cfadfc
SZ		 836 armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C |
837 ARMV8_PMU_PMCR_LC);
03089688
WD		 838}
839
6c833bb9
WD		 840static int __armv8_pmuv3_map_event(struct perf_event *event,
841 const unsigned (*extra_event_map)
842 [PERF_COUNT_HW_MAX],
843 const unsigned (*extra_cache_map)
844 [PERF_COUNT_HW_CACHE_MAX]
845 [PERF_COUNT_HW_CACHE_OP_MAX]
846 [PERF_COUNT_HW_CACHE_RESULT_MAX])
03089688 847{
236b9b91
JL		 848 int hw_event_id;
849 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
850
851 hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map,
852 &armv8_pmuv3_perf_cache_map,
853 ARMV8_PMU_EVTYPE_EVENT);
236b9b91 854
6c833bb9
WD		 855 /* Onl expose micro/arch events supported by this PMU */
856 if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
857 && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
858 return hw_event_id;
236b9b91
JL		 859 }
860
6c833bb9
WD		 861 return armpmu_map_event(event, extra_event_map, extra_cache_map,
862 ARMV8_PMU_EVTYPE_EVENT);
863}
864
865static int armv8_pmuv3_map_event(struct perf_event *event)
866{
867 return __armv8_pmuv3_map_event(event, NULL, NULL);
03089688
WD		 868}
869
ac82d127
MR		 870static int armv8_a53_map_event(struct perf_event *event)
871{
d0d09d4d 872 return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map);
ac82d127
MR		 873}
874
62a4dda9
MR		 875static int armv8_a57_map_event(struct perf_event *event)
876{
d0d09d4d 877 return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map);
62a4dda9
MR		 878}
879
5561b6c5
JT		 880static int armv8_a73_map_event(struct perf_event *event)
881{
882 return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map);
883}
884
d0aa2bff
JG		 885static int armv8_thunder_map_event(struct perf_event *event)
886{
d0d09d4d 887 return __armv8_pmuv3_map_event(event, NULL,
6c833bb9 888 &armv8_thunder_perf_cache_map);
d0aa2bff
JG		 889}
890
201a72b2
AK		 891static int armv8_vulcan_map_event(struct perf_event *event)
892{
d0d09d4d 893 return __armv8_pmuv3_map_event(event, NULL,
6c833bb9 894 &armv8_vulcan_perf_cache_map);
201a72b2
AK		 895}
896
f1b36dcb
MR		 897struct armv8pmu_probe_info {
898 struct arm_pmu *pmu;
899 bool present;
900};
901
4b1a9e69 902static void __armv8pmu_probe_pmu(void *info)
03089688 903{
f1b36dcb
MR		 904 struct armv8pmu_probe_info *probe = info;
905 struct arm_pmu *cpu_pmu = probe->pmu;
faa9a083 906 u64 dfr0;
4b1a9e69 907 u32 pmceid[2];
faa9a083 908 int pmuver;
03089688 909
f1b36dcb 910 dfr0 = read_sysreg(id_aa64dfr0_el1);
faa9a083 911 pmuver = cpuid_feature_extract_signed_field(dfr0,
f1b36dcb 912 ID_AA64DFR0_PMUVER_SHIFT);
faa9a083 913 if (pmuver < 1)
f1b36dcb
MR		 914 return;
915
916 probe->present = true;
917
03089688 918 /* Read the nb of CNTx counters supported from PMNC */
4b1a9e69
AK		 919 cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
920 & ARMV8_PMU_PMCR_N_MASK;
03089688 921
6475b2d8 922 /* Add the CPU cycles counter */
4b1a9e69
AK		 923 cpu_pmu->num_events += 1;
924
925 pmceid[0] = read_sysreg(pmceid0_el0);
926 pmceid[1] = read_sysreg(pmceid1_el0);
927
3aa56885
YN		 928 bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
929 pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
03089688
WD		 930}
931
4b1a9e69 932static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
03089688 933{
f1b36dcb
MR		 934 struct armv8pmu_probe_info probe = {
935 .pmu = cpu_pmu,
936 .present = false,
937 };
938 int ret;
939
940 ret = smp_call_function_any(&cpu_pmu->supported_cpus,
4b1a9e69 941 __armv8pmu_probe_pmu,
f1b36dcb
MR		 942 &probe, 1);
943 if (ret)
944 return ret;
945
946 return probe.present ? 0 : -ENODEV;
03089688
WD		 947}
948
f1b36dcb 949static int armv8_pmu_init(struct arm_pmu *cpu_pmu)
03089688 950{
f1b36dcb
MR		 951 int ret = armv8pmu_probe_pmu(cpu_pmu);
952 if (ret)
953 return ret;
954
6475b2d8
MR		 955 cpu_pmu->handle_irq = armv8pmu_handle_irq,
956 cpu_pmu->enable = armv8pmu_enable_event,
957 cpu_pmu->disable = armv8pmu_disable_event,
958 cpu_pmu->read_counter = armv8pmu_read_counter,
959 cpu_pmu->write_counter = armv8pmu_write_counter,
960 cpu_pmu->get_event_idx = armv8pmu_get_event_idx,
961 cpu_pmu->start = armv8pmu_start,
962 cpu_pmu->stop = armv8pmu_stop,
963 cpu_pmu->reset = armv8pmu_reset,
964 cpu_pmu->max_period = (1LLU << 32) - 1,
ac82d127 965 cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
f1b36dcb
MR		 966
967 return 0;
ac82d127
MR		 968}
969
970static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
971{
f1b36dcb
MR		 972 int ret = armv8_pmu_init(cpu_pmu);
973 if (ret)
974 return ret;
975
6475b2d8
MR		 976 cpu_pmu->name = "armv8_pmuv3";
977 cpu_pmu->map_event = armv8_pmuv3_map_event;
569de902
MR		 978 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
979 &armv8_pmuv3_events_attr_group;
980 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
981 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 982
983 return 0;
ac82d127
MR		 984}
985
e884f80c
JT		 986static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
987{
988 int ret = armv8_pmu_init(cpu_pmu);
989 if (ret)
990 return ret;
991
992 cpu_pmu->name = "armv8_cortex_a35";
993 cpu_pmu->map_event = armv8_a53_map_event;
994 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
995 &armv8_pmuv3_events_attr_group;
996 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
997 &armv8_pmuv3_format_attr_group;
998
999 return 0;
1000}
1001
ac82d127
MR		 1002static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1003{
f1b36dcb
MR		 1004 int ret = armv8_pmu_init(cpu_pmu);
1005 if (ret)
1006 return ret;
1007
ac82d127
MR		 1008 cpu_pmu->name = "armv8_cortex_a53";
1009 cpu_pmu->map_event = armv8_a53_map_event;
569de902
MR		 1010 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1011 &armv8_pmuv3_events_attr_group;
1012 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1013 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 1014
1015 return 0;
03089688 1016}
03089688 1017
62a4dda9
MR		 1018static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
1019{
f1b36dcb
MR		 1020 int ret = armv8_pmu_init(cpu_pmu);
1021 if (ret)
1022 return ret;
1023
62a4dda9
MR		 1024 cpu_pmu->name = "armv8_cortex_a57";
1025 cpu_pmu->map_event = armv8_a57_map_event;
569de902
MR		 1026 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1027 &armv8_pmuv3_events_attr_group;
1028 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1029 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 1030
1031 return 0;
62a4dda9
MR		 1032}
1033
5d7ee877
WD		 1034static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
1035{
f1b36dcb
MR		 1036 int ret = armv8_pmu_init(cpu_pmu);
1037 if (ret)
1038 return ret;
1039
5d7ee877
WD		 1040 cpu_pmu->name = "armv8_cortex_a72";
1041 cpu_pmu->map_event = armv8_a57_map_event;
569de902
MR		 1042 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1043 &armv8_pmuv3_events_attr_group;
1044 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1045 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 1046
1047 return 0;
5d7ee877
WD		 1048}
1049
5561b6c5
JT		 1050static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
1051{
1052 int ret = armv8_pmu_init(cpu_pmu);
1053 if (ret)
1054 return ret;
1055
1056 cpu_pmu->name = "armv8_cortex_a73";
1057 cpu_pmu->map_event = armv8_a73_map_event;
1058 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1059 &armv8_pmuv3_events_attr_group;
1060 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1061 &armv8_pmuv3_format_attr_group;
1062
1063 return 0;
1064}
1065
d0aa2bff
JG		 1066static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
1067{
f1b36dcb
MR		 1068 int ret = armv8_pmu_init(cpu_pmu);
1069 if (ret)
1070 return ret;
1071
d0aa2bff
JG		 1072 cpu_pmu->name = "armv8_cavium_thunder";
1073 cpu_pmu->map_event = armv8_thunder_map_event;
569de902
MR		 1074 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1075 &armv8_pmuv3_events_attr_group;
1076 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1077 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 1078
1079 return 0;
d0aa2bff
JG		 1080}
1081
201a72b2
AK		 1082static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
1083{
f1b36dcb
MR		 1084 int ret = armv8_pmu_init(cpu_pmu);
1085 if (ret)
1086 return ret;
1087
201a72b2
AK		 1088 cpu_pmu->name = "armv8_brcm_vulcan";
1089 cpu_pmu->map_event = armv8_vulcan_map_event;
569de902
MR		 1090 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1091 &armv8_pmuv3_events_attr_group;
1092 cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1093 &armv8_pmuv3_format_attr_group;
f1b36dcb
MR		 1094
1095 return 0;
201a72b2
AK		 1096}
1097
6475b2d8
MR		 1098static const struct of_device_id armv8_pmu_of_device_ids[] = {
1099 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},
e884f80c 1100 {.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init},
ac82d127 1101 {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},
62a4dda9 1102 {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},
5d7ee877 1103 {.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init},
5561b6c5 1104 {.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init},
d0aa2bff 1105 {.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init},
201a72b2 1106 {.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init},
03089688
WD		 1107 {},
1108};
1109
6475b2d8 1110static int armv8_pmu_device_probe(struct platform_device *pdev)
03089688 1111{
f00fa5f4 1112 return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
03089688
WD		 1113}
1114
6475b2d8 1115static struct platform_driver armv8_pmu_driver = {
03089688 1116 .driver = {
85023b2e 1117 .name = ARMV8_PMU_PDEV_NAME,
6475b2d8 1118 .of_match_table = armv8_pmu_of_device_ids,
03089688 1119 },
6475b2d8 1120 .probe = armv8_pmu_device_probe,
03089688
WD		 1121};
1122
f00fa5f4
MR		 1123static int __init armv8_pmu_driver_init(void)
1124{
1125 if (acpi_disabled)
1126 return platform_driver_register(&armv8_pmu_driver);
1127 else
1128 return arm_pmu_acpi_probe(armv8_pmuv3_init);
1129}
1130device_initcall(armv8_pmu_driver_init)
Linux 6.x block layer and io_uring tree(s)