Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* |
3 | * Copyright (C) 2004 Anton Blanchard <anton@au.ibm.com>, IBM | |
078f1940 | 4 | * Added mmcra[slot] support: |
5 | * Copyright (C) 2006-2007 Will Schmidt <willschm@us.ibm.com>, IBM | |
1da177e4 LT |
6 | */ |
7 | ||
8 | #include <linux/oprofile.h> | |
1da177e4 | 9 | #include <linux/smp.h> |
57cfb814 | 10 | #include <asm/firmware.h> |
1da177e4 | 11 | #include <asm/ptrace.h> |
1da177e4 LT |
12 | #include <asm/processor.h> |
13 | #include <asm/cputable.h> | |
1da177e4 | 14 | #include <asm/rtas.h> |
dca85932 | 15 | #include <asm/oprofile_impl.h> |
cb09cff3 | 16 | #include <asm/reg.h> |
1da177e4 LT |
17 | |
18 | #define dbg(args...) | |
adbf115d CL |
19 | #define OPROFILE_PM_PMCSEL_MSK 0xffULL |
20 | #define OPROFILE_PM_UNIT_SHIFT 60 | |
21 | #define OPROFILE_PM_UNIT_MSK 0xfULL | |
22 | #define OPROFILE_MAX_PMC_NUM 3 | |
23 | #define OPROFILE_PMSEL_FIELD_WIDTH 8 | |
24 | #define OPROFILE_UNIT_FIELD_WIDTH 4 | |
25 | #define MMCRA_SIAR_VALID_MASK 0x10000000ULL | |
1da177e4 | 26 | |
1da177e4 LT |
27 | static unsigned long reset_value[OP_MAX_COUNTER]; |
28 | ||
1da177e4 | 29 | static int oprofile_running; |
e5fc948b | 30 | static int use_slot_nums; |
1da177e4 LT |
31 | |
32 | /* mmcr values are set in power4_reg_setup, used in power4_cpu_setup */ | |
33 | static u32 mmcr0_val; | |
34 | static u64 mmcr1_val; | |
15e812ad | 35 | static u64 mmcra_val; |
adbf115d CL |
36 | static u32 cntr_marked_events; |
37 | ||
38 | static int power7_marked_instr_event(u64 mmcr1) | |
39 | { | |
40 | u64 psel, unit; | |
41 | int pmc, cntr_marked_events = 0; | |
42 | ||
43 | /* Given the MMCR1 value, look at the field for each counter to | |
44 | * determine if it is a marked event. Code based on the function | |
45 | * power7_marked_instr_event() in file arch/powerpc/perf/power7-pmu.c. | |
46 | */ | |
47 | for (pmc = 0; pmc < 4; pmc++) { | |
48 | psel = mmcr1 & (OPROFILE_PM_PMCSEL_MSK | |
49 | << (OPROFILE_MAX_PMC_NUM - pmc) | |
46ed7a76 | 50 | * OPROFILE_PMSEL_FIELD_WIDTH); |
adbf115d CL |
51 | psel = (psel >> ((OPROFILE_MAX_PMC_NUM - pmc) |
52 | * OPROFILE_PMSEL_FIELD_WIDTH)) & ~1ULL; | |
53 | unit = mmcr1 & (OPROFILE_PM_UNIT_MSK | |
54 | << (OPROFILE_PM_UNIT_SHIFT | |
55 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH ))); | |
56 | unit = unit >> (OPROFILE_PM_UNIT_SHIFT | |
57 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH)); | |
58 | ||
59 | switch (psel >> 4) { | |
60 | case 2: | |
61 | cntr_marked_events |= (pmc == 1 || pmc == 3) << pmc; | |
62 | break; | |
63 | case 3: | |
64 | if (psel == 0x3c) { | |
65 | cntr_marked_events |= (pmc == 0) << pmc; | |
66 | break; | |
67 | } | |
68 | ||
69 | if (psel == 0x3e) { | |
70 | cntr_marked_events |= (pmc != 1) << pmc; | |
71 | break; | |
72 | } | |
73 | ||
74 | cntr_marked_events |= 1 << pmc; | |
75 | break; | |
76 | case 4: | |
77 | case 5: | |
78 | cntr_marked_events |= (unit == 0xd) << pmc; | |
79 | break; | |
80 | case 6: | |
81 | if (psel == 0x64) | |
82 | cntr_marked_events |= (pmc >= 2) << pmc; | |
83 | break; | |
84 | case 8: | |
85 | cntr_marked_events |= (unit == 0xd) << pmc; | |
86 | break; | |
87 | } | |
88 | } | |
89 | return cntr_marked_events; | |
90 | } | |
1da177e4 | 91 | |
1474855d | 92 | static int power4_reg_setup(struct op_counter_config *ctr, |
1da177e4 LT |
93 | struct op_system_config *sys, |
94 | int num_ctrs) | |
95 | { | |
96 | int i; | |
97 | ||
1da177e4 LT |
98 | /* |
99 | * The performance counter event settings are given in the mmcr0, | |
100 | * mmcr1 and mmcra values passed from the user in the | |
101 | * op_system_config structure (sys variable). | |
102 | */ | |
103 | mmcr0_val = sys->mmcr0; | |
104 | mmcr1_val = sys->mmcr1; | |
105 | mmcra_val = sys->mmcra; | |
106 | ||
adbf115d CL |
107 | /* Power 7+ and newer architectures: |
108 | * Determine which counter events in the group (the group of events is | |
109 | * specified by the bit settings in the MMCR1 register) are marked | |
110 | * events for use in the interrupt handler. Do the calculation once | |
111 | * before OProfile starts. Information is used in the interrupt | |
112 | * handler. Starting with Power 7+ we only record the sample for | |
113 | * marked events if the SIAR valid bit is set. For non marked events | |
114 | * the sample is always recorded. | |
115 | */ | |
116 | if (pvr_version_is(PVR_POWER7p)) | |
117 | cntr_marked_events = power7_marked_instr_event(mmcr1_val); | |
118 | else | |
119 | cntr_marked_events = 0; /* For older processors, set the bit map | |
120 | * to zero so the sample will always be | |
121 | * be recorded. | |
122 | */ | |
123 | ||
a6908cd0 | 124 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) |
1da177e4 LT |
125 | reset_value[i] = 0x80000000UL - ctr[i].count; |
126 | ||
127 | /* setup user and kernel profiling */ | |
128 | if (sys->enable_kernel) | |
129 | mmcr0_val &= ~MMCR0_KERNEL_DISABLE; | |
130 | else | |
131 | mmcr0_val |= MMCR0_KERNEL_DISABLE; | |
132 | ||
133 | if (sys->enable_user) | |
134 | mmcr0_val &= ~MMCR0_PROBLEM_DISABLE; | |
135 | else | |
136 | mmcr0_val |= MMCR0_PROBLEM_DISABLE; | |
1474855d | 137 | |
d3dbeef6 ME |
138 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
139 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
140 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX) || | |
141 | pvr_version_is(PVR_POWER5) || pvr_version_is(PVR_POWER5p)) | |
e5fc948b MJ |
142 | use_slot_nums = 1; |
143 | ||
1474855d | 144 | return 0; |
1da177e4 LT |
145 | } |
146 | ||
b950bdd0 | 147 | extern void ppc_enable_pmcs(void); |
1da177e4 | 148 | |
cb09cff3 AB |
149 | /* |
150 | * Older CPUs require the MMCRA sample bit to be always set, but newer | |
151 | * CPUs only want it set for some groups. Eventually we will remove all | |
152 | * knowledge of this bit in the kernel, oprofile userspace should be | |
153 | * setting it when required. | |
154 | * | |
155 | * In order to keep current installations working we force the bit for | |
156 | * those older CPUs. Once everyone has updated their oprofile userspace we | |
157 | * can remove this hack. | |
158 | */ | |
159 | static inline int mmcra_must_set_sample(void) | |
160 | { | |
d3dbeef6 ME |
161 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
162 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
163 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX)) | |
cb09cff3 AB |
164 | return 1; |
165 | ||
166 | return 0; | |
167 | } | |
168 | ||
1474855d | 169 | static int power4_cpu_setup(struct op_counter_config *ctr) |
1da177e4 LT |
170 | { |
171 | unsigned int mmcr0 = mmcr0_val; | |
172 | unsigned long mmcra = mmcra_val; | |
173 | ||
b950bdd0 | 174 | ppc_enable_pmcs(); |
1da177e4 LT |
175 | |
176 | /* set the freeze bit */ | |
177 | mmcr0 |= MMCR0_FC; | |
178 | mtspr(SPRN_MMCR0, mmcr0); | |
179 | ||
180 | mmcr0 |= MMCR0_FCM1|MMCR0_PMXE|MMCR0_FCECE; | |
181 | mmcr0 |= MMCR0_PMC1CE|MMCR0_PMCjCE; | |
182 | mtspr(SPRN_MMCR0, mmcr0); | |
183 | ||
184 | mtspr(SPRN_MMCR1, mmcr1_val); | |
185 | ||
cb09cff3 AB |
186 | if (mmcra_must_set_sample()) |
187 | mmcra |= MMCRA_SAMPLE_ENABLE; | |
1da177e4 LT |
188 | mtspr(SPRN_MMCRA, mmcra); |
189 | ||
190 | dbg("setup on cpu %d, mmcr0 %lx\n", smp_processor_id(), | |
191 | mfspr(SPRN_MMCR0)); | |
192 | dbg("setup on cpu %d, mmcr1 %lx\n", smp_processor_id(), | |
193 | mfspr(SPRN_MMCR1)); | |
194 | dbg("setup on cpu %d, mmcra %lx\n", smp_processor_id(), | |
195 | mfspr(SPRN_MMCRA)); | |
1474855d BN |
196 | |
197 | return 0; | |
1da177e4 LT |
198 | } |
199 | ||
1474855d | 200 | static int power4_start(struct op_counter_config *ctr) |
1da177e4 LT |
201 | { |
202 | int i; | |
203 | unsigned int mmcr0; | |
204 | ||
205 | /* set the PMM bit (see comment below) */ | |
1c539731 | 206 | mtmsr(mfmsr() | MSR_PMM); |
1da177e4 | 207 | |
a6908cd0 | 208 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
1da177e4 | 209 | if (ctr[i].enabled) { |
c69b767a | 210 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 211 | } else { |
c69b767a | 212 | classic_ctr_write(i, 0); |
1da177e4 LT |
213 | } |
214 | } | |
215 | ||
216 | mmcr0 = mfspr(SPRN_MMCR0); | |
217 | ||
218 | /* | |
219 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
220 | * all the time | |
221 | */ | |
222 | mmcr0 &= ~MMCR0_PMAO; | |
223 | ||
224 | /* | |
225 | * now clear the freeze bit, counting will not start until we | |
226 | * rfid from this excetion, because only at that point will | |
227 | * the PMM bit be cleared | |
228 | */ | |
229 | mmcr0 &= ~MMCR0_FC; | |
230 | mtspr(SPRN_MMCR0, mmcr0); | |
231 | ||
232 | oprofile_running = 1; | |
233 | ||
234 | dbg("start on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
1474855d | 235 | return 0; |
1da177e4 LT |
236 | } |
237 | ||
238 | static void power4_stop(void) | |
239 | { | |
240 | unsigned int mmcr0; | |
241 | ||
242 | /* freeze counters */ | |
243 | mmcr0 = mfspr(SPRN_MMCR0); | |
244 | mmcr0 |= MMCR0_FC; | |
245 | mtspr(SPRN_MMCR0, mmcr0); | |
246 | ||
247 | oprofile_running = 0; | |
248 | ||
249 | dbg("stop on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
250 | ||
251 | mb(); | |
252 | } | |
253 | ||
254 | /* Fake functions used by canonicalize_pc */ | |
3ff6eecc | 255 | static void __used hypervisor_bucket(void) |
1da177e4 LT |
256 | { |
257 | } | |
258 | ||
3ff6eecc | 259 | static void __used rtas_bucket(void) |
1da177e4 LT |
260 | { |
261 | } | |
262 | ||
3ff6eecc | 263 | static void __used kernel_unknown_bucket(void) |
1da177e4 LT |
264 | { |
265 | } | |
266 | ||
1da177e4 LT |
267 | /* |
268 | * On GQ and newer the MMCRA stores the HV and PR bits at the time | |
269 | * the SIAR was sampled. We use that to work out if the SIAR was sampled in | |
270 | * the hypervisor, our exception vectors or RTAS. | |
078f1940 | 271 | * If the MMCRA_SAMPLE_ENABLE bit is set, we can use the MMCRA[slot] bits |
272 | * to more accurately identify the address of the sampled instruction. The | |
273 | * mmcra[slot] bits represent the slot number of a sampled instruction | |
274 | * within an instruction group. The slot will contain a value between 1 | |
275 | * and 5 if MMCRA_SAMPLE_ENABLE is set, otherwise 0. | |
1da177e4 LT |
276 | */ |
277 | static unsigned long get_pc(struct pt_regs *regs) | |
278 | { | |
279 | unsigned long pc = mfspr(SPRN_SIAR); | |
280 | unsigned long mmcra; | |
078f1940 | 281 | unsigned long slot; |
1da177e4 | 282 | |
25985edc | 283 | /* Can't do much about it */ |
e78dbc80 | 284 | if (!cur_cpu_spec->oprofile_mmcra_sihv) |
15e812ad | 285 | return pc; |
1da177e4 LT |
286 | |
287 | mmcra = mfspr(SPRN_MMCRA); | |
288 | ||
e5fc948b | 289 | if (use_slot_nums && (mmcra & MMCRA_SAMPLE_ENABLE)) { |
078f1940 | 290 | slot = ((mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT); |
291 | if (slot > 1) | |
292 | pc += 4 * (slot - 1); | |
293 | } | |
294 | ||
1da177e4 | 295 | /* Were we in the hypervisor? */ |
e78dbc80 MN |
296 | if (firmware_has_feature(FW_FEATURE_LPAR) && |
297 | (mmcra & cur_cpu_spec->oprofile_mmcra_sihv)) | |
1da177e4 LT |
298 | /* function descriptor madness */ |
299 | return *((unsigned long *)hypervisor_bucket); | |
300 | ||
301 | /* We were in userspace, nothing to do */ | |
e78dbc80 | 302 | if (mmcra & cur_cpu_spec->oprofile_mmcra_sipr) |
1da177e4 LT |
303 | return pc; |
304 | ||
305 | #ifdef CONFIG_PPC_RTAS | |
306 | /* Were we in RTAS? */ | |
307 | if (pc >= rtas.base && pc < (rtas.base + rtas.size)) | |
308 | /* function descriptor madness */ | |
309 | return *((unsigned long *)rtas_bucket); | |
310 | #endif | |
311 | ||
312 | /* Were we in our exception vectors or SLB real mode miss handler? */ | |
313 | if (pc < 0x1000000UL) | |
314 | return (unsigned long)__va(pc); | |
315 | ||
316 | /* Not sure where we were */ | |
51fae6de | 317 | if (!is_kernel_addr(pc)) |
1da177e4 LT |
318 | /* function descriptor madness */ |
319 | return *((unsigned long *)kernel_unknown_bucket); | |
320 | ||
15e812ad | 321 | return pc; |
1da177e4 LT |
322 | } |
323 | ||
e78dbc80 | 324 | static int get_kernel(unsigned long pc, unsigned long mmcra) |
1da177e4 LT |
325 | { |
326 | int is_kernel; | |
327 | ||
e78dbc80 | 328 | if (!cur_cpu_spec->oprofile_mmcra_sihv) { |
51fae6de | 329 | is_kernel = is_kernel_addr(pc); |
1da177e4 | 330 | } else { |
e78dbc80 | 331 | is_kernel = ((mmcra & cur_cpu_spec->oprofile_mmcra_sipr) == 0); |
1da177e4 LT |
332 | } |
333 | ||
334 | return is_kernel; | |
335 | } | |
336 | ||
ad5d5292 EM |
337 | static bool pmc_overflow(unsigned long val) |
338 | { | |
339 | if ((int)val < 0) | |
340 | return true; | |
341 | ||
342 | /* | |
343 | * Events on POWER7 can roll back if a speculative event doesn't | |
344 | * eventually complete. Unfortunately in some rare cases they will | |
345 | * raise a performance monitor exception. We need to catch this to | |
346 | * ensure we reset the PMC. In all cases the PMC will be 256 or less | |
347 | * cycles from overflow. | |
348 | * | |
349 | * We only do this if the first pass fails to find any overflowing | |
350 | * PMCs because a user might set a period of less than 256 and we | |
351 | * don't want to mistakenly reset them. | |
352 | */ | |
d3dbeef6 | 353 | if (pvr_version_is(PVR_POWER7) && ((0x80000000 - val) <= 256)) |
ad5d5292 EM |
354 | return true; |
355 | ||
356 | return false; | |
357 | } | |
358 | ||
1da177e4 LT |
359 | static void power4_handle_interrupt(struct pt_regs *regs, |
360 | struct op_counter_config *ctr) | |
361 | { | |
362 | unsigned long pc; | |
363 | int is_kernel; | |
364 | int val; | |
365 | int i; | |
366 | unsigned int mmcr0; | |
e78dbc80 | 367 | unsigned long mmcra; |
adbf115d | 368 | bool siar_valid = false; |
e78dbc80 MN |
369 | |
370 | mmcra = mfspr(SPRN_MMCRA); | |
1da177e4 LT |
371 | |
372 | pc = get_pc(regs); | |
e78dbc80 | 373 | is_kernel = get_kernel(pc, mmcra); |
1da177e4 LT |
374 | |
375 | /* set the PMM bit (see comment below) */ | |
1c539731 | 376 | mtmsr(mfmsr() | MSR_PMM); |
1da177e4 | 377 | |
adbf115d CL |
378 | /* Check that the SIAR valid bit in MMCRA is set to 1. */ |
379 | if ((mmcra & MMCRA_SIAR_VALID_MASK) == MMCRA_SIAR_VALID_MASK) | |
380 | siar_valid = true; | |
381 | ||
a6908cd0 | 382 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
c69b767a | 383 | val = classic_ctr_read(i); |
ad5d5292 | 384 | if (pmc_overflow(val)) { |
1da177e4 | 385 | if (oprofile_running && ctr[i].enabled) { |
adbf115d CL |
386 | /* Power 7+ and newer architectures: |
387 | * If the event is a marked event, then only | |
388 | * save the sample if the SIAR valid bit is | |
389 | * set. If the event is not marked, then | |
390 | * always save the sample. | |
391 | * Note, the Sample enable bit in the MMCRA | |
392 | * register must be set to 1 if the group | |
393 | * contains a marked event. | |
394 | */ | |
395 | if ((siar_valid && | |
396 | (cntr_marked_events & (1 << i))) | |
397 | || !(cntr_marked_events & (1 << i))) | |
398 | oprofile_add_ext_sample(pc, regs, i, | |
399 | is_kernel); | |
400 | ||
c69b767a | 401 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 402 | } else { |
c69b767a | 403 | classic_ctr_write(i, 0); |
1da177e4 LT |
404 | } |
405 | } | |
406 | } | |
407 | ||
408 | mmcr0 = mfspr(SPRN_MMCR0); | |
409 | ||
410 | /* reset the perfmon trigger */ | |
411 | mmcr0 |= MMCR0_PMXE; | |
412 | ||
413 | /* | |
414 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
415 | * all the time | |
416 | */ | |
417 | mmcr0 &= ~MMCR0_PMAO; | |
418 | ||
e78dbc80 MN |
419 | /* Clear the appropriate bits in the MMCRA */ |
420 | mmcra &= ~cur_cpu_spec->oprofile_mmcra_clear; | |
421 | mtspr(SPRN_MMCRA, mmcra); | |
422 | ||
1da177e4 LT |
423 | /* |
424 | * now clear the freeze bit, counting will not start until we | |
425 | * rfid from this exception, because only at that point will | |
426 | * the PMM bit be cleared | |
427 | */ | |
428 | mmcr0 &= ~MMCR0_FC; | |
429 | mtspr(SPRN_MMCR0, mmcr0); | |
430 | } | |
431 | ||
a3e48c10 | 432 | struct op_powerpc_model op_model_power4 = { |
1da177e4 LT |
433 | .reg_setup = power4_reg_setup, |
434 | .cpu_setup = power4_cpu_setup, | |
435 | .start = power4_start, | |
436 | .stop = power4_stop, | |
437 | .handle_interrupt = power4_handle_interrupt, | |
438 | }; |