Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (C) 2004 Anton Blanchard <anton@au.ibm.com>, IBM | |
078f1940 | 3 | * Added mmcra[slot] support: |
4 | * Copyright (C) 2006-2007 Will Schmidt <willschm@us.ibm.com>, IBM | |
1da177e4 LT |
5 | * |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/oprofile.h> | |
1da177e4 | 13 | #include <linux/smp.h> |
57cfb814 | 14 | #include <asm/firmware.h> |
1da177e4 | 15 | #include <asm/ptrace.h> |
1da177e4 LT |
16 | #include <asm/processor.h> |
17 | #include <asm/cputable.h> | |
1da177e4 | 18 | #include <asm/rtas.h> |
dca85932 | 19 | #include <asm/oprofile_impl.h> |
cb09cff3 | 20 | #include <asm/reg.h> |
1da177e4 LT |
21 | |
22 | #define dbg(args...) | |
adbf115d CL |
23 | #define OPROFILE_PM_PMCSEL_MSK 0xffULL |
24 | #define OPROFILE_PM_UNIT_SHIFT 60 | |
25 | #define OPROFILE_PM_UNIT_MSK 0xfULL | |
26 | #define OPROFILE_MAX_PMC_NUM 3 | |
27 | #define OPROFILE_PMSEL_FIELD_WIDTH 8 | |
28 | #define OPROFILE_UNIT_FIELD_WIDTH 4 | |
29 | #define MMCRA_SIAR_VALID_MASK 0x10000000ULL | |
1da177e4 | 30 | |
1da177e4 LT |
31 | static unsigned long reset_value[OP_MAX_COUNTER]; |
32 | ||
1da177e4 | 33 | static int oprofile_running; |
e5fc948b | 34 | static int use_slot_nums; |
1da177e4 LT |
35 | |
36 | /* mmcr values are set in power4_reg_setup, used in power4_cpu_setup */ | |
37 | static u32 mmcr0_val; | |
38 | static u64 mmcr1_val; | |
15e812ad | 39 | static u64 mmcra_val; |
adbf115d CL |
40 | static u32 cntr_marked_events; |
41 | ||
42 | static int power7_marked_instr_event(u64 mmcr1) | |
43 | { | |
44 | u64 psel, unit; | |
45 | int pmc, cntr_marked_events = 0; | |
46 | ||
47 | /* Given the MMCR1 value, look at the field for each counter to | |
48 | * determine if it is a marked event. Code based on the function | |
49 | * power7_marked_instr_event() in file arch/powerpc/perf/power7-pmu.c. | |
50 | */ | |
51 | for (pmc = 0; pmc < 4; pmc++) { | |
52 | psel = mmcr1 & (OPROFILE_PM_PMCSEL_MSK | |
53 | << (OPROFILE_MAX_PMC_NUM - pmc) | |
46ed7a76 | 54 | * OPROFILE_PMSEL_FIELD_WIDTH); |
adbf115d CL |
55 | psel = (psel >> ((OPROFILE_MAX_PMC_NUM - pmc) |
56 | * OPROFILE_PMSEL_FIELD_WIDTH)) & ~1ULL; | |
57 | unit = mmcr1 & (OPROFILE_PM_UNIT_MSK | |
58 | << (OPROFILE_PM_UNIT_SHIFT | |
59 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH ))); | |
60 | unit = unit >> (OPROFILE_PM_UNIT_SHIFT | |
61 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH)); | |
62 | ||
63 | switch (psel >> 4) { | |
64 | case 2: | |
65 | cntr_marked_events |= (pmc == 1 || pmc == 3) << pmc; | |
66 | break; | |
67 | case 3: | |
68 | if (psel == 0x3c) { | |
69 | cntr_marked_events |= (pmc == 0) << pmc; | |
70 | break; | |
71 | } | |
72 | ||
73 | if (psel == 0x3e) { | |
74 | cntr_marked_events |= (pmc != 1) << pmc; | |
75 | break; | |
76 | } | |
77 | ||
78 | cntr_marked_events |= 1 << pmc; | |
79 | break; | |
80 | case 4: | |
81 | case 5: | |
82 | cntr_marked_events |= (unit == 0xd) << pmc; | |
83 | break; | |
84 | case 6: | |
85 | if (psel == 0x64) | |
86 | cntr_marked_events |= (pmc >= 2) << pmc; | |
87 | break; | |
88 | case 8: | |
89 | cntr_marked_events |= (unit == 0xd) << pmc; | |
90 | break; | |
91 | } | |
92 | } | |
93 | return cntr_marked_events; | |
94 | } | |
1da177e4 | 95 | |
1474855d | 96 | static int power4_reg_setup(struct op_counter_config *ctr, |
1da177e4 LT |
97 | struct op_system_config *sys, |
98 | int num_ctrs) | |
99 | { | |
100 | int i; | |
101 | ||
1da177e4 LT |
102 | /* |
103 | * The performance counter event settings are given in the mmcr0, | |
104 | * mmcr1 and mmcra values passed from the user in the | |
105 | * op_system_config structure (sys variable). | |
106 | */ | |
107 | mmcr0_val = sys->mmcr0; | |
108 | mmcr1_val = sys->mmcr1; | |
109 | mmcra_val = sys->mmcra; | |
110 | ||
adbf115d CL |
111 | /* Power 7+ and newer architectures: |
112 | * Determine which counter events in the group (the group of events is | |
113 | * specified by the bit settings in the MMCR1 register) are marked | |
114 | * events for use in the interrupt handler. Do the calculation once | |
115 | * before OProfile starts. Information is used in the interrupt | |
116 | * handler. Starting with Power 7+ we only record the sample for | |
117 | * marked events if the SIAR valid bit is set. For non marked events | |
118 | * the sample is always recorded. | |
119 | */ | |
120 | if (pvr_version_is(PVR_POWER7p)) | |
121 | cntr_marked_events = power7_marked_instr_event(mmcr1_val); | |
122 | else | |
123 | cntr_marked_events = 0; /* For older processors, set the bit map | |
124 | * to zero so the sample will always be | |
125 | * be recorded. | |
126 | */ | |
127 | ||
a6908cd0 | 128 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) |
1da177e4 LT |
129 | reset_value[i] = 0x80000000UL - ctr[i].count; |
130 | ||
131 | /* setup user and kernel profiling */ | |
132 | if (sys->enable_kernel) | |
133 | mmcr0_val &= ~MMCR0_KERNEL_DISABLE; | |
134 | else | |
135 | mmcr0_val |= MMCR0_KERNEL_DISABLE; | |
136 | ||
137 | if (sys->enable_user) | |
138 | mmcr0_val &= ~MMCR0_PROBLEM_DISABLE; | |
139 | else | |
140 | mmcr0_val |= MMCR0_PROBLEM_DISABLE; | |
1474855d | 141 | |
d3dbeef6 ME |
142 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
143 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
144 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX) || | |
145 | pvr_version_is(PVR_POWER5) || pvr_version_is(PVR_POWER5p)) | |
e5fc948b MJ |
146 | use_slot_nums = 1; |
147 | ||
1474855d | 148 | return 0; |
1da177e4 LT |
149 | } |
150 | ||
b950bdd0 | 151 | extern void ppc_enable_pmcs(void); |
1da177e4 | 152 | |
cb09cff3 AB |
153 | /* |
154 | * Older CPUs require the MMCRA sample bit to be always set, but newer | |
155 | * CPUs only want it set for some groups. Eventually we will remove all | |
156 | * knowledge of this bit in the kernel, oprofile userspace should be | |
157 | * setting it when required. | |
158 | * | |
159 | * In order to keep current installations working we force the bit for | |
160 | * those older CPUs. Once everyone has updated their oprofile userspace we | |
161 | * can remove this hack. | |
162 | */ | |
163 | static inline int mmcra_must_set_sample(void) | |
164 | { | |
d3dbeef6 ME |
165 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
166 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
167 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX)) | |
cb09cff3 AB |
168 | return 1; |
169 | ||
170 | return 0; | |
171 | } | |
172 | ||
1474855d | 173 | static int power4_cpu_setup(struct op_counter_config *ctr) |
1da177e4 LT |
174 | { |
175 | unsigned int mmcr0 = mmcr0_val; | |
176 | unsigned long mmcra = mmcra_val; | |
177 | ||
b950bdd0 | 178 | ppc_enable_pmcs(); |
1da177e4 LT |
179 | |
180 | /* set the freeze bit */ | |
181 | mmcr0 |= MMCR0_FC; | |
182 | mtspr(SPRN_MMCR0, mmcr0); | |
183 | ||
184 | mmcr0 |= MMCR0_FCM1|MMCR0_PMXE|MMCR0_FCECE; | |
185 | mmcr0 |= MMCR0_PMC1CE|MMCR0_PMCjCE; | |
186 | mtspr(SPRN_MMCR0, mmcr0); | |
187 | ||
188 | mtspr(SPRN_MMCR1, mmcr1_val); | |
189 | ||
cb09cff3 AB |
190 | if (mmcra_must_set_sample()) |
191 | mmcra |= MMCRA_SAMPLE_ENABLE; | |
1da177e4 LT |
192 | mtspr(SPRN_MMCRA, mmcra); |
193 | ||
194 | dbg("setup on cpu %d, mmcr0 %lx\n", smp_processor_id(), | |
195 | mfspr(SPRN_MMCR0)); | |
196 | dbg("setup on cpu %d, mmcr1 %lx\n", smp_processor_id(), | |
197 | mfspr(SPRN_MMCR1)); | |
198 | dbg("setup on cpu %d, mmcra %lx\n", smp_processor_id(), | |
199 | mfspr(SPRN_MMCRA)); | |
1474855d BN |
200 | |
201 | return 0; | |
1da177e4 LT |
202 | } |
203 | ||
1474855d | 204 | static int power4_start(struct op_counter_config *ctr) |
1da177e4 LT |
205 | { |
206 | int i; | |
207 | unsigned int mmcr0; | |
208 | ||
209 | /* set the PMM bit (see comment below) */ | |
210 | mtmsrd(mfmsr() | MSR_PMM); | |
211 | ||
a6908cd0 | 212 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
1da177e4 | 213 | if (ctr[i].enabled) { |
c69b767a | 214 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 215 | } else { |
c69b767a | 216 | classic_ctr_write(i, 0); |
1da177e4 LT |
217 | } |
218 | } | |
219 | ||
220 | mmcr0 = mfspr(SPRN_MMCR0); | |
221 | ||
222 | /* | |
223 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
224 | * all the time | |
225 | */ | |
226 | mmcr0 &= ~MMCR0_PMAO; | |
227 | ||
228 | /* | |
229 | * now clear the freeze bit, counting will not start until we | |
230 | * rfid from this excetion, because only at that point will | |
231 | * the PMM bit be cleared | |
232 | */ | |
233 | mmcr0 &= ~MMCR0_FC; | |
234 | mtspr(SPRN_MMCR0, mmcr0); | |
235 | ||
236 | oprofile_running = 1; | |
237 | ||
238 | dbg("start on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
1474855d | 239 | return 0; |
1da177e4 LT |
240 | } |
241 | ||
242 | static void power4_stop(void) | |
243 | { | |
244 | unsigned int mmcr0; | |
245 | ||
246 | /* freeze counters */ | |
247 | mmcr0 = mfspr(SPRN_MMCR0); | |
248 | mmcr0 |= MMCR0_FC; | |
249 | mtspr(SPRN_MMCR0, mmcr0); | |
250 | ||
251 | oprofile_running = 0; | |
252 | ||
253 | dbg("stop on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
254 | ||
255 | mb(); | |
256 | } | |
257 | ||
258 | /* Fake functions used by canonicalize_pc */ | |
3ff6eecc | 259 | static void __used hypervisor_bucket(void) |
1da177e4 LT |
260 | { |
261 | } | |
262 | ||
3ff6eecc | 263 | static void __used rtas_bucket(void) |
1da177e4 LT |
264 | { |
265 | } | |
266 | ||
3ff6eecc | 267 | static void __used kernel_unknown_bucket(void) |
1da177e4 LT |
268 | { |
269 | } | |
270 | ||
1da177e4 LT |
271 | /* |
272 | * On GQ and newer the MMCRA stores the HV and PR bits at the time | |
273 | * the SIAR was sampled. We use that to work out if the SIAR was sampled in | |
274 | * the hypervisor, our exception vectors or RTAS. | |
078f1940 | 275 | * If the MMCRA_SAMPLE_ENABLE bit is set, we can use the MMCRA[slot] bits |
276 | * to more accurately identify the address of the sampled instruction. The | |
277 | * mmcra[slot] bits represent the slot number of a sampled instruction | |
278 | * within an instruction group. The slot will contain a value between 1 | |
279 | * and 5 if MMCRA_SAMPLE_ENABLE is set, otherwise 0. | |
1da177e4 LT |
280 | */ |
281 | static unsigned long get_pc(struct pt_regs *regs) | |
282 | { | |
283 | unsigned long pc = mfspr(SPRN_SIAR); | |
284 | unsigned long mmcra; | |
078f1940 | 285 | unsigned long slot; |
1da177e4 | 286 | |
25985edc | 287 | /* Can't do much about it */ |
e78dbc80 | 288 | if (!cur_cpu_spec->oprofile_mmcra_sihv) |
15e812ad | 289 | return pc; |
1da177e4 LT |
290 | |
291 | mmcra = mfspr(SPRN_MMCRA); | |
292 | ||
e5fc948b | 293 | if (use_slot_nums && (mmcra & MMCRA_SAMPLE_ENABLE)) { |
078f1940 | 294 | slot = ((mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT); |
295 | if (slot > 1) | |
296 | pc += 4 * (slot - 1); | |
297 | } | |
298 | ||
1da177e4 | 299 | /* Were we in the hypervisor? */ |
e78dbc80 MN |
300 | if (firmware_has_feature(FW_FEATURE_LPAR) && |
301 | (mmcra & cur_cpu_spec->oprofile_mmcra_sihv)) | |
1da177e4 LT |
302 | /* function descriptor madness */ |
303 | return *((unsigned long *)hypervisor_bucket); | |
304 | ||
305 | /* We were in userspace, nothing to do */ | |
e78dbc80 | 306 | if (mmcra & cur_cpu_spec->oprofile_mmcra_sipr) |
1da177e4 LT |
307 | return pc; |
308 | ||
309 | #ifdef CONFIG_PPC_RTAS | |
310 | /* Were we in RTAS? */ | |
311 | if (pc >= rtas.base && pc < (rtas.base + rtas.size)) | |
312 | /* function descriptor madness */ | |
313 | return *((unsigned long *)rtas_bucket); | |
314 | #endif | |
315 | ||
316 | /* Were we in our exception vectors or SLB real mode miss handler? */ | |
317 | if (pc < 0x1000000UL) | |
318 | return (unsigned long)__va(pc); | |
319 | ||
320 | /* Not sure where we were */ | |
51fae6de | 321 | if (!is_kernel_addr(pc)) |
1da177e4 LT |
322 | /* function descriptor madness */ |
323 | return *((unsigned long *)kernel_unknown_bucket); | |
324 | ||
15e812ad | 325 | return pc; |
1da177e4 LT |
326 | } |
327 | ||
e78dbc80 | 328 | static int get_kernel(unsigned long pc, unsigned long mmcra) |
1da177e4 LT |
329 | { |
330 | int is_kernel; | |
331 | ||
e78dbc80 | 332 | if (!cur_cpu_spec->oprofile_mmcra_sihv) { |
51fae6de | 333 | is_kernel = is_kernel_addr(pc); |
1da177e4 | 334 | } else { |
e78dbc80 | 335 | is_kernel = ((mmcra & cur_cpu_spec->oprofile_mmcra_sipr) == 0); |
1da177e4 LT |
336 | } |
337 | ||
338 | return is_kernel; | |
339 | } | |
340 | ||
ad5d5292 EM |
341 | static bool pmc_overflow(unsigned long val) |
342 | { | |
343 | if ((int)val < 0) | |
344 | return true; | |
345 | ||
346 | /* | |
347 | * Events on POWER7 can roll back if a speculative event doesn't | |
348 | * eventually complete. Unfortunately in some rare cases they will | |
349 | * raise a performance monitor exception. We need to catch this to | |
350 | * ensure we reset the PMC. In all cases the PMC will be 256 or less | |
351 | * cycles from overflow. | |
352 | * | |
353 | * We only do this if the first pass fails to find any overflowing | |
354 | * PMCs because a user might set a period of less than 256 and we | |
355 | * don't want to mistakenly reset them. | |
356 | */ | |
d3dbeef6 | 357 | if (pvr_version_is(PVR_POWER7) && ((0x80000000 - val) <= 256)) |
ad5d5292 EM |
358 | return true; |
359 | ||
360 | return false; | |
361 | } | |
362 | ||
1da177e4 LT |
363 | static void power4_handle_interrupt(struct pt_regs *regs, |
364 | struct op_counter_config *ctr) | |
365 | { | |
366 | unsigned long pc; | |
367 | int is_kernel; | |
368 | int val; | |
369 | int i; | |
370 | unsigned int mmcr0; | |
e78dbc80 | 371 | unsigned long mmcra; |
adbf115d | 372 | bool siar_valid = false; |
e78dbc80 MN |
373 | |
374 | mmcra = mfspr(SPRN_MMCRA); | |
1da177e4 LT |
375 | |
376 | pc = get_pc(regs); | |
e78dbc80 | 377 | is_kernel = get_kernel(pc, mmcra); |
1da177e4 LT |
378 | |
379 | /* set the PMM bit (see comment below) */ | |
380 | mtmsrd(mfmsr() | MSR_PMM); | |
381 | ||
adbf115d CL |
382 | /* Check that the SIAR valid bit in MMCRA is set to 1. */ |
383 | if ((mmcra & MMCRA_SIAR_VALID_MASK) == MMCRA_SIAR_VALID_MASK) | |
384 | siar_valid = true; | |
385 | ||
a6908cd0 | 386 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
c69b767a | 387 | val = classic_ctr_read(i); |
ad5d5292 | 388 | if (pmc_overflow(val)) { |
1da177e4 | 389 | if (oprofile_running && ctr[i].enabled) { |
adbf115d CL |
390 | /* Power 7+ and newer architectures: |
391 | * If the event is a marked event, then only | |
392 | * save the sample if the SIAR valid bit is | |
393 | * set. If the event is not marked, then | |
394 | * always save the sample. | |
395 | * Note, the Sample enable bit in the MMCRA | |
396 | * register must be set to 1 if the group | |
397 | * contains a marked event. | |
398 | */ | |
399 | if ((siar_valid && | |
400 | (cntr_marked_events & (1 << i))) | |
401 | || !(cntr_marked_events & (1 << i))) | |
402 | oprofile_add_ext_sample(pc, regs, i, | |
403 | is_kernel); | |
404 | ||
c69b767a | 405 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 406 | } else { |
c69b767a | 407 | classic_ctr_write(i, 0); |
1da177e4 LT |
408 | } |
409 | } | |
410 | } | |
411 | ||
412 | mmcr0 = mfspr(SPRN_MMCR0); | |
413 | ||
414 | /* reset the perfmon trigger */ | |
415 | mmcr0 |= MMCR0_PMXE; | |
416 | ||
417 | /* | |
418 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
419 | * all the time | |
420 | */ | |
421 | mmcr0 &= ~MMCR0_PMAO; | |
422 | ||
e78dbc80 MN |
423 | /* Clear the appropriate bits in the MMCRA */ |
424 | mmcra &= ~cur_cpu_spec->oprofile_mmcra_clear; | |
425 | mtspr(SPRN_MMCRA, mmcra); | |
426 | ||
1da177e4 LT |
427 | /* |
428 | * now clear the freeze bit, counting will not start until we | |
429 | * rfid from this exception, because only at that point will | |
430 | * the PMM bit be cleared | |
431 | */ | |
432 | mmcr0 &= ~MMCR0_FC; | |
433 | mtspr(SPRN_MMCR0, mmcr0); | |
434 | } | |
435 | ||
a3e48c10 | 436 | struct op_powerpc_model op_model_power4 = { |
1da177e4 LT |
437 | .reg_setup = power4_reg_setup, |
438 | .cpu_setup = power4_cpu_setup, | |
439 | .start = power4_start, | |
440 | .stop = power4_stop, | |
441 | .handle_interrupt = power4_handle_interrupt, | |
442 | }; |