Commit | Line | Data |
---|---|---|
c432fefe TR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Performance event support - Processor Activity Instrumentation Extension | |
4 | * Facility | |
5 | * | |
6 | * Copyright IBM Corp. 2022 | |
7 | * Author(s): Thomas Richter <tmricht@linux.ibm.com> | |
8 | */ | |
9 | #define KMSG_COMPONENT "pai_ext" | |
10 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt | |
11 | ||
12 | #include <linux/kernel.h> | |
13 | #include <linux/kernel_stat.h> | |
14 | #include <linux/percpu.h> | |
15 | #include <linux/notifier.h> | |
16 | #include <linux/init.h> | |
17 | #include <linux/export.h> | |
18 | #include <linux/io.h> | |
1e99c242 | 19 | #include <linux/perf_event.h> |
c432fefe | 20 | |
c432fefe TR |
21 | #include <asm/ctl_reg.h> |
22 | #include <asm/pai.h> | |
23 | #include <asm/debug.h> | |
24 | ||
25 | #define PAIE1_CB_SZ 0x200 /* Size of PAIE1 control block */ | |
26 | #define PAIE1_CTRBLOCK_SZ 0x400 /* Size of PAIE1 counter blocks */ | |
27 | ||
28 | static debug_info_t *paiext_dbg; | |
29 | static unsigned int paiext_cnt; /* Extracted with QPACI instruction */ | |
30 | ||
c432fefe TR |
31 | struct pai_userdata { |
32 | u16 num; | |
33 | u64 value; | |
34 | } __packed; | |
35 | ||
36 | /* Create the PAI extension 1 control block area. | |
37 | * The PAI extension control block 1 is pointed to by lowcore | |
38 | * address 0x1508 for each CPU. This control block is 512 bytes in size | |
39 | * and requires a 512 byte boundary alignment. | |
40 | */ | |
41 | struct paiext_cb { /* PAI extension 1 control block */ | |
42 | u64 header; /* Not used */ | |
43 | u64 reserved1; | |
44 | u64 acc; /* Addr to analytics counter control block */ | |
45 | u8 reserved2[488]; | |
46 | } __packed; | |
47 | ||
48 | struct paiext_map { | |
49 | unsigned long *area; /* Area for CPU to store counters */ | |
50 | struct pai_userdata *save; /* Area to store non-zero counters */ | |
4c787963 | 51 | enum paievt_mode mode; /* Type of event */ |
c432fefe | 52 | unsigned int active_events; /* # of PAI Extension users */ |
1f2597cd | 53 | refcount_t refcnt; |
c432fefe TR |
54 | struct perf_event *event; /* Perf event for sampling */ |
55 | struct paiext_cb *paiext_cb; /* PAI extension control block area */ | |
56 | }; | |
57 | ||
58 | struct paiext_mapptr { | |
59 | struct paiext_map *mapptr; | |
60 | }; | |
61 | ||
62 | static struct paiext_root { /* Anchor to per CPU data */ | |
1f2597cd | 63 | refcount_t refcnt; /* Overall active events */ |
c432fefe TR |
64 | struct paiext_mapptr __percpu *mapptr; |
65 | } paiext_root; | |
66 | ||
67 | /* Free per CPU data when the last event is removed. */ | |
68 | static void paiext_root_free(void) | |
69 | { | |
1f2597cd | 70 | if (refcount_dec_and_test(&paiext_root.refcnt)) { |
c432fefe TR |
71 | free_percpu(paiext_root.mapptr); |
72 | paiext_root.mapptr = NULL; | |
73 | } | |
74 | } | |
75 | ||
76 | /* On initialization of first event also allocate per CPU data dynamically. | |
77 | * Start with an array of pointers, the array size is the maximum number of | |
78 | * CPUs possible, which might be larger than the number of CPUs currently | |
79 | * online. | |
80 | */ | |
81 | static int paiext_root_alloc(void) | |
82 | { | |
1f2597cd | 83 | if (!refcount_inc_not_zero(&paiext_root.refcnt)) { |
c432fefe TR |
84 | /* The memory is already zeroed. */ |
85 | paiext_root.mapptr = alloc_percpu(struct paiext_mapptr); | |
86 | if (!paiext_root.mapptr) { | |
cada938a | 87 | /* Returning without refcnt adjustment is ok. The |
c432fefe TR |
88 | * error code is handled by paiext_alloc() which |
89 | * decrements refcnt when an event can not be | |
90 | * created. | |
91 | */ | |
92 | return -ENOMEM; | |
93 | } | |
1f2597cd | 94 | refcount_set(&paiext_root.refcnt, 1); |
c432fefe TR |
95 | } |
96 | return 0; | |
97 | } | |
98 | ||
99 | /* Protects against concurrent increment of sampler and counter member | |
100 | * increments at the same time and prohibits concurrent execution of | |
101 | * counting and sampling events. | |
102 | * Ensures that analytics counter block is deallocated only when the | |
103 | * sampling and counting on that cpu is zero. | |
104 | * For details see paiext_alloc(). | |
105 | */ | |
106 | static DEFINE_MUTEX(paiext_reserve_mutex); | |
107 | ||
108 | /* Free all memory allocated for event counting/sampling setup */ | |
109 | static void paiext_free(struct paiext_mapptr *mp) | |
110 | { | |
111 | kfree(mp->mapptr->area); | |
112 | kfree(mp->mapptr->paiext_cb); | |
113 | kvfree(mp->mapptr->save); | |
114 | kfree(mp->mapptr); | |
115 | mp->mapptr = NULL; | |
116 | } | |
117 | ||
118 | /* Release the PMU if event is the last perf event */ | |
119 | static void paiext_event_destroy(struct perf_event *event) | |
120 | { | |
121 | struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); | |
122 | struct paiext_map *cpump = mp->mapptr; | |
123 | ||
124 | mutex_lock(&paiext_reserve_mutex); | |
125 | cpump->event = NULL; | |
1f2597cd | 126 | if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ |
c432fefe TR |
127 | paiext_free(mp); |
128 | paiext_root_free(); | |
129 | mutex_unlock(&paiext_reserve_mutex); | |
130 | debug_sprintf_event(paiext_dbg, 4, "%s cpu %d mapptr %p\n", __func__, | |
131 | event->cpu, mp->mapptr); | |
132 | ||
133 | } | |
134 | ||
135 | /* Used to avoid races in checking concurrent access of counting and | |
136 | * sampling for pai_extension events. | |
137 | * | |
138 | * Only one instance of event pai_ext/NNPA_ALL/ for sampling is | |
139 | * allowed and when this event is running, no counting event is allowed. | |
140 | * Several counting events are allowed in parallel, but no sampling event | |
141 | * is allowed while one (or more) counting events are running. | |
142 | * | |
143 | * This function is called in process context and it is safe to block. | |
144 | * When the event initialization functions fails, no other call back will | |
145 | * be invoked. | |
146 | * | |
147 | * Allocate the memory for the event. | |
148 | */ | |
149 | static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) | |
150 | { | |
151 | struct paiext_mapptr *mp; | |
152 | struct paiext_map *cpump; | |
153 | int rc; | |
154 | ||
155 | mutex_lock(&paiext_reserve_mutex); | |
156 | ||
157 | rc = paiext_root_alloc(); | |
158 | if (rc) | |
159 | goto unlock; | |
160 | ||
161 | mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); | |
162 | cpump = mp->mapptr; | |
163 | if (!cpump) { /* Paiext_map allocated? */ | |
164 | rc = -ENOMEM; | |
165 | cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); | |
166 | if (!cpump) | |
1f2597cd | 167 | goto undo; |
c432fefe TR |
168 | |
169 | /* Allocate memory for counter area and counter extraction. | |
170 | * These are | |
171 | * - a 512 byte block and requires 512 byte boundary alignment. | |
172 | * - a 1KB byte block and requires 1KB boundary alignment. | |
173 | * Only the first counting event has to allocate the area. | |
174 | * | |
175 | * Note: This works with commit 59bb47985c1d by default. | |
176 | * Backporting this to kernels without this commit might | |
177 | * need adjustment. | |
178 | */ | |
179 | mp->mapptr = cpump; | |
180 | cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL); | |
181 | cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); | |
182 | cpump->save = kvmalloc_array(paiext_cnt + 1, | |
183 | sizeof(struct pai_userdata), | |
184 | GFP_KERNEL); | |
185 | if (!cpump->save || !cpump->area || !cpump->paiext_cb) { | |
186 | paiext_free(mp); | |
1f2597cd | 187 | goto undo; |
c432fefe | 188 | } |
1f2597cd | 189 | refcount_set(&cpump->refcnt, 1); |
c432fefe | 190 | cpump->mode = a->sample_period ? PAI_MODE_SAMPLING |
4c787963 | 191 | : PAI_MODE_COUNTING; |
c432fefe | 192 | } else { |
cada938a | 193 | /* Multiple invocation, check what is active. |
c432fefe TR |
194 | * Supported are multiple counter events or only one sampling |
195 | * event concurrently at any one time. | |
196 | */ | |
197 | if (cpump->mode == PAI_MODE_SAMPLING || | |
4c787963 | 198 | (cpump->mode == PAI_MODE_COUNTING && a->sample_period)) { |
c432fefe | 199 | rc = -EBUSY; |
1f2597cd | 200 | goto undo; |
c432fefe | 201 | } |
1f2597cd | 202 | refcount_inc(&cpump->refcnt); |
c432fefe TR |
203 | } |
204 | ||
205 | rc = 0; | |
206 | cpump->event = event; | |
c432fefe | 207 | |
1f2597cd | 208 | undo: |
c432fefe TR |
209 | if (rc) { |
210 | /* Error in allocation of event, decrement anchor. Since | |
211 | * the event in not created, its destroy() function is never | |
212 | * invoked. Adjust the reference counter for the anchor. | |
213 | */ | |
214 | paiext_root_free(); | |
215 | } | |
1f2597cd | 216 | unlock: |
c432fefe TR |
217 | mutex_unlock(&paiext_reserve_mutex); |
218 | /* If rc is non-zero, no increment of counter/sampler was done. */ | |
219 | return rc; | |
220 | } | |
221 | ||
222 | /* The PAI extension 1 control block supports up to 128 entries. Return | |
223 | * the index within PAIE1_CB given the event number. Also validate event | |
224 | * number. | |
225 | */ | |
226 | static int paiext_event_valid(struct perf_event *event) | |
227 | { | |
228 | u64 cfg = event->attr.config; | |
229 | ||
230 | if (cfg >= PAI_NNPA_BASE && cfg <= PAI_NNPA_BASE + paiext_cnt) { | |
231 | /* Offset NNPA in paiext_cb */ | |
232 | event->hw.config_base = offsetof(struct paiext_cb, acc); | |
233 | return 0; | |
234 | } | |
235 | return -EINVAL; | |
236 | } | |
237 | ||
238 | /* Might be called on different CPU than the one the event is intended for. */ | |
239 | static int paiext_event_init(struct perf_event *event) | |
240 | { | |
241 | struct perf_event_attr *a = &event->attr; | |
242 | int rc; | |
243 | ||
244 | /* PMU pai_ext registered as PERF_TYPE_RAW, check event type */ | |
245 | if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) | |
246 | return -ENOENT; | |
247 | /* PAI extension event must be valid and in supported range */ | |
248 | rc = paiext_event_valid(event); | |
249 | if (rc) | |
250 | return rc; | |
251 | /* Allow only CPU wide operation, no process context for now. */ | |
252 | if (event->hw.target || event->cpu == -1) | |
253 | return -ENOENT; | |
254 | /* Allow only event NNPA_ALL for sampling. */ | |
255 | if (a->sample_period && a->config != PAI_NNPA_BASE) | |
256 | return -EINVAL; | |
257 | /* Prohibit exclude_user event selection */ | |
258 | if (a->exclude_user) | |
259 | return -EINVAL; | |
260 | ||
261 | rc = paiext_alloc(a, event); | |
262 | if (rc) | |
263 | return rc; | |
264 | event->hw.last_tag = 0; | |
265 | event->destroy = paiext_event_destroy; | |
266 | ||
267 | if (a->sample_period) { | |
268 | a->sample_period = 1; | |
269 | a->freq = 0; | |
270 | /* Register for paicrypt_sched_task() to be called */ | |
271 | event->attach_state |= PERF_ATTACH_SCHED_CB; | |
272 | /* Add raw data which are the memory mapped counters */ | |
273 | a->sample_type |= PERF_SAMPLE_RAW; | |
274 | /* Turn off inheritance */ | |
275 | a->inherit = 0; | |
276 | } | |
277 | ||
278 | return 0; | |
279 | } | |
280 | ||
281 | static u64 paiext_getctr(struct paiext_map *cpump, int nr) | |
282 | { | |
283 | return cpump->area[nr]; | |
284 | } | |
285 | ||
286 | /* Read the counter values. Return value from location in buffer. For event | |
287 | * NNPA_ALL sum up all events. | |
288 | */ | |
289 | static u64 paiext_getdata(struct perf_event *event) | |
290 | { | |
291 | struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); | |
292 | struct paiext_map *cpump = mp->mapptr; | |
293 | u64 sum = 0; | |
294 | int i; | |
295 | ||
296 | if (event->attr.config != PAI_NNPA_BASE) | |
297 | return paiext_getctr(cpump, event->attr.config - PAI_NNPA_BASE); | |
298 | ||
299 | for (i = 1; i <= paiext_cnt; i++) | |
300 | sum += paiext_getctr(cpump, i); | |
301 | ||
302 | return sum; | |
303 | } | |
304 | ||
305 | static u64 paiext_getall(struct perf_event *event) | |
306 | { | |
307 | return paiext_getdata(event); | |
308 | } | |
309 | ||
310 | static void paiext_read(struct perf_event *event) | |
311 | { | |
312 | u64 prev, new, delta; | |
313 | ||
314 | prev = local64_read(&event->hw.prev_count); | |
315 | new = paiext_getall(event); | |
316 | local64_set(&event->hw.prev_count, new); | |
317 | delta = new - prev; | |
318 | local64_add(delta, &event->count); | |
319 | } | |
320 | ||
321 | static void paiext_start(struct perf_event *event, int flags) | |
322 | { | |
323 | u64 sum; | |
324 | ||
325 | if (event->hw.last_tag) | |
326 | return; | |
327 | event->hw.last_tag = 1; | |
328 | sum = paiext_getall(event); /* Get current value */ | |
329 | local64_set(&event->hw.prev_count, sum); | |
330 | local64_set(&event->count, 0); | |
331 | } | |
332 | ||
333 | static int paiext_add(struct perf_event *event, int flags) | |
334 | { | |
335 | struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); | |
336 | struct paiext_map *cpump = mp->mapptr; | |
337 | struct paiext_cb *pcb = cpump->paiext_cb; | |
338 | ||
339 | if (++cpump->active_events == 1) { | |
340 | S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); | |
341 | pcb->acc = virt_to_phys(cpump->area) | 0x1; | |
342 | /* Enable CPU instruction lookup for PAIE1 control block */ | |
343 | __ctl_set_bit(0, 49); | |
344 | debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", | |
345 | __func__, S390_lowcore.aicd, pcb->acc); | |
346 | } | |
347 | if (flags & PERF_EF_START && !event->attr.sample_period) { | |
348 | /* Only counting needs initial counter value */ | |
349 | paiext_start(event, PERF_EF_RELOAD); | |
350 | } | |
351 | event->hw.state = 0; | |
352 | if (event->attr.sample_period) { | |
353 | cpump->event = event; | |
354 | perf_sched_cb_inc(event->pmu); | |
355 | } | |
356 | return 0; | |
357 | } | |
358 | ||
359 | static void paiext_stop(struct perf_event *event, int flags) | |
360 | { | |
361 | paiext_read(event); | |
362 | event->hw.state = PERF_HES_STOPPED; | |
363 | } | |
364 | ||
365 | static void paiext_del(struct perf_event *event, int flags) | |
366 | { | |
367 | struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); | |
368 | struct paiext_map *cpump = mp->mapptr; | |
369 | struct paiext_cb *pcb = cpump->paiext_cb; | |
370 | ||
371 | if (event->attr.sample_period) | |
372 | perf_sched_cb_dec(event->pmu); | |
373 | if (!event->attr.sample_period) { | |
374 | /* Only counting needs to read counter */ | |
375 | paiext_stop(event, PERF_EF_UPDATE); | |
376 | } | |
377 | if (--cpump->active_events == 0) { | |
378 | /* Disable CPU instruction lookup for PAIE1 control block */ | |
379 | __ctl_clear_bit(0, 49); | |
380 | pcb->acc = 0; | |
381 | S390_lowcore.aicd = 0; | |
382 | debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", | |
383 | __func__, S390_lowcore.aicd, pcb->acc); | |
384 | } | |
385 | } | |
386 | ||
387 | /* Create raw data and save it in buffer. Returns number of bytes copied. | |
388 | * Saves only positive counter entries of the form | |
389 | * 2 bytes: Number of counter | |
390 | * 8 bytes: Value of counter | |
391 | */ | |
392 | static size_t paiext_copy(struct paiext_map *cpump) | |
393 | { | |
394 | struct pai_userdata *userdata = cpump->save; | |
395 | int i, outidx = 0; | |
396 | ||
397 | for (i = 1; i <= paiext_cnt; i++) { | |
398 | u64 val = paiext_getctr(cpump, i); | |
399 | ||
400 | if (val) { | |
401 | userdata[outidx].num = i; | |
402 | userdata[outidx].value = val; | |
403 | outidx++; | |
404 | } | |
405 | } | |
406 | return outidx * sizeof(*userdata); | |
407 | } | |
408 | ||
409 | /* Write sample when one or more counters values are nonzero. | |
410 | * | |
411 | * Note: The function paiext_sched_task() and paiext_push_sample() are not | |
412 | * invoked after function paiext_del() has been called because of function | |
413 | * perf_sched_cb_dec(). | |
414 | * The function paiext_sched_task() and paiext_push_sample() are only | |
415 | * called when sampling is active. Function perf_sched_cb_inc() | |
416 | * has been invoked to install function paiext_sched_task() as call back | |
417 | * to run at context switch time (see paiext_add()). | |
418 | * | |
419 | * This causes function perf_event_context_sched_out() and | |
420 | * perf_event_context_sched_in() to check whether the PMU has installed an | |
421 | * sched_task() callback. That callback is not active after paiext_del() | |
422 | * returns and has deleted the event on that CPU. | |
423 | */ | |
424 | static int paiext_push_sample(void) | |
425 | { | |
426 | struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); | |
427 | struct paiext_map *cpump = mp->mapptr; | |
428 | struct perf_event *event = cpump->event; | |
429 | struct perf_sample_data data; | |
430 | struct perf_raw_record raw; | |
431 | struct pt_regs regs; | |
432 | size_t rawsize; | |
433 | int overflow; | |
434 | ||
435 | rawsize = paiext_copy(cpump); | |
436 | if (!rawsize) /* No incremented counters */ | |
437 | return 0; | |
438 | ||
439 | /* Setup perf sample */ | |
440 | memset(®s, 0, sizeof(regs)); | |
441 | memset(&raw, 0, sizeof(raw)); | |
442 | memset(&data, 0, sizeof(data)); | |
443 | perf_sample_data_init(&data, 0, event->hw.last_period); | |
444 | if (event->attr.sample_type & PERF_SAMPLE_TID) { | |
445 | data.tid_entry.pid = task_tgid_nr(current); | |
446 | data.tid_entry.tid = task_pid_nr(current); | |
447 | } | |
448 | if (event->attr.sample_type & PERF_SAMPLE_TIME) | |
449 | data.time = event->clock(); | |
450 | if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) | |
451 | data.id = event->id; | |
452 | if (event->attr.sample_type & PERF_SAMPLE_CPU) | |
453 | data.cpu_entry.cpu = smp_processor_id(); | |
454 | if (event->attr.sample_type & PERF_SAMPLE_RAW) { | |
455 | raw.frag.size = rawsize; | |
456 | raw.frag.data = cpump->save; | |
0a9081cf | 457 | perf_sample_save_raw_data(&data, &raw); |
c432fefe TR |
458 | } |
459 | ||
460 | overflow = perf_event_overflow(event, &data, ®s); | |
461 | perf_event_update_userpage(event); | |
462 | /* Clear lowcore area after read */ | |
463 | memset(cpump->area, 0, PAIE1_CTRBLOCK_SZ); | |
464 | return overflow; | |
465 | } | |
466 | ||
467 | /* Called on schedule-in and schedule-out. No access to event structure, | |
468 | * but for sampling only event NNPA_ALL is allowed. | |
469 | */ | |
bd275681 | 470 | static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) |
c432fefe TR |
471 | { |
472 | /* We started with a clean page on event installation. So read out | |
473 | * results on schedule_out and if page was dirty, clear values. | |
474 | */ | |
475 | if (!sched_in) | |
476 | paiext_push_sample(); | |
477 | } | |
478 | ||
479 | /* Attribute definitions for pai extension1 interface. As with other CPU | |
480 | * Measurement Facilities, there is one attribute per mapped counter. | |
481 | * The number of mapped counters may vary per machine generation. Use | |
482 | * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction | |
483 | * to determine the number of mapped counters. The instructions returns | |
484 | * a positive number, which is the highest number of supported counters. | |
485 | * All counters less than this number are also supported, there are no | |
486 | * holes. A returned number of zero means no support for mapped counters. | |
487 | * | |
488 | * The identification of the counter is a unique number. The chosen range | |
489 | * is 0x1800 + offset in mapped kernel page. | |
490 | * All CPU Measurement Facility counters identifiers must be unique and | |
491 | * the numbers from 0 to 496 are already used for the CPU Measurement | |
492 | * Counter facility. Number 0x1000 to 0x103e are used for PAI cryptography | |
493 | * counters. | |
494 | * Numbers 0xb0000, 0xbc000 and 0xbd000 are already | |
495 | * used for the CPU Measurement Sampling facility. | |
496 | */ | |
497 | PMU_FORMAT_ATTR(event, "config:0-63"); | |
498 | ||
499 | static struct attribute *paiext_format_attr[] = { | |
500 | &format_attr_event.attr, | |
501 | NULL, | |
502 | }; | |
503 | ||
504 | static struct attribute_group paiext_events_group = { | |
505 | .name = "events", | |
506 | .attrs = NULL, /* Filled in attr_event_init() */ | |
507 | }; | |
508 | ||
509 | static struct attribute_group paiext_format_group = { | |
510 | .name = "format", | |
511 | .attrs = paiext_format_attr, | |
512 | }; | |
513 | ||
514 | static const struct attribute_group *paiext_attr_groups[] = { | |
515 | &paiext_events_group, | |
516 | &paiext_format_group, | |
517 | NULL, | |
518 | }; | |
519 | ||
520 | /* Performance monitoring unit for mapped counters */ | |
521 | static struct pmu paiext = { | |
522 | .task_ctx_nr = perf_invalid_context, | |
523 | .event_init = paiext_event_init, | |
524 | .add = paiext_add, | |
525 | .del = paiext_del, | |
526 | .start = paiext_start, | |
527 | .stop = paiext_stop, | |
528 | .read = paiext_read, | |
529 | .sched_task = paiext_sched_task, | |
530 | .attr_groups = paiext_attr_groups, | |
531 | }; | |
532 | ||
533 | /* List of symbolic PAI extension 1 NNPA counter names. */ | |
534 | static const char * const paiext_ctrnames[] = { | |
535 | [0] = "NNPA_ALL", | |
536 | [1] = "NNPA_ADD", | |
537 | [2] = "NNPA_SUB", | |
538 | [3] = "NNPA_MUL", | |
539 | [4] = "NNPA_DIV", | |
540 | [5] = "NNPA_MIN", | |
541 | [6] = "NNPA_MAX", | |
542 | [7] = "NNPA_LOG", | |
543 | [8] = "NNPA_EXP", | |
544 | [9] = "NNPA_IBM_RESERVED_9", | |
545 | [10] = "NNPA_RELU", | |
546 | [11] = "NNPA_TANH", | |
547 | [12] = "NNPA_SIGMOID", | |
548 | [13] = "NNPA_SOFTMAX", | |
549 | [14] = "NNPA_BATCHNORM", | |
550 | [15] = "NNPA_MAXPOOL2D", | |
551 | [16] = "NNPA_AVGPOOL2D", | |
552 | [17] = "NNPA_LSTMACT", | |
553 | [18] = "NNPA_GRUACT", | |
554 | [19] = "NNPA_CONVOLUTION", | |
555 | [20] = "NNPA_MATMUL_OP", | |
556 | [21] = "NNPA_MATMUL_OP_BCAST23", | |
557 | [22] = "NNPA_SMALLBATCH", | |
558 | [23] = "NNPA_LARGEDIM", | |
559 | [24] = "NNPA_SMALLTENSOR", | |
560 | [25] = "NNPA_1MFRAME", | |
561 | [26] = "NNPA_2GFRAME", | |
562 | [27] = "NNPA_ACCESSEXCEPT", | |
563 | }; | |
564 | ||
565 | static void __init attr_event_free(struct attribute **attrs, int num) | |
566 | { | |
567 | struct perf_pmu_events_attr *pa; | |
568 | struct device_attribute *dap; | |
569 | int i; | |
570 | ||
571 | for (i = 0; i < num; i++) { | |
572 | dap = container_of(attrs[i], struct device_attribute, attr); | |
573 | pa = container_of(dap, struct perf_pmu_events_attr, attr); | |
574 | kfree(pa); | |
575 | } | |
576 | kfree(attrs); | |
577 | } | |
578 | ||
579 | static int __init attr_event_init_one(struct attribute **attrs, int num) | |
580 | { | |
581 | struct perf_pmu_events_attr *pa; | |
582 | ||
583 | pa = kzalloc(sizeof(*pa), GFP_KERNEL); | |
584 | if (!pa) | |
585 | return -ENOMEM; | |
586 | ||
587 | sysfs_attr_init(&pa->attr.attr); | |
588 | pa->id = PAI_NNPA_BASE + num; | |
589 | pa->attr.attr.name = paiext_ctrnames[num]; | |
590 | pa->attr.attr.mode = 0444; | |
591 | pa->attr.show = cpumf_events_sysfs_show; | |
592 | pa->attr.store = NULL; | |
593 | attrs[num] = &pa->attr.attr; | |
594 | return 0; | |
595 | } | |
596 | ||
597 | /* Create PMU sysfs event attributes on the fly. */ | |
598 | static int __init attr_event_init(void) | |
599 | { | |
600 | struct attribute **attrs; | |
601 | int ret, i; | |
602 | ||
603 | attrs = kmalloc_array(ARRAY_SIZE(paiext_ctrnames) + 1, sizeof(*attrs), | |
604 | GFP_KERNEL); | |
605 | if (!attrs) | |
606 | return -ENOMEM; | |
607 | for (i = 0; i < ARRAY_SIZE(paiext_ctrnames); i++) { | |
608 | ret = attr_event_init_one(attrs, i); | |
609 | if (ret) { | |
610 | attr_event_free(attrs, i - 1); | |
611 | return ret; | |
612 | } | |
613 | } | |
614 | attrs[i] = NULL; | |
615 | paiext_events_group.attrs = attrs; | |
616 | return 0; | |
617 | } | |
618 | ||
619 | static int __init paiext_init(void) | |
620 | { | |
621 | struct qpaci_info_block ib; | |
622 | int rc = -ENOMEM; | |
623 | ||
624 | if (!test_facility(197)) | |
625 | return 0; | |
626 | ||
627 | qpaci(&ib); | |
628 | paiext_cnt = ib.num_nnpa; | |
629 | if (paiext_cnt >= PAI_NNPA_MAXCTR) | |
630 | paiext_cnt = PAI_NNPA_MAXCTR; | |
631 | if (!paiext_cnt) | |
632 | return 0; | |
633 | ||
634 | rc = attr_event_init(); | |
635 | if (rc) { | |
636 | pr_err("Creation of PMU " KMSG_COMPONENT " /sysfs failed\n"); | |
637 | return rc; | |
638 | } | |
639 | ||
640 | /* Setup s390dbf facility */ | |
641 | paiext_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); | |
642 | if (!paiext_dbg) { | |
643 | pr_err("Registration of s390dbf " KMSG_COMPONENT " failed\n"); | |
644 | rc = -ENOMEM; | |
645 | goto out_init; | |
646 | } | |
647 | debug_register_view(paiext_dbg, &debug_sprintf_view); | |
648 | ||
649 | rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1); | |
650 | if (rc) { | |
651 | pr_err("Registration of " KMSG_COMPONENT " PMU failed with " | |
652 | "rc=%i\n", rc); | |
653 | goto out_pmu; | |
654 | } | |
655 | ||
656 | return 0; | |
657 | ||
658 | out_pmu: | |
659 | debug_unregister_view(paiext_dbg, &debug_sprintf_view); | |
660 | debug_unregister(paiext_dbg); | |
661 | out_init: | |
662 | attr_event_free(paiext_events_group.attrs, | |
663 | ARRAY_SIZE(paiext_ctrnames) + 1); | |
664 | return rc; | |
665 | } | |
666 | ||
667 | device_initcall(paiext_init); |