projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
[linux-2.6-block.git] / arch / s390 / oprofile / hwsampler.c
CommitLineData
ec6a3df1
HG		 1/**
2 * arch/s390/oprofile/hwsampler.c
3 *
4 * Copyright IBM Corp. 2010
5 * Author: Heinz Graalfs <graalfs@de.ibm.com>
6 */
7
fcdd65b0 8#include <linux/kernel_stat.h>
ec6a3df1
HG		 9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/smp.h>
12#include <linux/errno.h>
13#include <linux/workqueue.h>
14#include <linux/interrupt.h>
15#include <linux/notifier.h>
16#include <linux/cpu.h>
17#include <linux/semaphore.h>
18#include <linux/oom.h>
19#include <linux/oprofile.h>
20
a0616cde 21#include <asm/facility.h>
b03d541a 22#include <asm/cpu_mf.h>
d7b250e2 23#include <asm/irq.h>
ec6a3df1
HG		 24
25#include "hwsampler.h"
dd3c4670 26#include "op_counter.h"
ec6a3df1
HG		 27
28#define MAX_NUM_SDB 511
29#define MIN_NUM_SDB 1
30
31#define ALERT_REQ_MASK 0x4000000000000000ul
32#define BUFFER_FULL_MASK 0x8000000000000000ul
33
ec6a3df1
HG		 34DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
35
36struct hws_execute_parms {
37 void *buffer;
38 signed int rc;
39};
40
41DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
42EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
43
44static DEFINE_MUTEX(hws_sem);
45static DEFINE_MUTEX(hws_sem_oom);
46
47static unsigned char hws_flush_all;
48static unsigned int hws_oom;
49static struct workqueue_struct *hws_wq;
50
51static unsigned int hws_state;
52enum {
53 HWS_INIT = 1,
54 HWS_DEALLOCATED,
55 HWS_STOPPED,
56 HWS_STARTED,
57 HWS_STOPPING };
58
59/* set to 1 if called by kernel during memory allocation */
60static unsigned char oom_killer_was_active;
61/* size of SDBT and SDB as of allocate API */
62static unsigned long num_sdbt = 100;
63static unsigned long num_sdb = 511;
64/* sampling interval (machine cycles) */
65static unsigned long interval;
66
67static unsigned long min_sampler_rate;
68static unsigned long max_sampler_rate;
69
70static int ssctl(void *buffer)
71{
72 int cc;
73
74 /* set in order to detect a program check */
75 cc = 1;
76
77 asm volatile(
78 "0: .insn s,0xB2870000,0(%1)\n"
79 "1: ipm %0\n"
80 " srl %0,28\n"
81 "2:\n"
82 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
83 : "+d" (cc), "+a" (buffer)
84 : "m" (*((struct hws_ssctl_request_block *)buffer))
85 : "cc", "memory");
86
87 return cc ? -EINVAL : 0 ;
88}
89
90static int qsi(void *buffer)
91{
92 int cc;
93 cc = 1;
94
95 asm volatile(
96 "0: .insn s,0xB2860000,0(%1)\n"
97 "1: lhi %0,0\n"
98 "2:\n"
99 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
100 : "=d" (cc), "+a" (buffer)
101 : "m" (*((struct hws_qsi_info_block *)buffer))
102 : "cc", "memory");
103
104 return cc ? -EINVAL : 0;
105}
106
107static void execute_qsi(void *parms)
108{
109 struct hws_execute_parms *ep = parms;
110
111 ep->rc = qsi(ep->buffer);
112}
113
114static void execute_ssctl(void *parms)
115{
116 struct hws_execute_parms *ep = parms;
117
118 ep->rc = ssctl(ep->buffer);
119}
120
121static int smp_ctl_ssctl_stop(int cpu)
122{
123 int rc;
124 struct hws_execute_parms ep;
125 struct hws_cpu_buffer *cb;
126
127 cb = &per_cpu(sampler_cpu_buffer, cpu);
128
129 cb->ssctl.es = 0;
130 cb->ssctl.cs = 0;
131
132 ep.buffer = &cb->ssctl;
133 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
134 rc = ep.rc;
135 if (rc) {
136 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
137 dump_stack();
138 }
139
140 ep.buffer = &cb->qsi;
141 smp_call_function_single(cpu, execute_qsi, &ep, 1);
142
143 if (cb->qsi.es || cb->qsi.cs) {
144 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
145 dump_stack();
146 }
147
148 return rc;
149}
150
151static int smp_ctl_ssctl_deactivate(int cpu)
152{
153 int rc;
154 struct hws_execute_parms ep;
155 struct hws_cpu_buffer *cb;
156
157 cb = &per_cpu(sampler_cpu_buffer, cpu);
158
159 cb->ssctl.es = 1;
160 cb->ssctl.cs = 0;
161
162 ep.buffer = &cb->ssctl;
163 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
164 rc = ep.rc;
165 if (rc)
166 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
167
168 ep.buffer = &cb->qsi;
169 smp_call_function_single(cpu, execute_qsi, &ep, 1);
170
171 if (cb->qsi.cs)
172 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
173
174 return rc;
175}
176
177static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
178{
179 int rc;
180 struct hws_execute_parms ep;
181 struct hws_cpu_buffer *cb;
182
183 cb = &per_cpu(sampler_cpu_buffer, cpu);
184
185 cb->ssctl.h = 1;
186 cb->ssctl.tear = cb->first_sdbt;
187 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
188 cb->ssctl.interval = interval;
189 cb->ssctl.es = 1;
190 cb->ssctl.cs = 1;
191
192 ep.buffer = &cb->ssctl;
193 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
194 rc = ep.rc;
195 if (rc)
196 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
197
198 ep.buffer = &cb->qsi;
199 smp_call_function_single(cpu, execute_qsi, &ep, 1);
200 if (ep.rc)
201 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
202
203 return rc;
204}
205
206static int smp_ctl_qsi(int cpu)
207{
208 struct hws_execute_parms ep;
209 struct hws_cpu_buffer *cb;
210
211 cb = &per_cpu(sampler_cpu_buffer, cpu);
212
213 ep.buffer = &cb->qsi;
214 smp_call_function_single(cpu, execute_qsi, &ep, 1);
215
216 return ep.rc;
217}
218
219static inline unsigned long *trailer_entry_ptr(unsigned long v)
220{
221 void *ret;
222
223 ret = (void *)v;
224 ret += PAGE_SIZE;
225 ret -= sizeof(struct hws_trailer_entry);
226
227 return (unsigned long *) ret;
228}
229
fde15c3a 230static void hws_ext_handler(struct ext_code ext_code,
b03d541a
JG		 231 unsigned int param32, unsigned long param64)
232{
233 struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
234
235 if (!(param32 & CPU_MF_INT_SF_MASK))
236 return;
237
02456169 238 kstat_cpu(smp_processor_id()).irqs[EXTINT_CMS]++;
b03d541a
JG		 239 atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
240
241 if (hws_wq)
242 queue_work(hws_wq, &cb->worker);
243}
ec6a3df1
HG		 244
245static void worker(struct work_struct *work);
246
247static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
248 unsigned long *dear);
249
250static void init_all_cpu_buffers(void)
251{
252 int cpu;
253 struct hws_cpu_buffer *cb;
254
255 for_each_online_cpu(cpu) {
256 cb = &per_cpu(sampler_cpu_buffer, cpu);
257 memset(cb, 0, sizeof(struct hws_cpu_buffer));
258 }
259}
260
261static int is_link_entry(unsigned long *s)
262{
263 return *s & 0x1ul ? 1 : 0;
264}
265
266static unsigned long *get_next_sdbt(unsigned long *s)
267{
268 return (unsigned long *) (*s & ~0x1ul);
269}
270
271static int prepare_cpu_buffers(void)
272{
273 int cpu;
274 int rc;
275 struct hws_cpu_buffer *cb;
276
277 rc = 0;
278 for_each_online_cpu(cpu) {
279 cb = &per_cpu(sampler_cpu_buffer, cpu);
280 atomic_set(&cb->ext_params, 0);
281 cb->worker_entry = 0;
282 cb->sample_overflow = 0;
283 cb->req_alert = 0;
284 cb->incorrect_sdbt_entry = 0;
285 cb->invalid_entry_address = 0;
286 cb->loss_of_sample_data = 0;
287 cb->sample_auth_change_alert = 0;
288 cb->finish = 0;
289 cb->oom = 0;
290 cb->stop_mode = 0;
291 }
292
293 return rc;
294}
295
296/*
297 * allocate_sdbt() - allocate sampler memory
298 * @cpu: the cpu for which sampler memory is allocated
299 *
300 * A 4K page is allocated for each requested SDBT.
301 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
302 * Set ALERT_REQ mask in each SDBs trailer.
303 * Returns zero if successful, <0 otherwise.
304 */
305static int allocate_sdbt(int cpu)
306{
307 int j, k, rc;
308 unsigned long *sdbt;
309 unsigned long sdb;
310 unsigned long *tail;
311 unsigned long *trailer;
312 struct hws_cpu_buffer *cb;
313
314 cb = &per_cpu(sampler_cpu_buffer, cpu);
315
316 if (cb->first_sdbt)
317 return -EINVAL;
318
319 sdbt = NULL;
320 tail = sdbt;
321
322 for (j = 0; j < num_sdbt; j++) {
323 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
324
325 mutex_lock(&hws_sem_oom);
326 /* OOM killer might have been activated */
327 barrier();
328 if (oom_killer_was_active || !sdbt) {
329 if (sdbt)
330 free_page((unsigned long)sdbt);
331
332 goto allocate_sdbt_error;
333 }
334 if (cb->first_sdbt == 0)
335 cb->first_sdbt = (unsigned long)sdbt;
336
337 /* link current page to tail of chain */
338 if (tail)
339 *tail = (unsigned long)(void *)sdbt + 1;
340
341 mutex_unlock(&hws_sem_oom);
342
343 for (k = 0; k < num_sdb; k++) {
344 /* get and set SDB page */
345 sdb = get_zeroed_page(GFP_KERNEL);
346
347 mutex_lock(&hws_sem_oom);
348 /* OOM killer might have been activated */
349 barrier();
350 if (oom_killer_was_active || !sdb) {
351 if (sdb)
352 free_page(sdb);
353
354 goto allocate_sdbt_error;
355 }
356 *sdbt = sdb;
357 trailer = trailer_entry_ptr(*sdbt);
358 *trailer = ALERT_REQ_MASK;
359 sdbt++;
360 mutex_unlock(&hws_sem_oom);
361 }
362 tail = sdbt;
363 }
364 mutex_lock(&hws_sem_oom);
365 if (oom_killer_was_active)
366 goto allocate_sdbt_error;
367
368 rc = 0;
369 if (tail)
370 *tail = (unsigned long)
371 ((void *)cb->first_sdbt) + 1;
372
373allocate_sdbt_exit:
374 mutex_unlock(&hws_sem_oom);
375 return rc;
376
377allocate_sdbt_error:
378 rc = -ENOMEM;
379 goto allocate_sdbt_exit;
380}
381
382/*
383 * deallocate_sdbt() - deallocate all sampler memory
384 *
385 * For each online CPU all SDBT trees are deallocated.
386 * Returns the number of freed pages.
387 */
388static int deallocate_sdbt(void)
389{
390 int cpu;
391 int counter;
392
393 counter = 0;
394
395 for_each_online_cpu(cpu) {
396 unsigned long start;
397 unsigned long sdbt;
398 unsigned long *curr;
399 struct hws_cpu_buffer *cb;
400
401 cb = &per_cpu(sampler_cpu_buffer, cpu);
402
403 if (!cb->first_sdbt)
404 continue;
405
406 sdbt = cb->first_sdbt;
407 curr = (unsigned long *) sdbt;
408 start = sdbt;
409
410 /* we'll free the SDBT after all SDBs are processed... */
411 while (1) {
412 if (!*curr || !sdbt)
413 break;
414
415 /* watch for link entry reset if found */
416 if (is_link_entry(curr)) {
417 curr = get_next_sdbt(curr);
418 if (sdbt)
419 free_page(sdbt);
420
421 /* we are done if we reach the start */
422 if ((unsigned long) curr == start)
423 break;
424 else
425 sdbt = (unsigned long) curr;
426 } else {
427 /* process SDB pointer */
428 if (*curr) {
429 free_page(*curr);
430 curr++;
431 }
432 }
433 counter++;
434 }
435 cb->first_sdbt = 0;
436 }
437 return counter;
438}
439
440static int start_sampling(int cpu)
441{
442 int rc;
443 struct hws_cpu_buffer *cb;
444
445 cb = &per_cpu(sampler_cpu_buffer, cpu);
446 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
447 if (rc) {
448 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
449 goto start_exit;
450 }
451
452 rc = -EINVAL;
453 if (!cb->qsi.es) {
454 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
455 goto start_exit;
456 }
457
458 if (!cb->qsi.cs) {
459 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
460 goto start_exit;
461 }
462
463 printk(KERN_INFO
464 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
465 cpu, interval);
466
467 rc = 0;
468
469start_exit:
470 return rc;
471}
472
473static int stop_sampling(int cpu)
474{
475 unsigned long v;
476 int rc;
477 struct hws_cpu_buffer *cb;
478
479 rc = smp_ctl_qsi(cpu);
480 WARN_ON(rc);
481
482 cb = &per_cpu(sampler_cpu_buffer, cpu);
483 if (!rc && !cb->qsi.es)
484 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
485
486 rc = smp_ctl_ssctl_stop(cpu);
487 if (rc) {
488 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
489 cpu, rc);
490 goto stop_exit;
491 }
492
493 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
494
495stop_exit:
496 v = cb->req_alert;
497 if (v)
498 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
499 " count=%lu.\n", cpu, v);
500
501 v = cb->loss_of_sample_data;
502 if (v)
503 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
504 " count=%lu.\n", cpu, v);
505
506 v = cb->invalid_entry_address;
507 if (v)
508 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
509 " count=%lu.\n", cpu, v);
510
511 v = cb->incorrect_sdbt_entry;
512 if (v)
513 printk(KERN_ERR
514 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
515 " count=%lu.\n", cpu, v);
516
517 v = cb->sample_auth_change_alert;
518 if (v)
519 printk(KERN_ERR
520 "hwsampler: CPU %d CPUMF Sample authorization change,"
521 " count=%lu.\n", cpu, v);
522
523 return rc;
524}
525
526static int check_hardware_prerequisites(void)
527{
65a94b14 528 if (!test_facility(68))
ec6a3df1 529 return -EOPNOTSUPP;
ec6a3df1
HG		 530 return 0;
531}
532/*
533 * hws_oom_callback() - the OOM callback function
534 *
535 * In case the callback is invoked during memory allocation for the
536 * hw sampler, all obtained memory is deallocated and a flag is set
537 * so main sampler memory allocation can exit with a failure code.
538 * In case the callback is invoked during sampling the hw sampler
539 * is deactivated for all CPUs.
540 */
541static int hws_oom_callback(struct notifier_block *nfb,
542 unsigned long dummy, void *parm)
543{
544 unsigned long *freed;
545 int cpu;
546 struct hws_cpu_buffer *cb;
547
548 freed = parm;
549
550 mutex_lock(&hws_sem_oom);
551
552 if (hws_state == HWS_DEALLOCATED) {
553 /* during memory allocation */
554 if (oom_killer_was_active == 0) {
555 oom_killer_was_active = 1;
556 *freed += deallocate_sdbt();
557 }
558 } else {
559 int i;
560 cpu = get_cpu();
561 cb = &per_cpu(sampler_cpu_buffer, cpu);
562
563 if (!cb->oom) {
564 for_each_online_cpu(i) {
565 smp_ctl_ssctl_deactivate(i);
566 cb->oom = 1;
567 }
568 cb->finish = 1;
569
570 printk(KERN_INFO
571 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
572 cpu);
573 }
574 }
575
576 mutex_unlock(&hws_sem_oom);
577
578 return NOTIFY_OK;
579}
580
581static struct notifier_block hws_oom_notifier = {
582 .notifier_call = hws_oom_callback
583};
584
7bb2e269 585static int hws_cpu_callback(struct notifier_block *nfb,
ec6a3df1
HG		 586 unsigned long action, void *hcpu)
587{
588 /* We do not have sampler space available for all possible CPUs.
589 All CPUs should be online when hw sampling is activated. */
b07c9015 590 return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
ec6a3df1
HG		 591}
592
593static struct notifier_block hws_cpu_notifier = {
594 .notifier_call = hws_cpu_callback
595};
596
597/**
598 * hwsampler_deactivate() - set hardware sampling temporarily inactive
599 * @cpu: specifies the CPU to be set inactive.
600 *
601 * Returns 0 on success, !0 on failure.
602 */
603int hwsampler_deactivate(unsigned int cpu)
604{
605 /*
606 * Deactivate hw sampling temporarily and flush the buffer
607 * by pushing all the pending samples to oprofile buffer.
608 *
609 * This function can be called under one of the following conditions:
610 * Memory unmap, task is exiting.
611 */
612 int rc;
613 struct hws_cpu_buffer *cb;
614
615 rc = 0;
616 mutex_lock(&hws_sem);
617
618 cb = &per_cpu(sampler_cpu_buffer, cpu);
619 if (hws_state == HWS_STARTED) {
620 rc = smp_ctl_qsi(cpu);
621 WARN_ON(rc);
622 if (cb->qsi.cs) {
623 rc = smp_ctl_ssctl_deactivate(cpu);
624 if (rc) {
625 printk(KERN_INFO
626 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
627 cb->finish = 1;
628 hws_state = HWS_STOPPING;
629 } else {
630 hws_flush_all = 1;
631 /* Add work to queue to read pending samples.*/
632 queue_work_on(cpu, hws_wq, &cb->worker);
633 }
634 }
635 }
636 mutex_unlock(&hws_sem);
637
638 if (hws_wq)
639 flush_workqueue(hws_wq);
640
641 return rc;
642}
643
644/**
645 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
646 * @cpu: specifies the CPU to be set active.
647 *
648 * Returns 0 on success, !0 on failure.
649 */
650int hwsampler_activate(unsigned int cpu)
651{
652 /*
653 * Re-activate hw sampling. This should be called in pair with
654 * hwsampler_deactivate().
655 */
656 int rc;
657 struct hws_cpu_buffer *cb;
658
659 rc = 0;
660 mutex_lock(&hws_sem);
661
662 cb = &per_cpu(sampler_cpu_buffer, cpu);
663 if (hws_state == HWS_STARTED) {
664 rc = smp_ctl_qsi(cpu);
665 WARN_ON(rc);
666 if (!cb->qsi.cs) {
667 hws_flush_all = 0;
668 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
669 if (rc) {
670 printk(KERN_ERR
671 "CPU %d, CPUMF activate sampling failed.\n",
672 cpu);
673 }
674 }
675 }
676
677 mutex_unlock(&hws_sem);
678
679 return rc;
680}
681
ec6a3df1
HG		 682static int check_qsi_on_setup(void)
683{
684 int rc;
685 unsigned int cpu;
686 struct hws_cpu_buffer *cb;
687
688 for_each_online_cpu(cpu) {
689 cb = &per_cpu(sampler_cpu_buffer, cpu);
690 rc = smp_ctl_qsi(cpu);
691 WARN_ON(rc);
692 if (rc)
693 return -EOPNOTSUPP;
694
695 if (!cb->qsi.as) {
696 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
697 return -EINVAL;
698 }
699
700 if (cb->qsi.es) {
701 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
702 rc = smp_ctl_ssctl_stop(cpu);
703 if (rc)
704 return -EINVAL;
705
706 printk(KERN_INFO
707 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
708 }
709 }
710 return 0;
711}
712
713static int check_qsi_on_start(void)
714{
715 unsigned int cpu;
716 int rc;
717 struct hws_cpu_buffer *cb;
718
719 for_each_online_cpu(cpu) {
720 cb = &per_cpu(sampler_cpu_buffer, cpu);
721 rc = smp_ctl_qsi(cpu);
722 WARN_ON(rc);
723
724 if (!cb->qsi.as)
725 return -EINVAL;
726
727 if (cb->qsi.es)
728 return -EINVAL;
729
730 if (cb->qsi.cs)
731 return -EINVAL;
732 }
733 return 0;
734}
735
736static void worker_on_start(unsigned int cpu)
737{
738 struct hws_cpu_buffer *cb;
739
740 cb = &per_cpu(sampler_cpu_buffer, cpu);
741 cb->worker_entry = cb->first_sdbt;
742}
743
744static int worker_check_error(unsigned int cpu, int ext_params)
745{
746 int rc;
747 unsigned long *sdbt;
748 struct hws_cpu_buffer *cb;
749
750 rc = 0;
751 cb = &per_cpu(sampler_cpu_buffer, cpu);
752 sdbt = (unsigned long *) cb->worker_entry;
753
754 if (!sdbt || !*sdbt)
755 return -EINVAL;
756
b03d541a 757 if (ext_params & CPU_MF_INT_SF_PRA)
ec6a3df1
HG		 758 cb->req_alert++;
759
b03d541a 760 if (ext_params & CPU_MF_INT_SF_LSDA)
ec6a3df1
HG		 761 cb->loss_of_sample_data++;
762
b03d541a 763 if (ext_params & CPU_MF_INT_SF_IAE) {
ec6a3df1
HG		 764 cb->invalid_entry_address++;
765 rc = -EINVAL;
766 }
767
b03d541a 768 if (ext_params & CPU_MF_INT_SF_ISE) {
ec6a3df1
HG		 769 cb->incorrect_sdbt_entry++;
770 rc = -EINVAL;
771 }
772
b03d541a 773 if (ext_params & CPU_MF_INT_SF_SACA) {
ec6a3df1
HG		 774 cb->sample_auth_change_alert++;
775 rc = -EINVAL;
776 }
777
778 return rc;
779}
780
781static void worker_on_finish(unsigned int cpu)
782{
783 int rc, i;
784 struct hws_cpu_buffer *cb;
785
786 cb = &per_cpu(sampler_cpu_buffer, cpu);
787
788 if (cb->finish) {
789 rc = smp_ctl_qsi(cpu);
790 WARN_ON(rc);
791 if (cb->qsi.es) {
792 printk(KERN_INFO
793 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
794 cpu);
795 rc = smp_ctl_ssctl_stop(cpu);
796 if (rc)
797 printk(KERN_INFO
798 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
799 cpu);
800
801 for_each_online_cpu(i) {
802 if (i == cpu)
803 continue;
804 if (!cb->finish) {
805 cb->finish = 1;
806 queue_work_on(i, hws_wq,
807 &cb->worker);
808 }
809 }
810 }
811 }
812}
813
814static void worker_on_interrupt(unsigned int cpu)
815{
816 unsigned long *sdbt;
817 unsigned char done;
818 struct hws_cpu_buffer *cb;
819
820 cb = &per_cpu(sampler_cpu_buffer, cpu);
821
822 sdbt = (unsigned long *) cb->worker_entry;
823
824 done = 0;
825 /* do not proceed if stop was entered,
826 * forget the buffers not yet processed */
827 while (!done && !cb->stop_mode) {
828 unsigned long *trailer;
829 struct hws_trailer_entry *te;
830 unsigned long *dear = 0;
831
832 trailer = trailer_entry_ptr(*sdbt);
833 /* leave loop if no more work to do */
834 if (!(*trailer & BUFFER_FULL_MASK)) {
835 done = 1;
836 if (!hws_flush_all)
837 continue;
838 }
839
840 te = (struct hws_trailer_entry *)trailer;
841 cb->sample_overflow += te->overflow;
842
843 add_samples_to_oprofile(cpu, sdbt, dear);
844
845 /* reset trailer */
846 xchg((unsigned char *) te, 0x40);
847
848 /* advance to next sdb slot in current sdbt */
849 sdbt++;
850 /* in case link bit is set use address w/o link bit */
851 if (is_link_entry(sdbt))
852 sdbt = get_next_sdbt(sdbt);
853
854 cb->worker_entry = (unsigned long)sdbt;
855 }
856}
857
858static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
859 unsigned long *dear)
860{
861 struct hws_data_entry *sample_data_ptr;
862 unsigned long *trailer;
863
864 trailer = trailer_entry_ptr(*sdbt);
865 if (dear) {
866 if (dear > trailer)
867 return;
868 trailer = dear;
869 }
870
871 sample_data_ptr = (struct hws_data_entry *)(*sdbt);
872
873 while ((unsigned long *)sample_data_ptr < trailer) {
874 struct pt_regs *regs = NULL;
875 struct task_struct *tsk = NULL;
876
877 /*
878 * Check sampling mode, 1 indicates basic (=customer) sampling
879 * mode.
880 */
881 if (sample_data_ptr->def != 1) {
882 /* sample slot is not yet written */
883 break;
884 } else {
885 /* make sure we don't use it twice,
886 * the next time the sampler will set it again */
887 sample_data_ptr->def = 0;
888 }
889
890 /* Get pt_regs. */
891 if (sample_data_ptr->P == 1) {
892 /* userspace sample */
893 unsigned int pid = sample_data_ptr->prim_asn;
dd3c4670
AK		 894 if (!counter_config.user)
895 goto skip_sample;
ec6a3df1
HG		 896 rcu_read_lock();
897 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
898 if (tsk)
899 regs = task_pt_regs(tsk);
900 rcu_read_unlock();
901 } else {
902 /* kernelspace sample */
dd3c4670
AK		 903 if (!counter_config.kernel)
904 goto skip_sample;
ec6a3df1
HG		 905 regs = task_pt_regs(current);
906 }
907
908 mutex_lock(&hws_sem);
909 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
910 !sample_data_ptr->P, tsk);
911 mutex_unlock(&hws_sem);
dd3c4670 912 skip_sample:
ec6a3df1
HG		 913 sample_data_ptr++;
914 }
915}
916
917static void worker(struct work_struct *work)
918{
919 unsigned int cpu;
920 int ext_params;
921 struct hws_cpu_buffer *cb;
922
923 cb = container_of(work, struct hws_cpu_buffer, worker);
924 cpu = smp_processor_id();
925 ext_params = atomic_xchg(&cb->ext_params, 0);
926
927 if (!cb->worker_entry)
928 worker_on_start(cpu);
929
930 if (worker_check_error(cpu, ext_params))
931 return;
932
933 if (!cb->finish)
934 worker_on_interrupt(cpu);
935
936 if (cb->finish)
937 worker_on_finish(cpu);
938}
939
940/**
941 * hwsampler_allocate() - allocate memory for the hardware sampler
942 * @sdbt: number of SDBTs per online CPU (must be > 0)
943 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
944 *
945 * Returns 0 on success, !0 on failure.
946 */
947int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
948{
949 int cpu, rc;
950 mutex_lock(&hws_sem);
951
952 rc = -EINVAL;
953 if (hws_state != HWS_DEALLOCATED)
954 goto allocate_exit;
955
956 if (sdbt < 1)
957 goto allocate_exit;
958
959 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
960 goto allocate_exit;
961
962 num_sdbt = sdbt;
963 num_sdb = sdb;
964
965 oom_killer_was_active = 0;
966 register_oom_notifier(&hws_oom_notifier);
967
968 for_each_online_cpu(cpu) {
969 if (allocate_sdbt(cpu)) {
970 unregister_oom_notifier(&hws_oom_notifier);
971 goto allocate_error;
972 }
973 }
974 unregister_oom_notifier(&hws_oom_notifier);
975 if (oom_killer_was_active)
976 goto allocate_error;
977
978 hws_state = HWS_STOPPED;
979 rc = 0;
980
981allocate_exit:
982 mutex_unlock(&hws_sem);
983 return rc;
984
985allocate_error:
986 rc = -ENOMEM;
987 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
988 goto allocate_exit;
989}
990
991/**
992 * hwsampler_deallocate() - deallocate hardware sampler memory
993 *
994 * Returns 0 on success, !0 on failure.
995 */
e54aafa0 996int hwsampler_deallocate(void)
ec6a3df1
HG		 997{
998 int rc;
999
1000 mutex_lock(&hws_sem);
1001
1002 rc = -EINVAL;
1003 if (hws_state != HWS_STOPPED)
1004 goto deallocate_exit;
1005
b03d541a 1006 measurement_alert_subclass_unregister();
ec6a3df1
HG		 1007 deallocate_sdbt();
1008
1009 hws_state = HWS_DEALLOCATED;
1010 rc = 0;
1011
1012deallocate_exit:
1013 mutex_unlock(&hws_sem);
1014
1015 return rc;
1016}
1017
3d8dcb3c 1018unsigned long hwsampler_query_min_interval(void)
ec6a3df1 1019{
3d8dcb3c 1020 return min_sampler_rate;
ec6a3df1
HG		 1021}
1022
3d8dcb3c 1023unsigned long hwsampler_query_max_interval(void)
ec6a3df1 1024{
3d8dcb3c 1025 return max_sampler_rate;
ec6a3df1
HG		 1026}
1027
1028unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1029{
1030 struct hws_cpu_buffer *cb;
1031
1032 cb = &per_cpu(sampler_cpu_buffer, cpu);
1033
1034 return cb->sample_overflow;
1035}
1036
e54aafa0 1037int hwsampler_setup(void)
ec6a3df1
HG		 1038{
1039 int rc;
1040 int cpu;
1041 struct hws_cpu_buffer *cb;
1042
1043 mutex_lock(&hws_sem);
1044
1045 rc = -EINVAL;
1046 if (hws_state)
1047 goto setup_exit;
1048
1049 hws_state = HWS_INIT;
1050
1051 init_all_cpu_buffers();
1052
1053 rc = check_hardware_prerequisites();
1054 if (rc)
1055 goto setup_exit;
1056
1057 rc = check_qsi_on_setup();
1058 if (rc)
1059 goto setup_exit;
1060
1061 rc = -EINVAL;
1062 hws_wq = create_workqueue("hwsampler");
1063 if (!hws_wq)
1064 goto setup_exit;
1065
1066 register_cpu_notifier(&hws_cpu_notifier);
1067
1068 for_each_online_cpu(cpu) {
1069 cb = &per_cpu(sampler_cpu_buffer, cpu);
1070 INIT_WORK(&cb->worker, worker);
1071 rc = smp_ctl_qsi(cpu);
1072 WARN_ON(rc);
1073 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1074 if (min_sampler_rate) {
1075 printk(KERN_WARNING
1076 "hwsampler: different min sampler rate values.\n");
1077 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1078 min_sampler_rate =
1079 cb->qsi.min_sampl_rate;
1080 } else
1081 min_sampler_rate = cb->qsi.min_sampl_rate;
1082 }
1083 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1084 if (max_sampler_rate) {
1085 printk(KERN_WARNING
1086 "hwsampler: different max sampler rate values.\n");
1087 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1088 max_sampler_rate =
1089 cb->qsi.max_sampl_rate;
1090 } else
1091 max_sampler_rate = cb->qsi.max_sampl_rate;
1092 }
1093 }
1094 register_external_interrupt(0x1407, hws_ext_handler);
1095
1096 hws_state = HWS_DEALLOCATED;
1097 rc = 0;
1098
1099setup_exit:
1100 mutex_unlock(&hws_sem);
1101 return rc;
1102}
1103
e54aafa0 1104int hwsampler_shutdown(void)
ec6a3df1
HG		 1105{
1106 int rc;
1107
1108 mutex_lock(&hws_sem);
1109
1110 rc = -EINVAL;
1111 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1112 mutex_unlock(&hws_sem);
1113
1114 if (hws_wq)
1115 flush_workqueue(hws_wq);
1116
1117 mutex_lock(&hws_sem);
1118
1119 if (hws_state == HWS_STOPPED) {
b03d541a 1120 measurement_alert_subclass_unregister();
ec6a3df1
HG		 1121 deallocate_sdbt();
1122 }
1123 if (hws_wq) {
1124 destroy_workqueue(hws_wq);
1125 hws_wq = NULL;
1126 }
1127
1128 unregister_external_interrupt(0x1407, hws_ext_handler);
1129 hws_state = HWS_INIT;
1130 rc = 0;
1131 }
1132 mutex_unlock(&hws_sem);
1133
1134 unregister_cpu_notifier(&hws_cpu_notifier);
1135
1136 return rc;
1137}
1138
1139/**
1140 * hwsampler_start_all() - start hardware sampling on all online CPUs
1141 * @rate: specifies the used interval when samples are taken
1142 *
1143 * Returns 0 on success, !0 on failure.
1144 */
1145int hwsampler_start_all(unsigned long rate)
1146{
1147 int rc, cpu;
1148
1149 mutex_lock(&hws_sem);
1150
1151 hws_oom = 0;
1152
1153 rc = -EINVAL;
1154 if (hws_state != HWS_STOPPED)
1155 goto start_all_exit;
1156
1157 interval = rate;
1158
1159 /* fail if rate is not valid */
1160 if (interval < min_sampler_rate || interval > max_sampler_rate)
1161 goto start_all_exit;
1162
1163 rc = check_qsi_on_start();
1164 if (rc)
1165 goto start_all_exit;
1166
1167 rc = prepare_cpu_buffers();
1168 if (rc)
1169 goto start_all_exit;
1170
1171 for_each_online_cpu(cpu) {
1172 rc = start_sampling(cpu);
1173 if (rc)
1174 break;
1175 }
1176 if (rc) {
1177 for_each_online_cpu(cpu) {
1178 stop_sampling(cpu);
1179 }
1180 goto start_all_exit;
1181 }
1182 hws_state = HWS_STARTED;
1183 rc = 0;
1184
1185start_all_exit:
1186 mutex_unlock(&hws_sem);
1187
1188 if (rc)
1189 return rc;
1190
1191 register_oom_notifier(&hws_oom_notifier);
1192 hws_oom = 1;
1193 hws_flush_all = 0;
1194 /* now let them in, 1407 CPUMF external interrupts */
b03d541a 1195 measurement_alert_subclass_register();
ec6a3df1
HG		 1196
1197 return 0;
1198}
1199
1200/**
1201 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1202 *
1203 * Returns 0 on success, !0 on failure.
1204 */
e54aafa0 1205int hwsampler_stop_all(void)
ec6a3df1
HG		 1206{
1207 int tmp_rc, rc, cpu;
1208 struct hws_cpu_buffer *cb;
1209
1210 mutex_lock(&hws_sem);
1211
1212 rc = 0;
1213 if (hws_state == HWS_INIT) {
1214 mutex_unlock(&hws_sem);
1215 return rc;
1216 }
1217 hws_state = HWS_STOPPING;
1218 mutex_unlock(&hws_sem);
1219
1220 for_each_online_cpu(cpu) {
1221 cb = &per_cpu(sampler_cpu_buffer, cpu);
1222 cb->stop_mode = 1;
1223 tmp_rc = stop_sampling(cpu);
1224 if (tmp_rc)
1225 rc = tmp_rc;
1226 }
1227
1228 if (hws_wq)
1229 flush_workqueue(hws_wq);
1230
1231 mutex_lock(&hws_sem);
1232 if (hws_oom) {
1233 unregister_oom_notifier(&hws_oom_notifier);
1234 hws_oom = 0;
1235 }
1236 hws_state = HWS_STOPPED;
1237 mutex_unlock(&hws_sem);
1238
1239 return rc;
1240}
Linux 6.x block layer and io_uring tree(s)