projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Add support for named triggers
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
22
23#include "tracing_map.h"
24#include "trace.h"
25
26struct hist_field;
27
28typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
29
30struct hist_field {
31 struct ftrace_event_field *field;
32 unsigned long flags;
33 hist_field_fn_t fn;
34 unsigned int size;
76a3b0c8 35 unsigned int offset;
7ef224d1
TZ		 36};
37
69a0200c
TZ		 38static u64 hist_field_none(struct hist_field *field, void *event)
39{
40 return 0;
41}
42
7ef224d1
TZ		 43static u64 hist_field_counter(struct hist_field *field, void *event)
44{
45 return 1;
46}
47
48static u64 hist_field_string(struct hist_field *hist_field, void *event)
49{
50 char *addr = (char *)(event + hist_field->field->offset);
51
52 return (u64)(unsigned long)addr;
53}
54
79e577cb
NK		 55static u64 hist_field_dynstring(struct hist_field *hist_field, void *event)
56{
57 u32 str_item = *(u32 *)(event + hist_field->field->offset);
58 int str_loc = str_item & 0xffff;
59 char *addr = (char *)(event + str_loc);
60
61 return (u64)(unsigned long)addr;
62}
63
64static u64 hist_field_pstring(struct hist_field *hist_field, void *event)
65{
66 char **addr = (char **)(event + hist_field->field->offset);
67
68 return (u64)(unsigned long)*addr;
69}
70
7ef224d1
TZ		 71#define DEFINE_HIST_FIELD_FN(type) \
72static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
73{ \
74 type *addr = (type *)(event + hist_field->field->offset); \
75 \
79e577cb 76 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 77}
78
79DEFINE_HIST_FIELD_FN(s64);
80DEFINE_HIST_FIELD_FN(u64);
81DEFINE_HIST_FIELD_FN(s32);
82DEFINE_HIST_FIELD_FN(u32);
83DEFINE_HIST_FIELD_FN(s16);
84DEFINE_HIST_FIELD_FN(u16);
85DEFINE_HIST_FIELD_FN(s8);
86DEFINE_HIST_FIELD_FN(u8);
87
88#define for_each_hist_field(i, hist_data) \
89 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
90
91#define for_each_hist_val_field(i, hist_data) \
92 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
93
94#define for_each_hist_key_field(i, hist_data) \
95 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
96
69a0200c
TZ		 97#define HIST_STACKTRACE_DEPTH 16
98#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
99#define HIST_STACKTRACE_SKIP 5
100
7ef224d1 101#define HITCOUNT_IDX 0
69a0200c 102#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 103
104enum hist_field_flags {
c6afad49
TZ		 105 HIST_FIELD_FL_HITCOUNT = 1,
106 HIST_FIELD_FL_KEY = 2,
107 HIST_FIELD_FL_STRING = 4,
108 HIST_FIELD_FL_HEX = 8,
109 HIST_FIELD_FL_SYM = 16,
110 HIST_FIELD_FL_SYM_OFFSET = 32,
6b4827ad 111 HIST_FIELD_FL_EXECNAME = 64,
31696198 112 HIST_FIELD_FL_SYSCALL = 128,
69a0200c 113 HIST_FIELD_FL_STACKTRACE = 256,
7ef224d1
TZ		 114};
115
116struct hist_trigger_attrs {
117 char *keys_str;
f2606835 118 char *vals_str;
e62347d2 119 char *sort_key_str;
83e99914
TZ		 120 bool pause;
121 bool cont;
e86ae9ba 122 bool clear;
7ef224d1
TZ		 123 unsigned int map_bits;
124};
125
126struct hist_trigger_data {
127 struct hist_field *fields[TRACING_MAP_FIELDS_MAX];
128 unsigned int n_vals;
129 unsigned int n_keys;
130 unsigned int n_fields;
131 unsigned int key_size;
132 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
133 unsigned int n_sort_keys;
134 struct trace_event_file *event_file;
135 struct hist_trigger_attrs *attrs;
136 struct tracing_map *map;
137};
138
139static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
140{
141 hist_field_fn_t fn = NULL;
142
143 switch (field_size) {
144 case 8:
145 if (field_is_signed)
146 fn = hist_field_s64;
147 else
148 fn = hist_field_u64;
149 break;
150 case 4:
151 if (field_is_signed)
152 fn = hist_field_s32;
153 else
154 fn = hist_field_u32;
155 break;
156 case 2:
157 if (field_is_signed)
158 fn = hist_field_s16;
159 else
160 fn = hist_field_u16;
161 break;
162 case 1:
163 if (field_is_signed)
164 fn = hist_field_s8;
165 else
166 fn = hist_field_u8;
167 break;
168 }
169
170 return fn;
171}
172
173static int parse_map_size(char *str)
174{
175 unsigned long size, map_bits;
176 int ret;
177
178 strsep(&str, "=");
179 if (!str) {
180 ret = -EINVAL;
181 goto out;
182 }
183
184 ret = kstrtoul(str, 0, &size);
185 if (ret)
186 goto out;
187
188 map_bits = ilog2(roundup_pow_of_two(size));
189 if (map_bits < TRACING_MAP_BITS_MIN ||
190 map_bits > TRACING_MAP_BITS_MAX)
191 ret = -EINVAL;
192 else
193 ret = map_bits;
194 out:
195 return ret;
196}
197
198static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
199{
200 if (!attrs)
201 return;
202
e62347d2 203 kfree(attrs->sort_key_str);
7ef224d1 204 kfree(attrs->keys_str);
f2606835 205 kfree(attrs->vals_str);
7ef224d1
TZ		 206 kfree(attrs);
207}
208
209static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
210{
211 struct hist_trigger_attrs *attrs;
212 int ret = 0;
213
214 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
215 if (!attrs)
216 return ERR_PTR(-ENOMEM);
217
218 while (trigger_str) {
219 char *str = strsep(&trigger_str, ":");
220
221 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
222 (strncmp(str, "keys=", strlen("keys=")) == 0))
223 attrs->keys_str = kstrdup(str, GFP_KERNEL);
f2606835
TZ		 224 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
225 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
226 (strncmp(str, "values=", strlen("values=")) == 0))
227 attrs->vals_str = kstrdup(str, GFP_KERNEL);
e62347d2
TZ		 228 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
229 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
83e99914
TZ		 230 else if (strcmp(str, "pause") == 0)
231 attrs->pause = true;
232 else if ((strcmp(str, "cont") == 0) ||
233 (strcmp(str, "continue") == 0))
234 attrs->cont = true;
e86ae9ba
TZ		 235 else if (strcmp(str, "clear") == 0)
236 attrs->clear = true;
7ef224d1
TZ		 237 else if (strncmp(str, "size=", strlen("size=")) == 0) {
238 int map_bits = parse_map_size(str);
239
240 if (map_bits < 0) {
241 ret = map_bits;
242 goto free;
243 }
244 attrs->map_bits = map_bits;
245 } else {
246 ret = -EINVAL;
247 goto free;
248 }
249 }
250
251 if (!attrs->keys_str) {
252 ret = -EINVAL;
253 goto free;
254 }
255
256 return attrs;
257 free:
258 destroy_hist_trigger_attrs(attrs);
259
260 return ERR_PTR(ret);
261}
262
6b4827ad
TZ		 263static inline void save_comm(char *comm, struct task_struct *task)
264{
265 if (!task->pid) {
266 strcpy(comm, "<idle>");
267 return;
268 }
269
270 if (WARN_ON_ONCE(task->pid < 0)) {
271 strcpy(comm, "<XXX>");
272 return;
273 }
274
275 memcpy(comm, task->comm, TASK_COMM_LEN);
276}
277
278static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
279{
280 kfree((char *)elt->private_data);
281}
282
283static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
284{
285 struct hist_trigger_data *hist_data = elt->map->private_data;
286 struct hist_field *key_field;
287 unsigned int i;
288
289 for_each_hist_key_field(i, hist_data) {
290 key_field = hist_data->fields[i];
291
292 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
293 unsigned int size = TASK_COMM_LEN + 1;
294
295 elt->private_data = kzalloc(size, GFP_KERNEL);
296 if (!elt->private_data)
297 return -ENOMEM;
298 break;
299 }
300 }
301
302 return 0;
303}
304
305static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
306 struct tracing_map_elt *from)
307{
308 char *comm_from = from->private_data;
309 char *comm_to = to->private_data;
310
311 if (comm_from)
312 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
313}
314
315static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
316{
317 char *comm = elt->private_data;
318
319 if (comm)
320 save_comm(comm, current);
321}
322
323static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
324 .elt_alloc = hist_trigger_elt_comm_alloc,
325 .elt_copy = hist_trigger_elt_comm_copy,
326 .elt_free = hist_trigger_elt_comm_free,
327 .elt_init = hist_trigger_elt_comm_init,
328};
329
7ef224d1
TZ		 330static void destroy_hist_field(struct hist_field *hist_field)
331{
332 kfree(hist_field);
333}
334
335static struct hist_field *create_hist_field(struct ftrace_event_field *field,
336 unsigned long flags)
337{
338 struct hist_field *hist_field;
339
340 if (field && is_function_field(field))
341 return NULL;
342
343 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
344 if (!hist_field)
345 return NULL;
346
347 if (flags & HIST_FIELD_FL_HITCOUNT) {
348 hist_field->fn = hist_field_counter;
349 goto out;
350 }
351
69a0200c
TZ		 352 if (flags & HIST_FIELD_FL_STACKTRACE) {
353 hist_field->fn = hist_field_none;
354 goto out;
355 }
356
7ef224d1
TZ		 357 if (is_string_field(field)) {
358 flags |= HIST_FIELD_FL_STRING;
79e577cb
NK		 359
360 if (field->filter_type == FILTER_STATIC_STRING)
361 hist_field->fn = hist_field_string;
362 else if (field->filter_type == FILTER_DYN_STRING)
363 hist_field->fn = hist_field_dynstring;
364 else
365 hist_field->fn = hist_field_pstring;
7ef224d1
TZ		 366 } else {
367 hist_field->fn = select_value_fn(field->size,
368 field->is_signed);
369 if (!hist_field->fn) {
370 destroy_hist_field(hist_field);
371 return NULL;
372 }
373 }
374 out:
375 hist_field->field = field;
376 hist_field->flags = flags;
377
378 return hist_field;
379}
380
381static void destroy_hist_fields(struct hist_trigger_data *hist_data)
382{
383 unsigned int i;
384
385 for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
386 if (hist_data->fields[i]) {
387 destroy_hist_field(hist_data->fields[i]);
388 hist_data->fields[i] = NULL;
389 }
390 }
391}
392
393static int create_hitcount_val(struct hist_trigger_data *hist_data)
394{
395 hist_data->fields[HITCOUNT_IDX] =
396 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
397 if (!hist_data->fields[HITCOUNT_IDX])
398 return -ENOMEM;
399
400 hist_data->n_vals++;
401
402 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
403 return -EINVAL;
404
405 return 0;
406}
407
f2606835
TZ		 408static int create_val_field(struct hist_trigger_data *hist_data,
409 unsigned int val_idx,
410 struct trace_event_file *file,
411 char *field_str)
412{
413 struct ftrace_event_field *field = NULL;
414 unsigned long flags = 0;
0c4a6b46 415 char *field_name;
f2606835
TZ		 416 int ret = 0;
417
418 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
419 return -EINVAL;
0c4a6b46
TZ		 420
421 field_name = strsep(&field_str, ".");
422 if (field_str) {
423 if (strcmp(field_str, "hex") == 0)
424 flags |= HIST_FIELD_FL_HEX;
425 else {
426 ret = -EINVAL;
427 goto out;
428 }
429 }
430
431 field = trace_find_event_field(file->event_call, field_name);
f2606835
TZ		 432 if (!field) {
433 ret = -EINVAL;
434 goto out;
435 }
436
437 hist_data->fields[val_idx] = create_hist_field(field, flags);
438 if (!hist_data->fields[val_idx]) {
439 ret = -ENOMEM;
440 goto out;
441 }
442
443 ++hist_data->n_vals;
444
445 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
446 ret = -EINVAL;
447 out:
448 return ret;
449}
450
7ef224d1
TZ		 451static int create_val_fields(struct hist_trigger_data *hist_data,
452 struct trace_event_file *file)
453{
f2606835
TZ		 454 char *fields_str, *field_str;
455 unsigned int i, j;
7ef224d1
TZ		 456 int ret;
457
458 ret = create_hitcount_val(hist_data);
f2606835
TZ		 459 if (ret)
460 goto out;
7ef224d1 461
f2606835
TZ		 462 fields_str = hist_data->attrs->vals_str;
463 if (!fields_str)
464 goto out;
465
466 strsep(&fields_str, "=");
467 if (!fields_str)
468 goto out;
469
470 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
471 j < TRACING_MAP_VALS_MAX; i++) {
472 field_str = strsep(&fields_str, ",");
473 if (!field_str)
474 break;
475 if (strcmp(field_str, "hitcount") == 0)
476 continue;
477 ret = create_val_field(hist_data, j++, file, field_str);
478 if (ret)
479 goto out;
480 }
481 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
482 ret = -EINVAL;
483 out:
7ef224d1
TZ		 484 return ret;
485}
486
487static int create_key_field(struct hist_trigger_data *hist_data,
488 unsigned int key_idx,
76a3b0c8 489 unsigned int key_offset,
7ef224d1
TZ		 490 struct trace_event_file *file,
491 char *field_str)
492{
493 struct ftrace_event_field *field = NULL;
494 unsigned long flags = 0;
495 unsigned int key_size;
496 int ret = 0;
497
498 if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
499 return -EINVAL;
500
501 flags |= HIST_FIELD_FL_KEY;
502
69a0200c
TZ		 503 if (strcmp(field_str, "stacktrace") == 0) {
504 flags |= HIST_FIELD_FL_STACKTRACE;
505 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
506 } else {
507 char *field_name = strsep(&field_str, ".");
508
509 if (field_str) {
510 if (strcmp(field_str, "hex") == 0)
511 flags |= HIST_FIELD_FL_HEX;
512 else if (strcmp(field_str, "sym") == 0)
513 flags |= HIST_FIELD_FL_SYM;
514 else if (strcmp(field_str, "sym-offset") == 0)
515 flags |= HIST_FIELD_FL_SYM_OFFSET;
516 else if ((strcmp(field_str, "execname") == 0) &&
517 (strcmp(field_name, "common_pid") == 0))
518 flags |= HIST_FIELD_FL_EXECNAME;
519 else if (strcmp(field_str, "syscall") == 0)
520 flags |= HIST_FIELD_FL_SYSCALL;
521 else {
522 ret = -EINVAL;
523 goto out;
524 }
525 }
526
527 field = trace_find_event_field(file->event_call, field_name);
528 if (!field) {
0c4a6b46
TZ		 529 ret = -EINVAL;
530 goto out;
531 }
0c4a6b46 532
6a475cb1
TZ		 533 if (is_string_field(field))
534 key_size = MAX_FILTER_STR_VAL;
79e577cb
NK		 535 else
536 key_size = field->size;
7ef224d1
TZ		 537 }
538
7ef224d1
TZ		 539 hist_data->fields[key_idx] = create_hist_field(field, flags);
540 if (!hist_data->fields[key_idx]) {
541 ret = -ENOMEM;
542 goto out;
543 }
544
545 key_size = ALIGN(key_size, sizeof(u64));
546 hist_data->fields[key_idx]->size = key_size;
76a3b0c8
TZ		 547 hist_data->fields[key_idx]->offset = key_offset;
548 hist_data->key_size += key_size;
7ef224d1
TZ		 549 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
550 ret = -EINVAL;
551 goto out;
552 }
553
554 hist_data->n_keys++;
555
556 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
557 return -EINVAL;
558
559 ret = key_size;
560 out:
561 return ret;
562}
563
564static int create_key_fields(struct hist_trigger_data *hist_data,
565 struct trace_event_file *file)
566{
76a3b0c8 567 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 568 char *fields_str, *field_str;
569 int ret = -EINVAL;
570
571 fields_str = hist_data->attrs->keys_str;
572 if (!fields_str)
573 goto out;
574
575 strsep(&fields_str, "=");
576 if (!fields_str)
577 goto out;
578
76a3b0c8 579 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 580 field_str = strsep(&fields_str, ",");
581 if (!field_str)
582 break;
76a3b0c8
TZ		 583 ret = create_key_field(hist_data, i, key_offset,
584 file, field_str);
7ef224d1
TZ		 585 if (ret < 0)
586 goto out;
76a3b0c8 587 key_offset += ret;
7ef224d1
TZ		 588 }
589 if (fields_str) {
590 ret = -EINVAL;
591 goto out;
592 }
593 ret = 0;
594 out:
595 return ret;
596}
597
598static int create_hist_fields(struct hist_trigger_data *hist_data,
599 struct trace_event_file *file)
600{
601 int ret;
602
603 ret = create_val_fields(hist_data, file);
604 if (ret)
605 goto out;
606
607 ret = create_key_fields(hist_data, file);
608 if (ret)
609 goto out;
610
611 hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
612 out:
613 return ret;
614}
615
e62347d2
TZ		 616static int is_descending(const char *str)
617{
618 if (!str)
619 return 0;
620
621 if (strcmp(str, "descending") == 0)
622 return 1;
623
624 if (strcmp(str, "ascending") == 0)
625 return 0;
626
627 return -EINVAL;
628}
629
7ef224d1
TZ		 630static int create_sort_keys(struct hist_trigger_data *hist_data)
631{
e62347d2
TZ		 632 char *fields_str = hist_data->attrs->sort_key_str;
633 struct ftrace_event_field *field = NULL;
634 struct tracing_map_sort_key *sort_key;
635 int descending, ret = 0;
636 unsigned int i, j;
637
638 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
639
640 if (!fields_str)
641 goto out;
642
643 strsep(&fields_str, "=");
644 if (!fields_str) {
645 ret = -EINVAL;
646 goto out;
647 }
648
649 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
650 char *field_str, *field_name;
651
652 sort_key = &hist_data->sort_keys[i];
653
654 field_str = strsep(&fields_str, ",");
655 if (!field_str) {
656 if (i == 0)
657 ret = -EINVAL;
658 break;
659 }
660
661 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
662 ret = -EINVAL;
663 break;
664 }
7ef224d1 665
e62347d2
TZ		 666 field_name = strsep(&field_str, ".");
667 if (!field_name) {
668 ret = -EINVAL;
669 break;
670 }
671
672 if (strcmp(field_name, "hitcount") == 0) {
673 descending = is_descending(field_str);
674 if (descending < 0) {
675 ret = descending;
676 break;
677 }
678 sort_key->descending = descending;
679 continue;
680 }
7ef224d1 681
e62347d2
TZ		 682 for (j = 1; j < hist_data->n_fields; j++) {
683 field = hist_data->fields[j]->field;
684 if (field && (strcmp(field_name, field->name) == 0)) {
685 sort_key->field_idx = j;
686 descending = is_descending(field_str);
687 if (descending < 0) {
688 ret = descending;
689 goto out;
690 }
691 sort_key->descending = descending;
692 break;
693 }
694 }
695 if (j == hist_data->n_fields) {
696 ret = -EINVAL;
697 break;
698 }
699 }
700 hist_data->n_sort_keys = i;
701 out:
7ef224d1
TZ		 702 return ret;
703}
704
705static void destroy_hist_data(struct hist_trigger_data *hist_data)
706{
707 destroy_hist_trigger_attrs(hist_data->attrs);
708 destroy_hist_fields(hist_data);
709 tracing_map_destroy(hist_data->map);
710 kfree(hist_data);
711}
712
713static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
714{
715 struct tracing_map *map = hist_data->map;
716 struct ftrace_event_field *field;
717 struct hist_field *hist_field;
718 unsigned int i, idx;
719
720 for_each_hist_field(i, hist_data) {
721 hist_field = hist_data->fields[i];
722 if (hist_field->flags & HIST_FIELD_FL_KEY) {
723 tracing_map_cmp_fn_t cmp_fn;
724
725 field = hist_field->field;
726
69a0200c
TZ		 727 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
728 cmp_fn = tracing_map_cmp_none;
729 else if (is_string_field(field))
7ef224d1
TZ		 730 cmp_fn = tracing_map_cmp_string;
731 else
732 cmp_fn = tracing_map_cmp_num(field->size,
733 field->is_signed);
76a3b0c8
TZ		 734 idx = tracing_map_add_key_field(map,
735 hist_field->offset,
736 cmp_fn);
737
7ef224d1
TZ		 738 } else
739 idx = tracing_map_add_sum_field(map);
740
741 if (idx < 0)
742 return idx;
743 }
744
745 return 0;
746}
747
6b4827ad
TZ		 748static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
749{
750 struct hist_field *key_field;
751 unsigned int i;
752
753 for_each_hist_key_field(i, hist_data) {
754 key_field = hist_data->fields[i];
755
756 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
757 return true;
758 }
759
760 return false;
761}
762
7ef224d1
TZ		 763static struct hist_trigger_data *
764create_hist_data(unsigned int map_bits,
765 struct hist_trigger_attrs *attrs,
766 struct trace_event_file *file)
767{
6b4827ad 768 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 769 struct hist_trigger_data *hist_data;
770 int ret = 0;
771
772 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
773 if (!hist_data)
774 return ERR_PTR(-ENOMEM);
775
776 hist_data->attrs = attrs;
777
778 ret = create_hist_fields(hist_data, file);
779 if (ret)
780 goto free;
781
782 ret = create_sort_keys(hist_data);
783 if (ret)
784 goto free;
785
6b4827ad
TZ		 786 if (need_tracing_map_ops(hist_data))
787 map_ops = &hist_trigger_elt_comm_ops;
788
7ef224d1 789 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 790 map_ops, hist_data);
7ef224d1
TZ		 791 if (IS_ERR(hist_data->map)) {
792 ret = PTR_ERR(hist_data->map);
793 hist_data->map = NULL;
794 goto free;
795 }
796
797 ret = create_tracing_map_fields(hist_data);
798 if (ret)
799 goto free;
800
801 ret = tracing_map_init(hist_data->map);
802 if (ret)
803 goto free;
804
805 hist_data->event_file = file;
806 out:
807 return hist_data;
808 free:
809 hist_data->attrs = NULL;
810
811 destroy_hist_data(hist_data);
812
813 hist_data = ERR_PTR(ret);
814
815 goto out;
816}
817
818static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
819 struct tracing_map_elt *elt,
820 void *rec)
821{
822 struct hist_field *hist_field;
823 unsigned int i;
824 u64 hist_val;
825
826 for_each_hist_val_field(i, hist_data) {
827 hist_field = hist_data->fields[i];
828 hist_val = hist_field->fn(hist_field, rec);
829 tracing_map_update_sum(elt, i, hist_val);
830 }
831}
832
6a475cb1
TZ		 833static inline void add_to_key(char *compound_key, void *key,
834 struct hist_field *key_field, void *rec)
835{
836 size_t size = key_field->size;
837
838 if (key_field->flags & HIST_FIELD_FL_STRING) {
839 struct ftrace_event_field *field;
840
841 field = key_field->field;
842 if (field->filter_type == FILTER_DYN_STRING)
843 size = *(u32 *)(rec + field->offset) >> 16;
844 else if (field->filter_type == FILTER_PTR_STRING)
845 size = strlen(key);
846 else if (field->filter_type == FILTER_STATIC_STRING)
847 size = field->size;
848
849 /* ensure NULL-termination */
850 if (size > key_field->size - 1)
851 size = key_field->size - 1;
852 }
853
854 memcpy(compound_key + key_field->offset, key, size);
855}
856
7ef224d1
TZ		 857static void event_hist_trigger(struct event_trigger_data *data, void *rec)
858{
859 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 860 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 861 unsigned long entries[HIST_STACKTRACE_DEPTH];
76a3b0c8 862 char compound_key[HIST_KEY_SIZE_MAX];
69a0200c 863 struct stack_trace stacktrace;
7ef224d1
TZ		 864 struct hist_field *key_field;
865 struct tracing_map_elt *elt;
866 u64 field_contents;
867 void *key = NULL;
868 unsigned int i;
869
6a475cb1 870 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 871
7ef224d1
TZ		 872 for_each_hist_key_field(i, hist_data) {
873 key_field = hist_data->fields[i];
874
69a0200c
TZ		 875 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
876 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
877 stacktrace.entries = entries;
878 stacktrace.nr_entries = 0;
879 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 880
69a0200c
TZ		 881 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
882 save_stack_trace(&stacktrace);
883
884 key = entries;
885 } else {
886 field_contents = key_field->fn(key_field, rec);
6a475cb1 887 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 888 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 889 use_compound_key = true;
890 } else
69a0200c 891 key = (void *)&field_contents;
76a3b0c8 892 }
6a475cb1
TZ		 893
894 if (use_compound_key)
895 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 896 }
897
6a475cb1 898 if (use_compound_key)
76a3b0c8
TZ		 899 key = compound_key;
900
7ef224d1
TZ		 901 elt = tracing_map_insert(hist_data->map, key);
902 if (elt)
903 hist_trigger_elt_update(hist_data, elt, rec);
904}
905
69a0200c
TZ		 906static void hist_trigger_stacktrace_print(struct seq_file *m,
907 unsigned long *stacktrace_entries,
908 unsigned int max_entries)
909{
910 char str[KSYM_SYMBOL_LEN];
911 unsigned int spaces = 8;
912 unsigned int i;
913
914 for (i = 0; i < max_entries; i++) {
915 if (stacktrace_entries[i] == ULONG_MAX)
916 return;
917
918 seq_printf(m, "%*c", 1 + spaces, ' ');
919 sprint_symbol(str, stacktrace_entries[i]);
920 seq_printf(m, "%s\n", str);
921 }
922}
923
7ef224d1
TZ		 924static void
925hist_trigger_entry_print(struct seq_file *m,
926 struct hist_trigger_data *hist_data, void *key,
927 struct tracing_map_elt *elt)
928{
929 struct hist_field *key_field;
c6afad49 930 char str[KSYM_SYMBOL_LEN];
69a0200c 931 bool multiline = false;
7ef224d1
TZ		 932 unsigned int i;
933 u64 uval;
934
935 seq_puts(m, "{ ");
936
937 for_each_hist_key_field(i, hist_data) {
938 key_field = hist_data->fields[i];
939
940 if (i > hist_data->n_vals)
941 seq_puts(m, ", ");
942
0c4a6b46
TZ		 943 if (key_field->flags & HIST_FIELD_FL_HEX) {
944 uval = *(u64 *)(key + key_field->offset);
945 seq_printf(m, "%s: %llx",
946 key_field->field->name, uval);
c6afad49
TZ		 947 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
948 uval = *(u64 *)(key + key_field->offset);
949 sprint_symbol_no_offset(str, uval);
950 seq_printf(m, "%s: [%llx] %-45s",
951 key_field->field->name, uval, str);
952 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
953 uval = *(u64 *)(key + key_field->offset);
954 sprint_symbol(str, uval);
955 seq_printf(m, "%s: [%llx] %-55s",
956 key_field->field->name, uval, str);
6b4827ad
TZ		 957 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
958 char *comm = elt->private_data;
959
960 uval = *(u64 *)(key + key_field->offset);
961 seq_printf(m, "%s: %-16s[%10llu]",
962 key_field->field->name, comm, uval);
31696198
TZ		 963 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
964 const char *syscall_name;
965
966 uval = *(u64 *)(key + key_field->offset);
967 syscall_name = get_syscall_name(uval);
968 if (!syscall_name)
969 syscall_name = "unknown_syscall";
970
971 seq_printf(m, "%s: %-30s[%3llu]",
972 key_field->field->name, syscall_name, uval);
69a0200c
TZ		 973 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
974 seq_puts(m, "stacktrace:\n");
975 hist_trigger_stacktrace_print(m,
976 key + key_field->offset,
977 HIST_STACKTRACE_DEPTH);
978 multiline = true;
0c4a6b46 979 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
7ef224d1 980 seq_printf(m, "%s: %-50s", key_field->field->name,
76a3b0c8 981 (char *)(key + key_field->offset));
7ef224d1 982 } else {
76a3b0c8
TZ		 983 uval = *(u64 *)(key + key_field->offset);
984 seq_printf(m, "%s: %10llu", key_field->field->name,
985 uval);
7ef224d1
TZ		 986 }
987 }
988
69a0200c
TZ		 989 if (!multiline)
990 seq_puts(m, " ");
991
992 seq_puts(m, "}");
7ef224d1
TZ		 993
994 seq_printf(m, " hitcount: %10llu",
995 tracing_map_read_sum(elt, HITCOUNT_IDX));
996
f2606835 997 for (i = 1; i < hist_data->n_vals; i++) {
0c4a6b46
TZ		 998 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
999 seq_printf(m, " %s: %10llx",
1000 hist_data->fields[i]->field->name,
1001 tracing_map_read_sum(elt, i));
1002 } else {
1003 seq_printf(m, " %s: %10llu",
1004 hist_data->fields[i]->field->name,
1005 tracing_map_read_sum(elt, i));
1006 }
f2606835
TZ		 1007 }
1008
7ef224d1
TZ		 1009 seq_puts(m, "\n");
1010}
1011
1012static int print_entries(struct seq_file *m,
1013 struct hist_trigger_data *hist_data)
1014{
1015 struct tracing_map_sort_entry **sort_entries = NULL;
1016 struct tracing_map *map = hist_data->map;
1017 unsigned int i, n_entries;
1018
1019 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1020 hist_data->n_sort_keys,
1021 &sort_entries);
1022 if (n_entries < 0)
1023 return n_entries;
1024
1025 for (i = 0; i < n_entries; i++)
1026 hist_trigger_entry_print(m, hist_data,
1027 sort_entries[i]->key,
1028 sort_entries[i]->elt);
1029
1030 tracing_map_destroy_sort_entries(sort_entries, n_entries);
1031
1032 return n_entries;
1033}
1034
52a7f16d
TZ		 1035static void hist_trigger_show(struct seq_file *m,
1036 struct event_trigger_data *data, int n)
7ef224d1 1037{
7ef224d1
TZ		 1038 struct hist_trigger_data *hist_data;
1039 int n_entries, ret = 0;
1040
52a7f16d
TZ		 1041 if (n > 0)
1042 seq_puts(m, "\n\n");
7ef224d1
TZ		 1043
1044 seq_puts(m, "# event histogram\n#\n# trigger info: ");
1045 data->ops->print(m, data->ops, data);
52a7f16d 1046 seq_puts(m, "#\n\n");
7ef224d1
TZ		 1047
1048 hist_data = data->private_data;
1049 n_entries = print_entries(m, hist_data);
1050 if (n_entries < 0) {
1051 ret = n_entries;
1052 n_entries = 0;
1053 }
1054
1055 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
1056 (u64)atomic64_read(&hist_data->map->hits),
1057 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 1058}
1059
1060static int hist_show(struct seq_file *m, void *v)
1061{
1062 struct event_trigger_data *data;
1063 struct trace_event_file *event_file;
1064 int n = 0, ret = 0;
1065
1066 mutex_lock(&event_mutex);
1067
1068 event_file = event_file_data(m->private);
1069 if (unlikely(!event_file)) {
1070 ret = -ENODEV;
1071 goto out_unlock;
1072 }
1073
1074 list_for_each_entry_rcu(data, &event_file->triggers, list) {
1075 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1076 hist_trigger_show(m, data, n++);
1077 }
1078
7ef224d1
TZ		 1079 out_unlock:
1080 mutex_unlock(&event_mutex);
1081
1082 return ret;
1083}
1084
1085static int event_hist_open(struct inode *inode, struct file *file)
1086{
1087 return single_open(file, hist_show, file);
1088}
1089
1090const struct file_operations event_hist_fops = {
1091 .open = event_hist_open,
1092 .read = seq_read,
1093 .llseek = seq_lseek,
1094 .release = single_release,
1095};
1096
0c4a6b46
TZ		 1097static const char *get_hist_field_flags(struct hist_field *hist_field)
1098{
1099 const char *flags_str = NULL;
1100
1101 if (hist_field->flags & HIST_FIELD_FL_HEX)
1102 flags_str = "hex";
c6afad49
TZ		 1103 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1104 flags_str = "sym";
1105 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1106 flags_str = "sym-offset";
6b4827ad
TZ		 1107 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1108 flags_str = "execname";
31696198
TZ		 1109 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1110 flags_str = "syscall";
0c4a6b46
TZ		 1111
1112 return flags_str;
1113}
1114
7ef224d1
TZ		 1115static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1116{
1117 seq_printf(m, "%s", hist_field->field->name);
0c4a6b46
TZ		 1118 if (hist_field->flags) {
1119 const char *flags_str = get_hist_field_flags(hist_field);
1120
1121 if (flags_str)
1122 seq_printf(m, ".%s", flags_str);
1123 }
7ef224d1
TZ		 1124}
1125
1126static int event_hist_trigger_print(struct seq_file *m,
1127 struct event_trigger_ops *ops,
1128 struct event_trigger_data *data)
1129{
1130 struct hist_trigger_data *hist_data = data->private_data;
1131 struct hist_field *key_field;
1132 unsigned int i;
1133
1134 seq_puts(m, "hist:keys=");
1135
1136 for_each_hist_key_field(i, hist_data) {
1137 key_field = hist_data->fields[i];
1138
1139 if (i > hist_data->n_vals)
1140 seq_puts(m, ",");
1141
69a0200c
TZ		 1142 if (key_field->flags & HIST_FIELD_FL_STACKTRACE)
1143 seq_puts(m, "stacktrace");
1144 else
1145 hist_field_print(m, key_field);
7ef224d1
TZ		 1146 }
1147
1148 seq_puts(m, ":vals=");
f2606835
TZ		 1149
1150 for_each_hist_val_field(i, hist_data) {
1151 if (i == HITCOUNT_IDX)
1152 seq_puts(m, "hitcount");
1153 else {
1154 seq_puts(m, ",");
1155 hist_field_print(m, hist_data->fields[i]);
1156 }
1157 }
7ef224d1
TZ		 1158
1159 seq_puts(m, ":sort=");
e62347d2
TZ		 1160
1161 for (i = 0; i < hist_data->n_sort_keys; i++) {
1162 struct tracing_map_sort_key *sort_key;
1163
1164 sort_key = &hist_data->sort_keys[i];
1165
1166 if (i > 0)
1167 seq_puts(m, ",");
1168
1169 if (sort_key->field_idx == HITCOUNT_IDX)
1170 seq_puts(m, "hitcount");
1171 else {
1172 unsigned int idx = sort_key->field_idx;
1173
1174 if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1175 return -EINVAL;
1176
1177 hist_field_print(m, hist_data->fields[idx]);
1178 }
1179
1180 if (sort_key->descending)
1181 seq_puts(m, ".descending");
1182 }
7ef224d1
TZ		 1183
1184 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1185
1186 if (data->filter_str)
1187 seq_printf(m, " if %s", data->filter_str);
1188
83e99914
TZ		 1189 if (data->paused)
1190 seq_puts(m, " [paused]");
1191 else
1192 seq_puts(m, " [active]");
7ef224d1
TZ		 1193
1194 seq_putc(m, '\n');
1195
1196 return 0;
1197}
1198
1199static void event_hist_trigger_free(struct event_trigger_ops *ops,
1200 struct event_trigger_data *data)
1201{
1202 struct hist_trigger_data *hist_data = data->private_data;
1203
1204 if (WARN_ON_ONCE(data->ref <= 0))
1205 return;
1206
1207 data->ref--;
1208 if (!data->ref) {
1209 trigger_data_free(data);
1210 destroy_hist_data(hist_data);
1211 }
1212}
1213
1214static struct event_trigger_ops event_hist_trigger_ops = {
1215 .func = event_hist_trigger,
1216 .print = event_hist_trigger_print,
1217 .init = event_trigger_init,
1218 .free = event_hist_trigger_free,
1219};
1220
1221static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1222 char *param)
1223{
1224 return &event_hist_trigger_ops;
1225}
1226
e86ae9ba
TZ		 1227static void hist_clear(struct event_trigger_data *data)
1228{
1229 struct hist_trigger_data *hist_data = data->private_data;
1230 bool paused;
1231
1232 paused = data->paused;
1233 data->paused = true;
1234
1235 synchronize_sched();
1236
1237 tracing_map_clear(hist_data->map);
1238
1239 data->paused = paused;
1240}
1241
52a7f16d
TZ		 1242static bool hist_trigger_match(struct event_trigger_data *data,
1243 struct event_trigger_data *data_test)
1244{
1245 struct tracing_map_sort_key *sort_key, *sort_key_test;
1246 struct hist_trigger_data *hist_data, *hist_data_test;
1247 struct hist_field *key_field, *key_field_test;
1248 unsigned int i;
1249
1250 hist_data = data->private_data;
1251 hist_data_test = data_test->private_data;
1252
1253 if (hist_data->n_vals != hist_data_test->n_vals ||
1254 hist_data->n_fields != hist_data_test->n_fields ||
1255 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
1256 return false;
1257
1258 if ((data->filter_str && !data_test->filter_str) ||
1259 (!data->filter_str && data_test->filter_str))
1260 return false;
1261
1262 for_each_hist_field(i, hist_data) {
1263 key_field = hist_data->fields[i];
1264 key_field_test = hist_data_test->fields[i];
1265
1266 if (key_field->flags != key_field_test->flags)
1267 return false;
1268 if (key_field->field != key_field_test->field)
1269 return false;
1270 if (key_field->offset != key_field_test->offset)
1271 return false;
1272 }
1273
1274 for (i = 0; i < hist_data->n_sort_keys; i++) {
1275 sort_key = &hist_data->sort_keys[i];
1276 sort_key_test = &hist_data_test->sort_keys[i];
1277
1278 if (sort_key->field_idx != sort_key_test->field_idx ||
1279 sort_key->descending != sort_key_test->descending)
1280 return false;
1281 }
1282
1283 if (data->filter_str &&
1284 (strcmp(data->filter_str, data_test->filter_str) != 0))
1285 return false;
1286
1287 return true;
1288}
1289
7ef224d1
TZ		 1290static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1291 struct event_trigger_data *data,
1292 struct trace_event_file *file)
1293{
83e99914 1294 struct hist_trigger_data *hist_data = data->private_data;
7ef224d1
TZ		 1295 struct event_trigger_data *test;
1296 int ret = 0;
1297
1298 list_for_each_entry_rcu(test, &file->triggers, list) {
1299 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
52a7f16d
TZ		 1300 if (!hist_trigger_match(data, test))
1301 continue;
83e99914
TZ		 1302 if (hist_data->attrs->pause)
1303 test->paused = true;
1304 else if (hist_data->attrs->cont)
1305 test->paused = false;
e86ae9ba
TZ		 1306 else if (hist_data->attrs->clear)
1307 hist_clear(test);
83e99914
TZ		 1308 else
1309 ret = -EEXIST;
7ef224d1
TZ		 1310 goto out;
1311 }
1312 }
1313
e86ae9ba 1314 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 1315 ret = -ENOENT;
1316 goto out;
1317 }
1318
1319 if (hist_data->attrs->pause)
1320 data->paused = true;
1321
7ef224d1
TZ		 1322 if (data->ops->init) {
1323 ret = data->ops->init(data->ops, data);
1324 if (ret < 0)
1325 goto out;
1326 }
1327
1328 list_add_rcu(&data->list, &file->triggers);
1329 ret++;
1330
1331 update_cond_flag(file);
1332 if (trace_event_trigger_enable_disable(file, 1) < 0) {
1333 list_del_rcu(&data->list);
1334 update_cond_flag(file);
1335 ret--;
1336 }
1337 out:
1338 return ret;
1339}
1340
52a7f16d
TZ		 1341static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
1342 struct event_trigger_data *data,
1343 struct trace_event_file *file)
1344{
1345 struct event_trigger_data *test;
1346 bool unregistered = false;
1347
1348 list_for_each_entry_rcu(test, &file->triggers, list) {
1349 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1350 if (!hist_trigger_match(data, test))
1351 continue;
1352 unregistered = true;
1353 list_del_rcu(&test->list);
1354 trace_event_trigger_enable_disable(file, 0);
1355 update_cond_flag(file);
1356 break;
1357 }
1358 }
1359
1360 if (unregistered && test->ops->free)
1361 test->ops->free(test->ops, test);
1362}
1363
1364static void hist_unreg_all(struct trace_event_file *file)
1365{
1366 struct event_trigger_data *test;
1367
1368 list_for_each_entry_rcu(test, &file->triggers, list) {
1369 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1370 list_del_rcu(&test->list);
1371 trace_event_trigger_enable_disable(file, 0);
1372 update_cond_flag(file);
1373 if (test->ops->free)
1374 test->ops->free(test->ops, test);
1375 }
1376 }
1377}
1378
7ef224d1
TZ		 1379static int event_hist_trigger_func(struct event_command *cmd_ops,
1380 struct trace_event_file *file,
1381 char *glob, char *cmd, char *param)
1382{
1383 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1384 struct event_trigger_data *trigger_data;
1385 struct hist_trigger_attrs *attrs;
1386 struct event_trigger_ops *trigger_ops;
1387 struct hist_trigger_data *hist_data;
1388 char *trigger;
1389 int ret = 0;
1390
1391 if (!param)
1392 return -EINVAL;
1393
1394 /* separate the trigger from the filter (k:v [if filter]) */
1395 trigger = strsep(&param, " \t");
1396 if (!trigger)
1397 return -EINVAL;
1398
1399 attrs = parse_hist_trigger_attrs(trigger);
1400 if (IS_ERR(attrs))
1401 return PTR_ERR(attrs);
1402
1403 if (attrs->map_bits)
1404 hist_trigger_bits = attrs->map_bits;
1405
1406 hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1407 if (IS_ERR(hist_data)) {
1408 destroy_hist_trigger_attrs(attrs);
1409 return PTR_ERR(hist_data);
1410 }
1411
1412 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1413
1414 ret = -ENOMEM;
1415 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1416 if (!trigger_data)
1417 goto out_free;
1418
1419 trigger_data->count = -1;
1420 trigger_data->ops = trigger_ops;
1421 trigger_data->cmd_ops = cmd_ops;
1422
1423 INIT_LIST_HEAD(&trigger_data->list);
1424 RCU_INIT_POINTER(trigger_data->filter, NULL);
1425
1426 trigger_data->private_data = hist_data;
1427
52a7f16d
TZ		 1428 /* if param is non-empty, it's supposed to be a filter */
1429 if (param && cmd_ops->set_filter) {
1430 ret = cmd_ops->set_filter(param, trigger_data, file);
1431 if (ret < 0)
1432 goto out_free;
1433 }
1434
7ef224d1
TZ		 1435 if (glob[0] == '!') {
1436 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1437 ret = 0;
1438 goto out_free;
1439 }
1440
7ef224d1
TZ		 1441 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1442 /*
1443 * The above returns on success the # of triggers registered,
1444 * but if it didn't register any it returns zero. Consider no
1445 * triggers registered a failure too.
1446 */
1447 if (!ret) {
e86ae9ba 1448 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 1449 ret = -ENOENT;
7ef224d1
TZ		 1450 goto out_free;
1451 } else if (ret < 0)
1452 goto out_free;
1453 /* Just return zero, not the number of registered triggers */
1454 ret = 0;
1455 out:
1456 return ret;
1457 out_free:
1458 if (cmd_ops->set_filter)
1459 cmd_ops->set_filter(NULL, trigger_data, NULL);
1460
1461 kfree(trigger_data);
1462
1463 destroy_hist_data(hist_data);
1464 goto out;
1465}
1466
1467static struct event_command trigger_hist_cmd = {
1468 .name = "hist",
1469 .trigger_type = ETT_EVENT_HIST,
1470 .flags = EVENT_CMD_FL_NEEDS_REC,
1471 .func = event_hist_trigger_func,
1472 .reg = hist_register_trigger,
52a7f16d
TZ		 1473 .unreg = hist_unregister_trigger,
1474 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 1475 .get_trigger_ops = event_hist_get_trigger_ops,
1476 .set_filter = set_trigger_filter,
1477};
1478
1479__init int register_trigger_hist_cmd(void)
1480{
1481 int ret;
1482
1483 ret = register_event_command(&trigger_hist_cmd);
1484 WARN_ON(ret < 0);
1485
1486 return ret;
1487}
d0bad49b
TZ		 1488
1489static void
1490hist_enable_trigger(struct event_trigger_data *data, void *rec)
1491{
1492 struct enable_trigger_data *enable_data = data->private_data;
1493 struct event_trigger_data *test;
1494
1495 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
1496 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1497 if (enable_data->enable)
1498 test->paused = false;
1499 else
1500 test->paused = true;
d0bad49b
TZ		 1501 }
1502 }
1503}
1504
1505static void
1506hist_enable_count_trigger(struct event_trigger_data *data, void *rec)
1507{
1508 if (!data->count)
1509 return;
1510
1511 if (data->count != -1)
1512 (data->count)--;
1513
1514 hist_enable_trigger(data, rec);
1515}
1516
1517static struct event_trigger_ops hist_enable_trigger_ops = {
1518 .func = hist_enable_trigger,
1519 .print = event_enable_trigger_print,
1520 .init = event_trigger_init,
1521 .free = event_enable_trigger_free,
1522};
1523
1524static struct event_trigger_ops hist_enable_count_trigger_ops = {
1525 .func = hist_enable_count_trigger,
1526 .print = event_enable_trigger_print,
1527 .init = event_trigger_init,
1528 .free = event_enable_trigger_free,
1529};
1530
1531static struct event_trigger_ops hist_disable_trigger_ops = {
1532 .func = hist_enable_trigger,
1533 .print = event_enable_trigger_print,
1534 .init = event_trigger_init,
1535 .free = event_enable_trigger_free,
1536};
1537
1538static struct event_trigger_ops hist_disable_count_trigger_ops = {
1539 .func = hist_enable_count_trigger,
1540 .print = event_enable_trigger_print,
1541 .init = event_trigger_init,
1542 .free = event_enable_trigger_free,
1543};
1544
1545static struct event_trigger_ops *
1546hist_enable_get_trigger_ops(char *cmd, char *param)
1547{
1548 struct event_trigger_ops *ops;
1549 bool enable;
1550
1551 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
1552
1553 if (enable)
1554 ops = param ? &hist_enable_count_trigger_ops :
1555 &hist_enable_trigger_ops;
1556 else
1557 ops = param ? &hist_disable_count_trigger_ops :
1558 &hist_disable_trigger_ops;
1559
1560 return ops;
1561}
1562
52a7f16d
TZ		 1563static void hist_enable_unreg_all(struct trace_event_file *file)
1564{
1565 struct event_trigger_data *test;
1566
1567 list_for_each_entry_rcu(test, &file->triggers, list) {
1568 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
1569 list_del_rcu(&test->list);
1570 update_cond_flag(file);
1571 trace_event_trigger_enable_disable(file, 0);
1572 if (test->ops->free)
1573 test->ops->free(test->ops, test);
1574 }
1575 }
1576}
1577
d0bad49b
TZ		 1578static struct event_command trigger_hist_enable_cmd = {
1579 .name = ENABLE_HIST_STR,
1580 .trigger_type = ETT_HIST_ENABLE,
1581 .func = event_enable_trigger_func,
1582 .reg = event_enable_register_trigger,
1583 .unreg = event_enable_unregister_trigger,
52a7f16d 1584 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 1585 .get_trigger_ops = hist_enable_get_trigger_ops,
1586 .set_filter = set_trigger_filter,
1587};
1588
1589static struct event_command trigger_hist_disable_cmd = {
1590 .name = DISABLE_HIST_STR,
1591 .trigger_type = ETT_HIST_ENABLE,
1592 .func = event_enable_trigger_func,
1593 .reg = event_enable_register_trigger,
1594 .unreg = event_enable_unregister_trigger,
52a7f16d 1595 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 1596 .get_trigger_ops = hist_enable_get_trigger_ops,
1597 .set_filter = set_trigger_filter,
1598};
1599
1600static __init void unregister_trigger_hist_enable_disable_cmds(void)
1601{
1602 unregister_event_command(&trigger_hist_enable_cmd);
1603 unregister_event_command(&trigger_hist_disable_cmd);
1604}
1605
1606__init int register_trigger_hist_enable_disable_cmds(void)
1607{
1608 int ret;
1609
1610 ret = register_event_command(&trigger_hist_enable_cmd);
1611 if (WARN_ON(ret < 0))
1612 return ret;
1613 ret = register_event_command(&trigger_hist_disable_cmd);
1614 if (WARN_ON(ret < 0))
1615 unregister_trigger_hist_enable_disable_cmds();
1616
1617 return ret;
1618}
Linux 6.x block layer and io_uring tree(s)