projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
kselftests/ftrace: Add hist trigger testcases
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
22
23#include "tracing_map.h"
24#include "trace.h"
25
26struct hist_field;
27
28typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
29
30struct hist_field {
31 struct ftrace_event_field *field;
32 unsigned long flags;
33 hist_field_fn_t fn;
34 unsigned int size;
76a3b0c8 35 unsigned int offset;
7ef224d1
TZ		 36};
37
69a0200c
TZ		 38static u64 hist_field_none(struct hist_field *field, void *event)
39{
40 return 0;
41}
42
7ef224d1
TZ		 43static u64 hist_field_counter(struct hist_field *field, void *event)
44{
45 return 1;
46}
47
48static u64 hist_field_string(struct hist_field *hist_field, void *event)
49{
50 char *addr = (char *)(event + hist_field->field->offset);
51
52 return (u64)(unsigned long)addr;
53}
54
79e577cb
NK		 55static u64 hist_field_dynstring(struct hist_field *hist_field, void *event)
56{
57 u32 str_item = *(u32 *)(event + hist_field->field->offset);
58 int str_loc = str_item & 0xffff;
59 char *addr = (char *)(event + str_loc);
60
61 return (u64)(unsigned long)addr;
62}
63
64static u64 hist_field_pstring(struct hist_field *hist_field, void *event)
65{
66 char **addr = (char **)(event + hist_field->field->offset);
67
68 return (u64)(unsigned long)*addr;
69}
70
7ef224d1
TZ		 71#define DEFINE_HIST_FIELD_FN(type) \
72static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
73{ \
74 type *addr = (type *)(event + hist_field->field->offset); \
75 \
79e577cb 76 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 77}
78
79DEFINE_HIST_FIELD_FN(s64);
80DEFINE_HIST_FIELD_FN(u64);
81DEFINE_HIST_FIELD_FN(s32);
82DEFINE_HIST_FIELD_FN(u32);
83DEFINE_HIST_FIELD_FN(s16);
84DEFINE_HIST_FIELD_FN(u16);
85DEFINE_HIST_FIELD_FN(s8);
86DEFINE_HIST_FIELD_FN(u8);
87
88#define for_each_hist_field(i, hist_data) \
89 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
90
91#define for_each_hist_val_field(i, hist_data) \
92 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
93
94#define for_each_hist_key_field(i, hist_data) \
95 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
96
69a0200c
TZ		 97#define HIST_STACKTRACE_DEPTH 16
98#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
99#define HIST_STACKTRACE_SKIP 5
100
7ef224d1 101#define HITCOUNT_IDX 0
69a0200c 102#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 103
104enum hist_field_flags {
c6afad49
TZ		 105 HIST_FIELD_FL_HITCOUNT = 1,
106 HIST_FIELD_FL_KEY = 2,
107 HIST_FIELD_FL_STRING = 4,
108 HIST_FIELD_FL_HEX = 8,
109 HIST_FIELD_FL_SYM = 16,
110 HIST_FIELD_FL_SYM_OFFSET = 32,
6b4827ad 111 HIST_FIELD_FL_EXECNAME = 64,
31696198 112 HIST_FIELD_FL_SYSCALL = 128,
69a0200c 113 HIST_FIELD_FL_STACKTRACE = 256,
7ef224d1
TZ		 114};
115
116struct hist_trigger_attrs {
117 char *keys_str;
f2606835 118 char *vals_str;
e62347d2 119 char *sort_key_str;
5463bfda 120 char *name;
83e99914
TZ		 121 bool pause;
122 bool cont;
e86ae9ba 123 bool clear;
7ef224d1
TZ		 124 unsigned int map_bits;
125};
126
127struct hist_trigger_data {
128 struct hist_field *fields[TRACING_MAP_FIELDS_MAX];
129 unsigned int n_vals;
130 unsigned int n_keys;
131 unsigned int n_fields;
132 unsigned int key_size;
133 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
134 unsigned int n_sort_keys;
135 struct trace_event_file *event_file;
136 struct hist_trigger_attrs *attrs;
137 struct tracing_map *map;
138};
139
140static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
141{
142 hist_field_fn_t fn = NULL;
143
144 switch (field_size) {
145 case 8:
146 if (field_is_signed)
147 fn = hist_field_s64;
148 else
149 fn = hist_field_u64;
150 break;
151 case 4:
152 if (field_is_signed)
153 fn = hist_field_s32;
154 else
155 fn = hist_field_u32;
156 break;
157 case 2:
158 if (field_is_signed)
159 fn = hist_field_s16;
160 else
161 fn = hist_field_u16;
162 break;
163 case 1:
164 if (field_is_signed)
165 fn = hist_field_s8;
166 else
167 fn = hist_field_u8;
168 break;
169 }
170
171 return fn;
172}
173
174static int parse_map_size(char *str)
175{
176 unsigned long size, map_bits;
177 int ret;
178
179 strsep(&str, "=");
180 if (!str) {
181 ret = -EINVAL;
182 goto out;
183 }
184
185 ret = kstrtoul(str, 0, &size);
186 if (ret)
187 goto out;
188
189 map_bits = ilog2(roundup_pow_of_two(size));
190 if (map_bits < TRACING_MAP_BITS_MIN ||
191 map_bits > TRACING_MAP_BITS_MAX)
192 ret = -EINVAL;
193 else
194 ret = map_bits;
195 out:
196 return ret;
197}
198
199static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
200{
201 if (!attrs)
202 return;
203
5463bfda 204 kfree(attrs->name);
e62347d2 205 kfree(attrs->sort_key_str);
7ef224d1 206 kfree(attrs->keys_str);
f2606835 207 kfree(attrs->vals_str);
7ef224d1
TZ		 208 kfree(attrs);
209}
210
211static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
212{
213 struct hist_trigger_attrs *attrs;
214 int ret = 0;
215
216 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
217 if (!attrs)
218 return ERR_PTR(-ENOMEM);
219
220 while (trigger_str) {
221 char *str = strsep(&trigger_str, ":");
222
223 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
224 (strncmp(str, "keys=", strlen("keys=")) == 0))
225 attrs->keys_str = kstrdup(str, GFP_KERNEL);
f2606835
TZ		 226 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
227 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
228 (strncmp(str, "values=", strlen("values=")) == 0))
229 attrs->vals_str = kstrdup(str, GFP_KERNEL);
e62347d2
TZ		 230 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
231 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
5463bfda
TZ		 232 else if (strncmp(str, "name=", strlen("name=")) == 0)
233 attrs->name = kstrdup(str, GFP_KERNEL);
83e99914
TZ		 234 else if (strcmp(str, "pause") == 0)
235 attrs->pause = true;
236 else if ((strcmp(str, "cont") == 0) ||
237 (strcmp(str, "continue") == 0))
238 attrs->cont = true;
e86ae9ba
TZ		 239 else if (strcmp(str, "clear") == 0)
240 attrs->clear = true;
7ef224d1
TZ		 241 else if (strncmp(str, "size=", strlen("size=")) == 0) {
242 int map_bits = parse_map_size(str);
243
244 if (map_bits < 0) {
245 ret = map_bits;
246 goto free;
247 }
248 attrs->map_bits = map_bits;
249 } else {
250 ret = -EINVAL;
251 goto free;
252 }
253 }
254
255 if (!attrs->keys_str) {
256 ret = -EINVAL;
257 goto free;
258 }
259
260 return attrs;
261 free:
262 destroy_hist_trigger_attrs(attrs);
263
264 return ERR_PTR(ret);
265}
266
6b4827ad
TZ		 267static inline void save_comm(char *comm, struct task_struct *task)
268{
269 if (!task->pid) {
270 strcpy(comm, "<idle>");
271 return;
272 }
273
274 if (WARN_ON_ONCE(task->pid < 0)) {
275 strcpy(comm, "<XXX>");
276 return;
277 }
278
279 memcpy(comm, task->comm, TASK_COMM_LEN);
280}
281
282static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
283{
284 kfree((char *)elt->private_data);
285}
286
287static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
288{
289 struct hist_trigger_data *hist_data = elt->map->private_data;
290 struct hist_field *key_field;
291 unsigned int i;
292
293 for_each_hist_key_field(i, hist_data) {
294 key_field = hist_data->fields[i];
295
296 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
297 unsigned int size = TASK_COMM_LEN + 1;
298
299 elt->private_data = kzalloc(size, GFP_KERNEL);
300 if (!elt->private_data)
301 return -ENOMEM;
302 break;
303 }
304 }
305
306 return 0;
307}
308
309static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
310 struct tracing_map_elt *from)
311{
312 char *comm_from = from->private_data;
313 char *comm_to = to->private_data;
314
315 if (comm_from)
316 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
317}
318
319static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
320{
321 char *comm = elt->private_data;
322
323 if (comm)
324 save_comm(comm, current);
325}
326
327static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
328 .elt_alloc = hist_trigger_elt_comm_alloc,
329 .elt_copy = hist_trigger_elt_comm_copy,
330 .elt_free = hist_trigger_elt_comm_free,
331 .elt_init = hist_trigger_elt_comm_init,
332};
333
7ef224d1
TZ		 334static void destroy_hist_field(struct hist_field *hist_field)
335{
336 kfree(hist_field);
337}
338
339static struct hist_field *create_hist_field(struct ftrace_event_field *field,
340 unsigned long flags)
341{
342 struct hist_field *hist_field;
343
344 if (field && is_function_field(field))
345 return NULL;
346
347 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
348 if (!hist_field)
349 return NULL;
350
351 if (flags & HIST_FIELD_FL_HITCOUNT) {
352 hist_field->fn = hist_field_counter;
353 goto out;
354 }
355
69a0200c
TZ		 356 if (flags & HIST_FIELD_FL_STACKTRACE) {
357 hist_field->fn = hist_field_none;
358 goto out;
359 }
360
7ef224d1
TZ		 361 if (is_string_field(field)) {
362 flags |= HIST_FIELD_FL_STRING;
79e577cb
NK		 363
364 if (field->filter_type == FILTER_STATIC_STRING)
365 hist_field->fn = hist_field_string;
366 else if (field->filter_type == FILTER_DYN_STRING)
367 hist_field->fn = hist_field_dynstring;
368 else
369 hist_field->fn = hist_field_pstring;
7ef224d1
TZ		 370 } else {
371 hist_field->fn = select_value_fn(field->size,
372 field->is_signed);
373 if (!hist_field->fn) {
374 destroy_hist_field(hist_field);
375 return NULL;
376 }
377 }
378 out:
379 hist_field->field = field;
380 hist_field->flags = flags;
381
382 return hist_field;
383}
384
385static void destroy_hist_fields(struct hist_trigger_data *hist_data)
386{
387 unsigned int i;
388
389 for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
390 if (hist_data->fields[i]) {
391 destroy_hist_field(hist_data->fields[i]);
392 hist_data->fields[i] = NULL;
393 }
394 }
395}
396
397static int create_hitcount_val(struct hist_trigger_data *hist_data)
398{
399 hist_data->fields[HITCOUNT_IDX] =
400 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
401 if (!hist_data->fields[HITCOUNT_IDX])
402 return -ENOMEM;
403
404 hist_data->n_vals++;
405
406 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
407 return -EINVAL;
408
409 return 0;
410}
411
f2606835
TZ		 412static int create_val_field(struct hist_trigger_data *hist_data,
413 unsigned int val_idx,
414 struct trace_event_file *file,
415 char *field_str)
416{
417 struct ftrace_event_field *field = NULL;
418 unsigned long flags = 0;
0c4a6b46 419 char *field_name;
f2606835
TZ		 420 int ret = 0;
421
422 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
423 return -EINVAL;
0c4a6b46
TZ		 424
425 field_name = strsep(&field_str, ".");
426 if (field_str) {
427 if (strcmp(field_str, "hex") == 0)
428 flags |= HIST_FIELD_FL_HEX;
429 else {
430 ret = -EINVAL;
431 goto out;
432 }
433 }
434
435 field = trace_find_event_field(file->event_call, field_name);
f2606835
TZ		 436 if (!field) {
437 ret = -EINVAL;
438 goto out;
439 }
440
441 hist_data->fields[val_idx] = create_hist_field(field, flags);
442 if (!hist_data->fields[val_idx]) {
443 ret = -ENOMEM;
444 goto out;
445 }
446
447 ++hist_data->n_vals;
448
449 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
450 ret = -EINVAL;
451 out:
452 return ret;
453}
454
7ef224d1
TZ		 455static int create_val_fields(struct hist_trigger_data *hist_data,
456 struct trace_event_file *file)
457{
f2606835
TZ		 458 char *fields_str, *field_str;
459 unsigned int i, j;
7ef224d1
TZ		 460 int ret;
461
462 ret = create_hitcount_val(hist_data);
f2606835
TZ		 463 if (ret)
464 goto out;
7ef224d1 465
f2606835
TZ		 466 fields_str = hist_data->attrs->vals_str;
467 if (!fields_str)
468 goto out;
469
470 strsep(&fields_str, "=");
471 if (!fields_str)
472 goto out;
473
474 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
475 j < TRACING_MAP_VALS_MAX; i++) {
476 field_str = strsep(&fields_str, ",");
477 if (!field_str)
478 break;
479 if (strcmp(field_str, "hitcount") == 0)
480 continue;
481 ret = create_val_field(hist_data, j++, file, field_str);
482 if (ret)
483 goto out;
484 }
485 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
486 ret = -EINVAL;
487 out:
7ef224d1
TZ		 488 return ret;
489}
490
491static int create_key_field(struct hist_trigger_data *hist_data,
492 unsigned int key_idx,
76a3b0c8 493 unsigned int key_offset,
7ef224d1
TZ		 494 struct trace_event_file *file,
495 char *field_str)
496{
497 struct ftrace_event_field *field = NULL;
498 unsigned long flags = 0;
499 unsigned int key_size;
500 int ret = 0;
501
502 if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
503 return -EINVAL;
504
505 flags |= HIST_FIELD_FL_KEY;
506
69a0200c
TZ		 507 if (strcmp(field_str, "stacktrace") == 0) {
508 flags |= HIST_FIELD_FL_STACKTRACE;
509 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
510 } else {
511 char *field_name = strsep(&field_str, ".");
512
513 if (field_str) {
514 if (strcmp(field_str, "hex") == 0)
515 flags |= HIST_FIELD_FL_HEX;
516 else if (strcmp(field_str, "sym") == 0)
517 flags |= HIST_FIELD_FL_SYM;
518 else if (strcmp(field_str, "sym-offset") == 0)
519 flags |= HIST_FIELD_FL_SYM_OFFSET;
520 else if ((strcmp(field_str, "execname") == 0) &&
521 (strcmp(field_name, "common_pid") == 0))
522 flags |= HIST_FIELD_FL_EXECNAME;
523 else if (strcmp(field_str, "syscall") == 0)
524 flags |= HIST_FIELD_FL_SYSCALL;
525 else {
526 ret = -EINVAL;
527 goto out;
528 }
529 }
530
531 field = trace_find_event_field(file->event_call, field_name);
532 if (!field) {
0c4a6b46
TZ		 533 ret = -EINVAL;
534 goto out;
535 }
0c4a6b46 536
6a475cb1
TZ		 537 if (is_string_field(field))
538 key_size = MAX_FILTER_STR_VAL;
79e577cb
NK		 539 else
540 key_size = field->size;
7ef224d1
TZ		 541 }
542
7ef224d1
TZ		 543 hist_data->fields[key_idx] = create_hist_field(field, flags);
544 if (!hist_data->fields[key_idx]) {
545 ret = -ENOMEM;
546 goto out;
547 }
548
549 key_size = ALIGN(key_size, sizeof(u64));
550 hist_data->fields[key_idx]->size = key_size;
76a3b0c8
TZ		 551 hist_data->fields[key_idx]->offset = key_offset;
552 hist_data->key_size += key_size;
7ef224d1
TZ		 553 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
554 ret = -EINVAL;
555 goto out;
556 }
557
558 hist_data->n_keys++;
559
560 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
561 return -EINVAL;
562
563 ret = key_size;
564 out:
565 return ret;
566}
567
568static int create_key_fields(struct hist_trigger_data *hist_data,
569 struct trace_event_file *file)
570{
76a3b0c8 571 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 572 char *fields_str, *field_str;
573 int ret = -EINVAL;
574
575 fields_str = hist_data->attrs->keys_str;
576 if (!fields_str)
577 goto out;
578
579 strsep(&fields_str, "=");
580 if (!fields_str)
581 goto out;
582
76a3b0c8 583 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 584 field_str = strsep(&fields_str, ",");
585 if (!field_str)
586 break;
76a3b0c8
TZ		 587 ret = create_key_field(hist_data, i, key_offset,
588 file, field_str);
7ef224d1
TZ		 589 if (ret < 0)
590 goto out;
76a3b0c8 591 key_offset += ret;
7ef224d1
TZ		 592 }
593 if (fields_str) {
594 ret = -EINVAL;
595 goto out;
596 }
597 ret = 0;
598 out:
599 return ret;
600}
601
602static int create_hist_fields(struct hist_trigger_data *hist_data,
603 struct trace_event_file *file)
604{
605 int ret;
606
607 ret = create_val_fields(hist_data, file);
608 if (ret)
609 goto out;
610
611 ret = create_key_fields(hist_data, file);
612 if (ret)
613 goto out;
614
615 hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
616 out:
617 return ret;
618}
619
e62347d2
TZ		 620static int is_descending(const char *str)
621{
622 if (!str)
623 return 0;
624
625 if (strcmp(str, "descending") == 0)
626 return 1;
627
628 if (strcmp(str, "ascending") == 0)
629 return 0;
630
631 return -EINVAL;
632}
633
7ef224d1
TZ		 634static int create_sort_keys(struct hist_trigger_data *hist_data)
635{
e62347d2
TZ		 636 char *fields_str = hist_data->attrs->sort_key_str;
637 struct ftrace_event_field *field = NULL;
638 struct tracing_map_sort_key *sort_key;
639 int descending, ret = 0;
640 unsigned int i, j;
641
642 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
643
644 if (!fields_str)
645 goto out;
646
647 strsep(&fields_str, "=");
648 if (!fields_str) {
649 ret = -EINVAL;
650 goto out;
651 }
652
653 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
654 char *field_str, *field_name;
655
656 sort_key = &hist_data->sort_keys[i];
657
658 field_str = strsep(&fields_str, ",");
659 if (!field_str) {
660 if (i == 0)
661 ret = -EINVAL;
662 break;
663 }
664
665 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
666 ret = -EINVAL;
667 break;
668 }
7ef224d1 669
e62347d2
TZ		 670 field_name = strsep(&field_str, ".");
671 if (!field_name) {
672 ret = -EINVAL;
673 break;
674 }
675
676 if (strcmp(field_name, "hitcount") == 0) {
677 descending = is_descending(field_str);
678 if (descending < 0) {
679 ret = descending;
680 break;
681 }
682 sort_key->descending = descending;
683 continue;
684 }
7ef224d1 685
e62347d2
TZ		 686 for (j = 1; j < hist_data->n_fields; j++) {
687 field = hist_data->fields[j]->field;
688 if (field && (strcmp(field_name, field->name) == 0)) {
689 sort_key->field_idx = j;
690 descending = is_descending(field_str);
691 if (descending < 0) {
692 ret = descending;
693 goto out;
694 }
695 sort_key->descending = descending;
696 break;
697 }
698 }
699 if (j == hist_data->n_fields) {
700 ret = -EINVAL;
701 break;
702 }
703 }
704 hist_data->n_sort_keys = i;
705 out:
7ef224d1
TZ		 706 return ret;
707}
708
709static void destroy_hist_data(struct hist_trigger_data *hist_data)
710{
711 destroy_hist_trigger_attrs(hist_data->attrs);
712 destroy_hist_fields(hist_data);
713 tracing_map_destroy(hist_data->map);
714 kfree(hist_data);
715}
716
717static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
718{
719 struct tracing_map *map = hist_data->map;
720 struct ftrace_event_field *field;
721 struct hist_field *hist_field;
722 unsigned int i, idx;
723
724 for_each_hist_field(i, hist_data) {
725 hist_field = hist_data->fields[i];
726 if (hist_field->flags & HIST_FIELD_FL_KEY) {
727 tracing_map_cmp_fn_t cmp_fn;
728
729 field = hist_field->field;
730
69a0200c
TZ		 731 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
732 cmp_fn = tracing_map_cmp_none;
733 else if (is_string_field(field))
7ef224d1
TZ		 734 cmp_fn = tracing_map_cmp_string;
735 else
736 cmp_fn = tracing_map_cmp_num(field->size,
737 field->is_signed);
76a3b0c8
TZ		 738 idx = tracing_map_add_key_field(map,
739 hist_field->offset,
740 cmp_fn);
741
7ef224d1
TZ		 742 } else
743 idx = tracing_map_add_sum_field(map);
744
745 if (idx < 0)
746 return idx;
747 }
748
749 return 0;
750}
751
6b4827ad
TZ		 752static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
753{
754 struct hist_field *key_field;
755 unsigned int i;
756
757 for_each_hist_key_field(i, hist_data) {
758 key_field = hist_data->fields[i];
759
760 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
761 return true;
762 }
763
764 return false;
765}
766
7ef224d1
TZ		 767static struct hist_trigger_data *
768create_hist_data(unsigned int map_bits,
769 struct hist_trigger_attrs *attrs,
770 struct trace_event_file *file)
771{
6b4827ad 772 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 773 struct hist_trigger_data *hist_data;
774 int ret = 0;
775
776 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
777 if (!hist_data)
778 return ERR_PTR(-ENOMEM);
779
780 hist_data->attrs = attrs;
781
782 ret = create_hist_fields(hist_data, file);
783 if (ret)
784 goto free;
785
786 ret = create_sort_keys(hist_data);
787 if (ret)
788 goto free;
789
6b4827ad
TZ		 790 if (need_tracing_map_ops(hist_data))
791 map_ops = &hist_trigger_elt_comm_ops;
792
7ef224d1 793 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 794 map_ops, hist_data);
7ef224d1
TZ		 795 if (IS_ERR(hist_data->map)) {
796 ret = PTR_ERR(hist_data->map);
797 hist_data->map = NULL;
798 goto free;
799 }
800
801 ret = create_tracing_map_fields(hist_data);
802 if (ret)
803 goto free;
804
805 ret = tracing_map_init(hist_data->map);
806 if (ret)
807 goto free;
808
809 hist_data->event_file = file;
810 out:
811 return hist_data;
812 free:
813 hist_data->attrs = NULL;
814
815 destroy_hist_data(hist_data);
816
817 hist_data = ERR_PTR(ret);
818
819 goto out;
820}
821
822static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
823 struct tracing_map_elt *elt,
824 void *rec)
825{
826 struct hist_field *hist_field;
827 unsigned int i;
828 u64 hist_val;
829
830 for_each_hist_val_field(i, hist_data) {
831 hist_field = hist_data->fields[i];
832 hist_val = hist_field->fn(hist_field, rec);
833 tracing_map_update_sum(elt, i, hist_val);
834 }
835}
836
6a475cb1
TZ		 837static inline void add_to_key(char *compound_key, void *key,
838 struct hist_field *key_field, void *rec)
839{
840 size_t size = key_field->size;
841
842 if (key_field->flags & HIST_FIELD_FL_STRING) {
843 struct ftrace_event_field *field;
844
845 field = key_field->field;
846 if (field->filter_type == FILTER_DYN_STRING)
847 size = *(u32 *)(rec + field->offset) >> 16;
848 else if (field->filter_type == FILTER_PTR_STRING)
849 size = strlen(key);
850 else if (field->filter_type == FILTER_STATIC_STRING)
851 size = field->size;
852
853 /* ensure NULL-termination */
854 if (size > key_field->size - 1)
855 size = key_field->size - 1;
856 }
857
858 memcpy(compound_key + key_field->offset, key, size);
859}
860
7ef224d1
TZ		 861static void event_hist_trigger(struct event_trigger_data *data, void *rec)
862{
863 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 864 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 865 unsigned long entries[HIST_STACKTRACE_DEPTH];
76a3b0c8 866 char compound_key[HIST_KEY_SIZE_MAX];
69a0200c 867 struct stack_trace stacktrace;
7ef224d1
TZ		 868 struct hist_field *key_field;
869 struct tracing_map_elt *elt;
870 u64 field_contents;
871 void *key = NULL;
872 unsigned int i;
873
6a475cb1 874 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 875
7ef224d1
TZ		 876 for_each_hist_key_field(i, hist_data) {
877 key_field = hist_data->fields[i];
878
69a0200c
TZ		 879 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
880 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
881 stacktrace.entries = entries;
882 stacktrace.nr_entries = 0;
883 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 884
69a0200c
TZ		 885 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
886 save_stack_trace(&stacktrace);
887
888 key = entries;
889 } else {
890 field_contents = key_field->fn(key_field, rec);
6a475cb1 891 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 892 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 893 use_compound_key = true;
894 } else
69a0200c 895 key = (void *)&field_contents;
76a3b0c8 896 }
6a475cb1
TZ		 897
898 if (use_compound_key)
899 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 900 }
901
6a475cb1 902 if (use_compound_key)
76a3b0c8
TZ		 903 key = compound_key;
904
7ef224d1
TZ		 905 elt = tracing_map_insert(hist_data->map, key);
906 if (elt)
907 hist_trigger_elt_update(hist_data, elt, rec);
908}
909
69a0200c
TZ		 910static void hist_trigger_stacktrace_print(struct seq_file *m,
911 unsigned long *stacktrace_entries,
912 unsigned int max_entries)
913{
914 char str[KSYM_SYMBOL_LEN];
915 unsigned int spaces = 8;
916 unsigned int i;
917
918 for (i = 0; i < max_entries; i++) {
919 if (stacktrace_entries[i] == ULONG_MAX)
920 return;
921
922 seq_printf(m, "%*c", 1 + spaces, ' ');
923 sprint_symbol(str, stacktrace_entries[i]);
924 seq_printf(m, "%s\n", str);
925 }
926}
927
7ef224d1
TZ		 928static void
929hist_trigger_entry_print(struct seq_file *m,
930 struct hist_trigger_data *hist_data, void *key,
931 struct tracing_map_elt *elt)
932{
933 struct hist_field *key_field;
c6afad49 934 char str[KSYM_SYMBOL_LEN];
69a0200c 935 bool multiline = false;
7ef224d1
TZ		 936 unsigned int i;
937 u64 uval;
938
939 seq_puts(m, "{ ");
940
941 for_each_hist_key_field(i, hist_data) {
942 key_field = hist_data->fields[i];
943
944 if (i > hist_data->n_vals)
945 seq_puts(m, ", ");
946
0c4a6b46
TZ		 947 if (key_field->flags & HIST_FIELD_FL_HEX) {
948 uval = *(u64 *)(key + key_field->offset);
949 seq_printf(m, "%s: %llx",
950 key_field->field->name, uval);
c6afad49
TZ		 951 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
952 uval = *(u64 *)(key + key_field->offset);
953 sprint_symbol_no_offset(str, uval);
954 seq_printf(m, "%s: [%llx] %-45s",
955 key_field->field->name, uval, str);
956 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
957 uval = *(u64 *)(key + key_field->offset);
958 sprint_symbol(str, uval);
959 seq_printf(m, "%s: [%llx] %-55s",
960 key_field->field->name, uval, str);
6b4827ad
TZ		 961 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
962 char *comm = elt->private_data;
963
964 uval = *(u64 *)(key + key_field->offset);
965 seq_printf(m, "%s: %-16s[%10llu]",
966 key_field->field->name, comm, uval);
31696198
TZ		 967 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
968 const char *syscall_name;
969
970 uval = *(u64 *)(key + key_field->offset);
971 syscall_name = get_syscall_name(uval);
972 if (!syscall_name)
973 syscall_name = "unknown_syscall";
974
975 seq_printf(m, "%s: %-30s[%3llu]",
976 key_field->field->name, syscall_name, uval);
69a0200c
TZ		 977 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
978 seq_puts(m, "stacktrace:\n");
979 hist_trigger_stacktrace_print(m,
980 key + key_field->offset,
981 HIST_STACKTRACE_DEPTH);
982 multiline = true;
0c4a6b46 983 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
7ef224d1 984 seq_printf(m, "%s: %-50s", key_field->field->name,
76a3b0c8 985 (char *)(key + key_field->offset));
7ef224d1 986 } else {
76a3b0c8
TZ		 987 uval = *(u64 *)(key + key_field->offset);
988 seq_printf(m, "%s: %10llu", key_field->field->name,
989 uval);
7ef224d1
TZ		 990 }
991 }
992
69a0200c
TZ		 993 if (!multiline)
994 seq_puts(m, " ");
995
996 seq_puts(m, "}");
7ef224d1
TZ		 997
998 seq_printf(m, " hitcount: %10llu",
999 tracing_map_read_sum(elt, HITCOUNT_IDX));
1000
f2606835 1001 for (i = 1; i < hist_data->n_vals; i++) {
0c4a6b46
TZ		 1002 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
1003 seq_printf(m, " %s: %10llx",
1004 hist_data->fields[i]->field->name,
1005 tracing_map_read_sum(elt, i));
1006 } else {
1007 seq_printf(m, " %s: %10llu",
1008 hist_data->fields[i]->field->name,
1009 tracing_map_read_sum(elt, i));
1010 }
f2606835
TZ		 1011 }
1012
7ef224d1
TZ		 1013 seq_puts(m, "\n");
1014}
1015
1016static int print_entries(struct seq_file *m,
1017 struct hist_trigger_data *hist_data)
1018{
1019 struct tracing_map_sort_entry **sort_entries = NULL;
1020 struct tracing_map *map = hist_data->map;
1021 unsigned int i, n_entries;
1022
1023 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1024 hist_data->n_sort_keys,
1025 &sort_entries);
1026 if (n_entries < 0)
1027 return n_entries;
1028
1029 for (i = 0; i < n_entries; i++)
1030 hist_trigger_entry_print(m, hist_data,
1031 sort_entries[i]->key,
1032 sort_entries[i]->elt);
1033
1034 tracing_map_destroy_sort_entries(sort_entries, n_entries);
1035
1036 return n_entries;
1037}
1038
52a7f16d
TZ		 1039static void hist_trigger_show(struct seq_file *m,
1040 struct event_trigger_data *data, int n)
7ef224d1 1041{
7ef224d1
TZ		 1042 struct hist_trigger_data *hist_data;
1043 int n_entries, ret = 0;
1044
52a7f16d
TZ		 1045 if (n > 0)
1046 seq_puts(m, "\n\n");
7ef224d1
TZ		 1047
1048 seq_puts(m, "# event histogram\n#\n# trigger info: ");
1049 data->ops->print(m, data->ops, data);
52a7f16d 1050 seq_puts(m, "#\n\n");
7ef224d1
TZ		 1051
1052 hist_data = data->private_data;
1053 n_entries = print_entries(m, hist_data);
1054 if (n_entries < 0) {
1055 ret = n_entries;
1056 n_entries = 0;
1057 }
1058
1059 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
1060 (u64)atomic64_read(&hist_data->map->hits),
1061 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 1062}
1063
1064static int hist_show(struct seq_file *m, void *v)
1065{
1066 struct event_trigger_data *data;
1067 struct trace_event_file *event_file;
1068 int n = 0, ret = 0;
1069
1070 mutex_lock(&event_mutex);
1071
1072 event_file = event_file_data(m->private);
1073 if (unlikely(!event_file)) {
1074 ret = -ENODEV;
1075 goto out_unlock;
1076 }
1077
1078 list_for_each_entry_rcu(data, &event_file->triggers, list) {
1079 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1080 hist_trigger_show(m, data, n++);
1081 }
1082
7ef224d1
TZ		 1083 out_unlock:
1084 mutex_unlock(&event_mutex);
1085
1086 return ret;
1087}
1088
1089static int event_hist_open(struct inode *inode, struct file *file)
1090{
1091 return single_open(file, hist_show, file);
1092}
1093
1094const struct file_operations event_hist_fops = {
1095 .open = event_hist_open,
1096 .read = seq_read,
1097 .llseek = seq_lseek,
1098 .release = single_release,
1099};
1100
0c4a6b46
TZ		 1101static const char *get_hist_field_flags(struct hist_field *hist_field)
1102{
1103 const char *flags_str = NULL;
1104
1105 if (hist_field->flags & HIST_FIELD_FL_HEX)
1106 flags_str = "hex";
c6afad49
TZ		 1107 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1108 flags_str = "sym";
1109 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1110 flags_str = "sym-offset";
6b4827ad
TZ		 1111 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1112 flags_str = "execname";
31696198
TZ		 1113 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1114 flags_str = "syscall";
0c4a6b46
TZ		 1115
1116 return flags_str;
1117}
1118
7ef224d1
TZ		 1119static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1120{
1121 seq_printf(m, "%s", hist_field->field->name);
0c4a6b46
TZ		 1122 if (hist_field->flags) {
1123 const char *flags_str = get_hist_field_flags(hist_field);
1124
1125 if (flags_str)
1126 seq_printf(m, ".%s", flags_str);
1127 }
7ef224d1
TZ		 1128}
1129
1130static int event_hist_trigger_print(struct seq_file *m,
1131 struct event_trigger_ops *ops,
1132 struct event_trigger_data *data)
1133{
1134 struct hist_trigger_data *hist_data = data->private_data;
1135 struct hist_field *key_field;
1136 unsigned int i;
1137
5463bfda
TZ		 1138 seq_puts(m, "hist:");
1139
1140 if (data->name)
1141 seq_printf(m, "%s:", data->name);
1142
1143 seq_puts(m, "keys=");
7ef224d1
TZ		 1144
1145 for_each_hist_key_field(i, hist_data) {
1146 key_field = hist_data->fields[i];
1147
1148 if (i > hist_data->n_vals)
1149 seq_puts(m, ",");
1150
69a0200c
TZ		 1151 if (key_field->flags & HIST_FIELD_FL_STACKTRACE)
1152 seq_puts(m, "stacktrace");
1153 else
1154 hist_field_print(m, key_field);
7ef224d1
TZ		 1155 }
1156
1157 seq_puts(m, ":vals=");
f2606835
TZ		 1158
1159 for_each_hist_val_field(i, hist_data) {
1160 if (i == HITCOUNT_IDX)
1161 seq_puts(m, "hitcount");
1162 else {
1163 seq_puts(m, ",");
1164 hist_field_print(m, hist_data->fields[i]);
1165 }
1166 }
7ef224d1
TZ		 1167
1168 seq_puts(m, ":sort=");
e62347d2
TZ		 1169
1170 for (i = 0; i < hist_data->n_sort_keys; i++) {
1171 struct tracing_map_sort_key *sort_key;
1172
1173 sort_key = &hist_data->sort_keys[i];
1174
1175 if (i > 0)
1176 seq_puts(m, ",");
1177
1178 if (sort_key->field_idx == HITCOUNT_IDX)
1179 seq_puts(m, "hitcount");
1180 else {
1181 unsigned int idx = sort_key->field_idx;
1182
1183 if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1184 return -EINVAL;
1185
1186 hist_field_print(m, hist_data->fields[idx]);
1187 }
1188
1189 if (sort_key->descending)
1190 seq_puts(m, ".descending");
1191 }
7ef224d1
TZ		 1192
1193 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1194
1195 if (data->filter_str)
1196 seq_printf(m, " if %s", data->filter_str);
1197
83e99914
TZ		 1198 if (data->paused)
1199 seq_puts(m, " [paused]");
1200 else
1201 seq_puts(m, " [active]");
7ef224d1
TZ		 1202
1203 seq_putc(m, '\n');
1204
1205 return 0;
1206}
1207
5463bfda
TZ		 1208static int event_hist_trigger_init(struct event_trigger_ops *ops,
1209 struct event_trigger_data *data)
1210{
1211 struct hist_trigger_data *hist_data = data->private_data;
1212
1213 if (!data->ref && hist_data->attrs->name)
1214 save_named_trigger(hist_data->attrs->name, data);
1215
1216 data->ref++;
1217
1218 return 0;
1219}
1220
7ef224d1
TZ		 1221static void event_hist_trigger_free(struct event_trigger_ops *ops,
1222 struct event_trigger_data *data)
1223{
1224 struct hist_trigger_data *hist_data = data->private_data;
1225
1226 if (WARN_ON_ONCE(data->ref <= 0))
1227 return;
1228
1229 data->ref--;
1230 if (!data->ref) {
5463bfda
TZ		 1231 if (data->name)
1232 del_named_trigger(data);
7ef224d1
TZ		 1233 trigger_data_free(data);
1234 destroy_hist_data(hist_data);
1235 }
1236}
1237
1238static struct event_trigger_ops event_hist_trigger_ops = {
1239 .func = event_hist_trigger,
1240 .print = event_hist_trigger_print,
5463bfda 1241 .init = event_hist_trigger_init,
7ef224d1
TZ		 1242 .free = event_hist_trigger_free,
1243};
1244
5463bfda
TZ		 1245static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
1246 struct event_trigger_data *data)
1247{
1248 data->ref++;
1249
1250 save_named_trigger(data->named_data->name, data);
1251
1252 event_hist_trigger_init(ops, data->named_data);
1253
1254 return 0;
1255}
1256
1257static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
1258 struct event_trigger_data *data)
1259{
1260 if (WARN_ON_ONCE(data->ref <= 0))
1261 return;
1262
1263 event_hist_trigger_free(ops, data->named_data);
1264
1265 data->ref--;
1266 if (!data->ref) {
1267 del_named_trigger(data);
1268 trigger_data_free(data);
1269 }
1270}
1271
1272static struct event_trigger_ops event_hist_trigger_named_ops = {
1273 .func = event_hist_trigger,
1274 .print = event_hist_trigger_print,
1275 .init = event_hist_trigger_named_init,
1276 .free = event_hist_trigger_named_free,
1277};
1278
7ef224d1
TZ		 1279static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1280 char *param)
1281{
1282 return &event_hist_trigger_ops;
1283}
1284
e86ae9ba
TZ		 1285static void hist_clear(struct event_trigger_data *data)
1286{
1287 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 1288
5463bfda
TZ		 1289 if (data->name)
1290 pause_named_trigger(data);
e86ae9ba
TZ		 1291
1292 synchronize_sched();
1293
1294 tracing_map_clear(hist_data->map);
1295
5463bfda
TZ		 1296 if (data->name)
1297 unpause_named_trigger(data);
1298}
1299
1300static bool compatible_field(struct ftrace_event_field *field,
1301 struct ftrace_event_field *test_field)
1302{
1303 if (field == test_field)
1304 return true;
1305 if (field == NULL || test_field == NULL)
1306 return false;
1307 if (strcmp(field->name, test_field->name) != 0)
1308 return false;
1309 if (strcmp(field->type, test_field->type) != 0)
1310 return false;
1311 if (field->size != test_field->size)
1312 return false;
1313 if (field->is_signed != test_field->is_signed)
1314 return false;
1315
1316 return true;
e86ae9ba
TZ		 1317}
1318
52a7f16d 1319static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 1320 struct event_trigger_data *data_test,
1321 struct event_trigger_data *named_data,
1322 bool ignore_filter)
52a7f16d
TZ		 1323{
1324 struct tracing_map_sort_key *sort_key, *sort_key_test;
1325 struct hist_trigger_data *hist_data, *hist_data_test;
1326 struct hist_field *key_field, *key_field_test;
1327 unsigned int i;
1328
5463bfda
TZ		 1329 if (named_data && (named_data != data_test) &&
1330 (named_data != data_test->named_data))
1331 return false;
1332
1333 if (!named_data && is_named_trigger(data_test))
1334 return false;
1335
52a7f16d
TZ		 1336 hist_data = data->private_data;
1337 hist_data_test = data_test->private_data;
1338
1339 if (hist_data->n_vals != hist_data_test->n_vals ||
1340 hist_data->n_fields != hist_data_test->n_fields ||
1341 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
1342 return false;
1343
5463bfda
TZ		 1344 if (!ignore_filter) {
1345 if ((data->filter_str && !data_test->filter_str) ||
1346 (!data->filter_str && data_test->filter_str))
1347 return false;
1348 }
52a7f16d
TZ		 1349
1350 for_each_hist_field(i, hist_data) {
1351 key_field = hist_data->fields[i];
1352 key_field_test = hist_data_test->fields[i];
1353
1354 if (key_field->flags != key_field_test->flags)
1355 return false;
5463bfda 1356 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 1357 return false;
1358 if (key_field->offset != key_field_test->offset)
1359 return false;
1360 }
1361
1362 for (i = 0; i < hist_data->n_sort_keys; i++) {
1363 sort_key = &hist_data->sort_keys[i];
1364 sort_key_test = &hist_data_test->sort_keys[i];
1365
1366 if (sort_key->field_idx != sort_key_test->field_idx ||
1367 sort_key->descending != sort_key_test->descending)
1368 return false;
1369 }
1370
5463bfda 1371 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 1372 (strcmp(data->filter_str, data_test->filter_str) != 0))
1373 return false;
1374
1375 return true;
1376}
1377
7ef224d1
TZ		 1378static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1379 struct event_trigger_data *data,
1380 struct trace_event_file *file)
1381{
83e99914 1382 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 1383 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 1384 int ret = 0;
1385
5463bfda
TZ		 1386 if (hist_data->attrs->name) {
1387 named_data = find_named_trigger(hist_data->attrs->name);
1388 if (named_data) {
1389 if (!hist_trigger_match(data, named_data, named_data,
1390 true)) {
1391 ret = -EINVAL;
1392 goto out;
1393 }
1394 }
1395 }
1396
1397 if (hist_data->attrs->name && !named_data)
1398 goto new;
1399
7ef224d1
TZ		 1400 list_for_each_entry_rcu(test, &file->triggers, list) {
1401 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 1402 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 1403 continue;
83e99914
TZ		 1404 if (hist_data->attrs->pause)
1405 test->paused = true;
1406 else if (hist_data->attrs->cont)
1407 test->paused = false;
e86ae9ba
TZ		 1408 else if (hist_data->attrs->clear)
1409 hist_clear(test);
83e99914
TZ		 1410 else
1411 ret = -EEXIST;
7ef224d1
TZ		 1412 goto out;
1413 }
1414 }
5463bfda 1415 new:
e86ae9ba 1416 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 1417 ret = -ENOENT;
1418 goto out;
1419 }
1420
5463bfda
TZ		 1421 if (named_data) {
1422 destroy_hist_data(data->private_data);
1423 data->private_data = named_data->private_data;
1424 set_named_trigger_data(data, named_data);
1425 data->ops = &event_hist_trigger_named_ops;
1426 }
1427
83e99914
TZ		 1428 if (hist_data->attrs->pause)
1429 data->paused = true;
1430
7ef224d1
TZ		 1431 if (data->ops->init) {
1432 ret = data->ops->init(data->ops, data);
1433 if (ret < 0)
1434 goto out;
1435 }
1436
1437 list_add_rcu(&data->list, &file->triggers);
1438 ret++;
1439
1440 update_cond_flag(file);
5463bfda 1441
7ef224d1
TZ		 1442 if (trace_event_trigger_enable_disable(file, 1) < 0) {
1443 list_del_rcu(&data->list);
1444 update_cond_flag(file);
1445 ret--;
1446 }
1447 out:
1448 return ret;
1449}
1450
52a7f16d
TZ		 1451static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
1452 struct event_trigger_data *data,
1453 struct trace_event_file *file)
1454{
5463bfda
TZ		 1455 struct hist_trigger_data *hist_data = data->private_data;
1456 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 1457 bool unregistered = false;
1458
5463bfda
TZ		 1459 if (hist_data->attrs->name)
1460 named_data = find_named_trigger(hist_data->attrs->name);
1461
52a7f16d
TZ		 1462 list_for_each_entry_rcu(test, &file->triggers, list) {
1463 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 1464 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 1465 continue;
1466 unregistered = true;
1467 list_del_rcu(&test->list);
1468 trace_event_trigger_enable_disable(file, 0);
1469 update_cond_flag(file);
1470 break;
1471 }
1472 }
1473
1474 if (unregistered && test->ops->free)
1475 test->ops->free(test->ops, test);
1476}
1477
1478static void hist_unreg_all(struct trace_event_file *file)
1479{
1480 struct event_trigger_data *test;
1481
1482 list_for_each_entry_rcu(test, &file->triggers, list) {
1483 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1484 list_del_rcu(&test->list);
1485 trace_event_trigger_enable_disable(file, 0);
1486 update_cond_flag(file);
1487 if (test->ops->free)
1488 test->ops->free(test->ops, test);
1489 }
1490 }
1491}
1492
7ef224d1
TZ		 1493static int event_hist_trigger_func(struct event_command *cmd_ops,
1494 struct trace_event_file *file,
1495 char *glob, char *cmd, char *param)
1496{
1497 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1498 struct event_trigger_data *trigger_data;
1499 struct hist_trigger_attrs *attrs;
1500 struct event_trigger_ops *trigger_ops;
1501 struct hist_trigger_data *hist_data;
1502 char *trigger;
1503 int ret = 0;
1504
1505 if (!param)
1506 return -EINVAL;
1507
1508 /* separate the trigger from the filter (k:v [if filter]) */
1509 trigger = strsep(&param, " \t");
1510 if (!trigger)
1511 return -EINVAL;
1512
1513 attrs = parse_hist_trigger_attrs(trigger);
1514 if (IS_ERR(attrs))
1515 return PTR_ERR(attrs);
1516
1517 if (attrs->map_bits)
1518 hist_trigger_bits = attrs->map_bits;
1519
1520 hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1521 if (IS_ERR(hist_data)) {
1522 destroy_hist_trigger_attrs(attrs);
1523 return PTR_ERR(hist_data);
1524 }
1525
1526 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1527
1528 ret = -ENOMEM;
1529 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1530 if (!trigger_data)
1531 goto out_free;
1532
1533 trigger_data->count = -1;
1534 trigger_data->ops = trigger_ops;
1535 trigger_data->cmd_ops = cmd_ops;
1536
1537 INIT_LIST_HEAD(&trigger_data->list);
1538 RCU_INIT_POINTER(trigger_data->filter, NULL);
1539
1540 trigger_data->private_data = hist_data;
1541
52a7f16d
TZ		 1542 /* if param is non-empty, it's supposed to be a filter */
1543 if (param && cmd_ops->set_filter) {
1544 ret = cmd_ops->set_filter(param, trigger_data, file);
1545 if (ret < 0)
1546 goto out_free;
1547 }
1548
7ef224d1
TZ		 1549 if (glob[0] == '!') {
1550 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1551 ret = 0;
1552 goto out_free;
1553 }
1554
7ef224d1
TZ		 1555 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1556 /*
1557 * The above returns on success the # of triggers registered,
1558 * but if it didn't register any it returns zero. Consider no
1559 * triggers registered a failure too.
1560 */
1561 if (!ret) {
e86ae9ba 1562 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 1563 ret = -ENOENT;
7ef224d1
TZ		 1564 goto out_free;
1565 } else if (ret < 0)
1566 goto out_free;
1567 /* Just return zero, not the number of registered triggers */
1568 ret = 0;
1569 out:
1570 return ret;
1571 out_free:
1572 if (cmd_ops->set_filter)
1573 cmd_ops->set_filter(NULL, trigger_data, NULL);
1574
1575 kfree(trigger_data);
1576
1577 destroy_hist_data(hist_data);
1578 goto out;
1579}
1580
1581static struct event_command trigger_hist_cmd = {
1582 .name = "hist",
1583 .trigger_type = ETT_EVENT_HIST,
1584 .flags = EVENT_CMD_FL_NEEDS_REC,
1585 .func = event_hist_trigger_func,
1586 .reg = hist_register_trigger,
52a7f16d
TZ		 1587 .unreg = hist_unregister_trigger,
1588 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 1589 .get_trigger_ops = event_hist_get_trigger_ops,
1590 .set_filter = set_trigger_filter,
1591};
1592
1593__init int register_trigger_hist_cmd(void)
1594{
1595 int ret;
1596
1597 ret = register_event_command(&trigger_hist_cmd);
1598 WARN_ON(ret < 0);
1599
1600 return ret;
1601}
d0bad49b
TZ		 1602
1603static void
1604hist_enable_trigger(struct event_trigger_data *data, void *rec)
1605{
1606 struct enable_trigger_data *enable_data = data->private_data;
1607 struct event_trigger_data *test;
1608
1609 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
1610 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1611 if (enable_data->enable)
1612 test->paused = false;
1613 else
1614 test->paused = true;
d0bad49b
TZ		 1615 }
1616 }
1617}
1618
1619static void
1620hist_enable_count_trigger(struct event_trigger_data *data, void *rec)
1621{
1622 if (!data->count)
1623 return;
1624
1625 if (data->count != -1)
1626 (data->count)--;
1627
1628 hist_enable_trigger(data, rec);
1629}
1630
1631static struct event_trigger_ops hist_enable_trigger_ops = {
1632 .func = hist_enable_trigger,
1633 .print = event_enable_trigger_print,
1634 .init = event_trigger_init,
1635 .free = event_enable_trigger_free,
1636};
1637
1638static struct event_trigger_ops hist_enable_count_trigger_ops = {
1639 .func = hist_enable_count_trigger,
1640 .print = event_enable_trigger_print,
1641 .init = event_trigger_init,
1642 .free = event_enable_trigger_free,
1643};
1644
1645static struct event_trigger_ops hist_disable_trigger_ops = {
1646 .func = hist_enable_trigger,
1647 .print = event_enable_trigger_print,
1648 .init = event_trigger_init,
1649 .free = event_enable_trigger_free,
1650};
1651
1652static struct event_trigger_ops hist_disable_count_trigger_ops = {
1653 .func = hist_enable_count_trigger,
1654 .print = event_enable_trigger_print,
1655 .init = event_trigger_init,
1656 .free = event_enable_trigger_free,
1657};
1658
1659static struct event_trigger_ops *
1660hist_enable_get_trigger_ops(char *cmd, char *param)
1661{
1662 struct event_trigger_ops *ops;
1663 bool enable;
1664
1665 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
1666
1667 if (enable)
1668 ops = param ? &hist_enable_count_trigger_ops :
1669 &hist_enable_trigger_ops;
1670 else
1671 ops = param ? &hist_disable_count_trigger_ops :
1672 &hist_disable_trigger_ops;
1673
1674 return ops;
1675}
1676
52a7f16d
TZ		 1677static void hist_enable_unreg_all(struct trace_event_file *file)
1678{
1679 struct event_trigger_data *test;
1680
1681 list_for_each_entry_rcu(test, &file->triggers, list) {
1682 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
1683 list_del_rcu(&test->list);
1684 update_cond_flag(file);
1685 trace_event_trigger_enable_disable(file, 0);
1686 if (test->ops->free)
1687 test->ops->free(test->ops, test);
1688 }
1689 }
1690}
1691
d0bad49b
TZ		 1692static struct event_command trigger_hist_enable_cmd = {
1693 .name = ENABLE_HIST_STR,
1694 .trigger_type = ETT_HIST_ENABLE,
1695 .func = event_enable_trigger_func,
1696 .reg = event_enable_register_trigger,
1697 .unreg = event_enable_unregister_trigger,
52a7f16d 1698 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 1699 .get_trigger_ops = hist_enable_get_trigger_ops,
1700 .set_filter = set_trigger_filter,
1701};
1702
1703static struct event_command trigger_hist_disable_cmd = {
1704 .name = DISABLE_HIST_STR,
1705 .trigger_type = ETT_HIST_ENABLE,
1706 .func = event_enable_trigger_func,
1707 .reg = event_enable_register_trigger,
1708 .unreg = event_enable_unregister_trigger,
52a7f16d 1709 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 1710 .get_trigger_ops = hist_enable_get_trigger_ops,
1711 .set_filter = set_trigger_filter,
1712};
1713
1714static __init void unregister_trigger_hist_enable_disable_cmds(void)
1715{
1716 unregister_event_command(&trigger_hist_enable_cmd);
1717 unregister_event_command(&trigger_hist_disable_cmd);
1718}
1719
1720__init int register_trigger_hist_enable_disable_cmds(void)
1721{
1722 int ret;
1723
1724 ret = register_event_command(&trigger_hist_enable_cmd);
1725 if (WARN_ON(ret < 0))
1726 return ret;
1727 ret = register_event_command(&trigger_hist_disable_cmd);
1728 if (WARN_ON(ret < 0))
1729 unregister_trigger_hist_enable_disable_cmds();
1730
1731 return ret;
1732}
Linux 6.x block layer and io_uring tree(s)