projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Add hist trigger 'execname' modifier
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
22
23#include "tracing_map.h"
24#include "trace.h"
25
26struct hist_field;
27
28typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
29
30struct hist_field {
31 struct ftrace_event_field *field;
32 unsigned long flags;
33 hist_field_fn_t fn;
34 unsigned int size;
76a3b0c8 35 unsigned int offset;
7ef224d1
TZ		 36};
37
38static u64 hist_field_counter(struct hist_field *field, void *event)
39{
40 return 1;
41}
42
43static u64 hist_field_string(struct hist_field *hist_field, void *event)
44{
45 char *addr = (char *)(event + hist_field->field->offset);
46
47 return (u64)(unsigned long)addr;
48}
49
50#define DEFINE_HIST_FIELD_FN(type) \
51static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
52{ \
53 type *addr = (type *)(event + hist_field->field->offset); \
54 \
55 return (u64)*addr; \
56}
57
58DEFINE_HIST_FIELD_FN(s64);
59DEFINE_HIST_FIELD_FN(u64);
60DEFINE_HIST_FIELD_FN(s32);
61DEFINE_HIST_FIELD_FN(u32);
62DEFINE_HIST_FIELD_FN(s16);
63DEFINE_HIST_FIELD_FN(u16);
64DEFINE_HIST_FIELD_FN(s8);
65DEFINE_HIST_FIELD_FN(u8);
66
67#define for_each_hist_field(i, hist_data) \
68 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
69
70#define for_each_hist_val_field(i, hist_data) \
71 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
72
73#define for_each_hist_key_field(i, hist_data) \
74 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
75
76#define HITCOUNT_IDX 0
76a3b0c8 77#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + sizeof(u64))
7ef224d1
TZ		 78
79enum hist_field_flags {
c6afad49
TZ		 80 HIST_FIELD_FL_HITCOUNT = 1,
81 HIST_FIELD_FL_KEY = 2,
82 HIST_FIELD_FL_STRING = 4,
83 HIST_FIELD_FL_HEX = 8,
84 HIST_FIELD_FL_SYM = 16,
85 HIST_FIELD_FL_SYM_OFFSET = 32,
6b4827ad 86 HIST_FIELD_FL_EXECNAME = 64,
7ef224d1
TZ		 87};
88
89struct hist_trigger_attrs {
90 char *keys_str;
f2606835 91 char *vals_str;
e62347d2 92 char *sort_key_str;
83e99914
TZ		 93 bool pause;
94 bool cont;
e86ae9ba 95 bool clear;
7ef224d1
TZ		 96 unsigned int map_bits;
97};
98
99struct hist_trigger_data {
100 struct hist_field *fields[TRACING_MAP_FIELDS_MAX];
101 unsigned int n_vals;
102 unsigned int n_keys;
103 unsigned int n_fields;
104 unsigned int key_size;
105 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
106 unsigned int n_sort_keys;
107 struct trace_event_file *event_file;
108 struct hist_trigger_attrs *attrs;
109 struct tracing_map *map;
110};
111
112static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
113{
114 hist_field_fn_t fn = NULL;
115
116 switch (field_size) {
117 case 8:
118 if (field_is_signed)
119 fn = hist_field_s64;
120 else
121 fn = hist_field_u64;
122 break;
123 case 4:
124 if (field_is_signed)
125 fn = hist_field_s32;
126 else
127 fn = hist_field_u32;
128 break;
129 case 2:
130 if (field_is_signed)
131 fn = hist_field_s16;
132 else
133 fn = hist_field_u16;
134 break;
135 case 1:
136 if (field_is_signed)
137 fn = hist_field_s8;
138 else
139 fn = hist_field_u8;
140 break;
141 }
142
143 return fn;
144}
145
146static int parse_map_size(char *str)
147{
148 unsigned long size, map_bits;
149 int ret;
150
151 strsep(&str, "=");
152 if (!str) {
153 ret = -EINVAL;
154 goto out;
155 }
156
157 ret = kstrtoul(str, 0, &size);
158 if (ret)
159 goto out;
160
161 map_bits = ilog2(roundup_pow_of_two(size));
162 if (map_bits < TRACING_MAP_BITS_MIN ||
163 map_bits > TRACING_MAP_BITS_MAX)
164 ret = -EINVAL;
165 else
166 ret = map_bits;
167 out:
168 return ret;
169}
170
171static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
172{
173 if (!attrs)
174 return;
175
e62347d2 176 kfree(attrs->sort_key_str);
7ef224d1 177 kfree(attrs->keys_str);
f2606835 178 kfree(attrs->vals_str);
7ef224d1
TZ		 179 kfree(attrs);
180}
181
182static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
183{
184 struct hist_trigger_attrs *attrs;
185 int ret = 0;
186
187 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
188 if (!attrs)
189 return ERR_PTR(-ENOMEM);
190
191 while (trigger_str) {
192 char *str = strsep(&trigger_str, ":");
193
194 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
195 (strncmp(str, "keys=", strlen("keys=")) == 0))
196 attrs->keys_str = kstrdup(str, GFP_KERNEL);
f2606835
TZ		 197 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
198 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
199 (strncmp(str, "values=", strlen("values=")) == 0))
200 attrs->vals_str = kstrdup(str, GFP_KERNEL);
e62347d2
TZ		 201 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
202 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
83e99914
TZ		 203 else if (strcmp(str, "pause") == 0)
204 attrs->pause = true;
205 else if ((strcmp(str, "cont") == 0) ||
206 (strcmp(str, "continue") == 0))
207 attrs->cont = true;
e86ae9ba
TZ		 208 else if (strcmp(str, "clear") == 0)
209 attrs->clear = true;
7ef224d1
TZ		 210 else if (strncmp(str, "size=", strlen("size=")) == 0) {
211 int map_bits = parse_map_size(str);
212
213 if (map_bits < 0) {
214 ret = map_bits;
215 goto free;
216 }
217 attrs->map_bits = map_bits;
218 } else {
219 ret = -EINVAL;
220 goto free;
221 }
222 }
223
224 if (!attrs->keys_str) {
225 ret = -EINVAL;
226 goto free;
227 }
228
229 return attrs;
230 free:
231 destroy_hist_trigger_attrs(attrs);
232
233 return ERR_PTR(ret);
234}
235
6b4827ad
TZ		 236static inline void save_comm(char *comm, struct task_struct *task)
237{
238 if (!task->pid) {
239 strcpy(comm, "<idle>");
240 return;
241 }
242
243 if (WARN_ON_ONCE(task->pid < 0)) {
244 strcpy(comm, "<XXX>");
245 return;
246 }
247
248 memcpy(comm, task->comm, TASK_COMM_LEN);
249}
250
251static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
252{
253 kfree((char *)elt->private_data);
254}
255
256static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
257{
258 struct hist_trigger_data *hist_data = elt->map->private_data;
259 struct hist_field *key_field;
260 unsigned int i;
261
262 for_each_hist_key_field(i, hist_data) {
263 key_field = hist_data->fields[i];
264
265 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
266 unsigned int size = TASK_COMM_LEN + 1;
267
268 elt->private_data = kzalloc(size, GFP_KERNEL);
269 if (!elt->private_data)
270 return -ENOMEM;
271 break;
272 }
273 }
274
275 return 0;
276}
277
278static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
279 struct tracing_map_elt *from)
280{
281 char *comm_from = from->private_data;
282 char *comm_to = to->private_data;
283
284 if (comm_from)
285 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
286}
287
288static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
289{
290 char *comm = elt->private_data;
291
292 if (comm)
293 save_comm(comm, current);
294}
295
296static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
297 .elt_alloc = hist_trigger_elt_comm_alloc,
298 .elt_copy = hist_trigger_elt_comm_copy,
299 .elt_free = hist_trigger_elt_comm_free,
300 .elt_init = hist_trigger_elt_comm_init,
301};
302
7ef224d1
TZ		 303static void destroy_hist_field(struct hist_field *hist_field)
304{
305 kfree(hist_field);
306}
307
308static struct hist_field *create_hist_field(struct ftrace_event_field *field,
309 unsigned long flags)
310{
311 struct hist_field *hist_field;
312
313 if (field && is_function_field(field))
314 return NULL;
315
316 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
317 if (!hist_field)
318 return NULL;
319
320 if (flags & HIST_FIELD_FL_HITCOUNT) {
321 hist_field->fn = hist_field_counter;
322 goto out;
323 }
324
325 if (is_string_field(field)) {
326 flags |= HIST_FIELD_FL_STRING;
327 hist_field->fn = hist_field_string;
328 } else {
329 hist_field->fn = select_value_fn(field->size,
330 field->is_signed);
331 if (!hist_field->fn) {
332 destroy_hist_field(hist_field);
333 return NULL;
334 }
335 }
336 out:
337 hist_field->field = field;
338 hist_field->flags = flags;
339
340 return hist_field;
341}
342
343static void destroy_hist_fields(struct hist_trigger_data *hist_data)
344{
345 unsigned int i;
346
347 for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
348 if (hist_data->fields[i]) {
349 destroy_hist_field(hist_data->fields[i]);
350 hist_data->fields[i] = NULL;
351 }
352 }
353}
354
355static int create_hitcount_val(struct hist_trigger_data *hist_data)
356{
357 hist_data->fields[HITCOUNT_IDX] =
358 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
359 if (!hist_data->fields[HITCOUNT_IDX])
360 return -ENOMEM;
361
362 hist_data->n_vals++;
363
364 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
365 return -EINVAL;
366
367 return 0;
368}
369
f2606835
TZ		 370static int create_val_field(struct hist_trigger_data *hist_data,
371 unsigned int val_idx,
372 struct trace_event_file *file,
373 char *field_str)
374{
375 struct ftrace_event_field *field = NULL;
376 unsigned long flags = 0;
0c4a6b46 377 char *field_name;
f2606835
TZ		 378 int ret = 0;
379
380 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
381 return -EINVAL;
0c4a6b46
TZ		 382
383 field_name = strsep(&field_str, ".");
384 if (field_str) {
385 if (strcmp(field_str, "hex") == 0)
386 flags |= HIST_FIELD_FL_HEX;
387 else {
388 ret = -EINVAL;
389 goto out;
390 }
391 }
392
393 field = trace_find_event_field(file->event_call, field_name);
f2606835
TZ		 394 if (!field) {
395 ret = -EINVAL;
396 goto out;
397 }
398
399 hist_data->fields[val_idx] = create_hist_field(field, flags);
400 if (!hist_data->fields[val_idx]) {
401 ret = -ENOMEM;
402 goto out;
403 }
404
405 ++hist_data->n_vals;
406
407 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
408 ret = -EINVAL;
409 out:
410 return ret;
411}
412
7ef224d1
TZ		 413static int create_val_fields(struct hist_trigger_data *hist_data,
414 struct trace_event_file *file)
415{
f2606835
TZ		 416 char *fields_str, *field_str;
417 unsigned int i, j;
7ef224d1
TZ		 418 int ret;
419
420 ret = create_hitcount_val(hist_data);
f2606835
TZ		 421 if (ret)
422 goto out;
7ef224d1 423
f2606835
TZ		 424 fields_str = hist_data->attrs->vals_str;
425 if (!fields_str)
426 goto out;
427
428 strsep(&fields_str, "=");
429 if (!fields_str)
430 goto out;
431
432 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
433 j < TRACING_MAP_VALS_MAX; i++) {
434 field_str = strsep(&fields_str, ",");
435 if (!field_str)
436 break;
437 if (strcmp(field_str, "hitcount") == 0)
438 continue;
439 ret = create_val_field(hist_data, j++, file, field_str);
440 if (ret)
441 goto out;
442 }
443 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
444 ret = -EINVAL;
445 out:
7ef224d1
TZ		 446 return ret;
447}
448
449static int create_key_field(struct hist_trigger_data *hist_data,
450 unsigned int key_idx,
76a3b0c8 451 unsigned int key_offset,
7ef224d1
TZ		 452 struct trace_event_file *file,
453 char *field_str)
454{
455 struct ftrace_event_field *field = NULL;
456 unsigned long flags = 0;
457 unsigned int key_size;
0c4a6b46 458 char *field_name;
7ef224d1
TZ		 459 int ret = 0;
460
461 if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
462 return -EINVAL;
463
464 flags |= HIST_FIELD_FL_KEY;
465
0c4a6b46
TZ		 466 field_name = strsep(&field_str, ".");
467 if (field_str) {
468 if (strcmp(field_str, "hex") == 0)
469 flags |= HIST_FIELD_FL_HEX;
c6afad49
TZ		 470 else if (strcmp(field_str, "sym") == 0)
471 flags |= HIST_FIELD_FL_SYM;
472 else if (strcmp(field_str, "sym-offset") == 0)
473 flags |= HIST_FIELD_FL_SYM_OFFSET;
6b4827ad
TZ		 474 else if ((strcmp(field_str, "execname") == 0) &&
475 (strcmp(field_name, "common_pid") == 0))
476 flags |= HIST_FIELD_FL_EXECNAME;
0c4a6b46
TZ		 477 else {
478 ret = -EINVAL;
479 goto out;
480 }
481 }
482
483 field = trace_find_event_field(file->event_call, field_name);
7ef224d1
TZ		 484 if (!field) {
485 ret = -EINVAL;
486 goto out;
487 }
488
489 key_size = field->size;
490
491 hist_data->fields[key_idx] = create_hist_field(field, flags);
492 if (!hist_data->fields[key_idx]) {
493 ret = -ENOMEM;
494 goto out;
495 }
496
497 key_size = ALIGN(key_size, sizeof(u64));
498 hist_data->fields[key_idx]->size = key_size;
76a3b0c8
TZ		 499 hist_data->fields[key_idx]->offset = key_offset;
500 hist_data->key_size += key_size;
7ef224d1
TZ		 501 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
502 ret = -EINVAL;
503 goto out;
504 }
505
506 hist_data->n_keys++;
507
508 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
509 return -EINVAL;
510
511 ret = key_size;
512 out:
513 return ret;
514}
515
516static int create_key_fields(struct hist_trigger_data *hist_data,
517 struct trace_event_file *file)
518{
76a3b0c8 519 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 520 char *fields_str, *field_str;
521 int ret = -EINVAL;
522
523 fields_str = hist_data->attrs->keys_str;
524 if (!fields_str)
525 goto out;
526
527 strsep(&fields_str, "=");
528 if (!fields_str)
529 goto out;
530
76a3b0c8 531 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 532 field_str = strsep(&fields_str, ",");
533 if (!field_str)
534 break;
76a3b0c8
TZ		 535 ret = create_key_field(hist_data, i, key_offset,
536 file, field_str);
7ef224d1
TZ		 537 if (ret < 0)
538 goto out;
76a3b0c8 539 key_offset += ret;
7ef224d1
TZ		 540 }
541 if (fields_str) {
542 ret = -EINVAL;
543 goto out;
544 }
545 ret = 0;
546 out:
547 return ret;
548}
549
550static int create_hist_fields(struct hist_trigger_data *hist_data,
551 struct trace_event_file *file)
552{
553 int ret;
554
555 ret = create_val_fields(hist_data, file);
556 if (ret)
557 goto out;
558
559 ret = create_key_fields(hist_data, file);
560 if (ret)
561 goto out;
562
563 hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
564 out:
565 return ret;
566}
567
e62347d2
TZ		 568static int is_descending(const char *str)
569{
570 if (!str)
571 return 0;
572
573 if (strcmp(str, "descending") == 0)
574 return 1;
575
576 if (strcmp(str, "ascending") == 0)
577 return 0;
578
579 return -EINVAL;
580}
581
7ef224d1
TZ		 582static int create_sort_keys(struct hist_trigger_data *hist_data)
583{
e62347d2
TZ		 584 char *fields_str = hist_data->attrs->sort_key_str;
585 struct ftrace_event_field *field = NULL;
586 struct tracing_map_sort_key *sort_key;
587 int descending, ret = 0;
588 unsigned int i, j;
589
590 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
591
592 if (!fields_str)
593 goto out;
594
595 strsep(&fields_str, "=");
596 if (!fields_str) {
597 ret = -EINVAL;
598 goto out;
599 }
600
601 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
602 char *field_str, *field_name;
603
604 sort_key = &hist_data->sort_keys[i];
605
606 field_str = strsep(&fields_str, ",");
607 if (!field_str) {
608 if (i == 0)
609 ret = -EINVAL;
610 break;
611 }
612
613 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
614 ret = -EINVAL;
615 break;
616 }
7ef224d1 617
e62347d2
TZ		 618 field_name = strsep(&field_str, ".");
619 if (!field_name) {
620 ret = -EINVAL;
621 break;
622 }
623
624 if (strcmp(field_name, "hitcount") == 0) {
625 descending = is_descending(field_str);
626 if (descending < 0) {
627 ret = descending;
628 break;
629 }
630 sort_key->descending = descending;
631 continue;
632 }
7ef224d1 633
e62347d2
TZ		 634 for (j = 1; j < hist_data->n_fields; j++) {
635 field = hist_data->fields[j]->field;
636 if (field && (strcmp(field_name, field->name) == 0)) {
637 sort_key->field_idx = j;
638 descending = is_descending(field_str);
639 if (descending < 0) {
640 ret = descending;
641 goto out;
642 }
643 sort_key->descending = descending;
644 break;
645 }
646 }
647 if (j == hist_data->n_fields) {
648 ret = -EINVAL;
649 break;
650 }
651 }
652 hist_data->n_sort_keys = i;
653 out:
7ef224d1
TZ		 654 return ret;
655}
656
657static void destroy_hist_data(struct hist_trigger_data *hist_data)
658{
659 destroy_hist_trigger_attrs(hist_data->attrs);
660 destroy_hist_fields(hist_data);
661 tracing_map_destroy(hist_data->map);
662 kfree(hist_data);
663}
664
665static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
666{
667 struct tracing_map *map = hist_data->map;
668 struct ftrace_event_field *field;
669 struct hist_field *hist_field;
670 unsigned int i, idx;
671
672 for_each_hist_field(i, hist_data) {
673 hist_field = hist_data->fields[i];
674 if (hist_field->flags & HIST_FIELD_FL_KEY) {
675 tracing_map_cmp_fn_t cmp_fn;
676
677 field = hist_field->field;
678
679 if (is_string_field(field))
680 cmp_fn = tracing_map_cmp_string;
681 else
682 cmp_fn = tracing_map_cmp_num(field->size,
683 field->is_signed);
76a3b0c8
TZ		 684 idx = tracing_map_add_key_field(map,
685 hist_field->offset,
686 cmp_fn);
687
7ef224d1
TZ		 688 } else
689 idx = tracing_map_add_sum_field(map);
690
691 if (idx < 0)
692 return idx;
693 }
694
695 return 0;
696}
697
6b4827ad
TZ		 698static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
699{
700 struct hist_field *key_field;
701 unsigned int i;
702
703 for_each_hist_key_field(i, hist_data) {
704 key_field = hist_data->fields[i];
705
706 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
707 return true;
708 }
709
710 return false;
711}
712
7ef224d1
TZ		 713static struct hist_trigger_data *
714create_hist_data(unsigned int map_bits,
715 struct hist_trigger_attrs *attrs,
716 struct trace_event_file *file)
717{
6b4827ad 718 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 719 struct hist_trigger_data *hist_data;
720 int ret = 0;
721
722 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
723 if (!hist_data)
724 return ERR_PTR(-ENOMEM);
725
726 hist_data->attrs = attrs;
727
728 ret = create_hist_fields(hist_data, file);
729 if (ret)
730 goto free;
731
732 ret = create_sort_keys(hist_data);
733 if (ret)
734 goto free;
735
6b4827ad
TZ		 736 if (need_tracing_map_ops(hist_data))
737 map_ops = &hist_trigger_elt_comm_ops;
738
7ef224d1 739 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 740 map_ops, hist_data);
7ef224d1
TZ		 741 if (IS_ERR(hist_data->map)) {
742 ret = PTR_ERR(hist_data->map);
743 hist_data->map = NULL;
744 goto free;
745 }
746
747 ret = create_tracing_map_fields(hist_data);
748 if (ret)
749 goto free;
750
751 ret = tracing_map_init(hist_data->map);
752 if (ret)
753 goto free;
754
755 hist_data->event_file = file;
756 out:
757 return hist_data;
758 free:
759 hist_data->attrs = NULL;
760
761 destroy_hist_data(hist_data);
762
763 hist_data = ERR_PTR(ret);
764
765 goto out;
766}
767
768static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
769 struct tracing_map_elt *elt,
770 void *rec)
771{
772 struct hist_field *hist_field;
773 unsigned int i;
774 u64 hist_val;
775
776 for_each_hist_val_field(i, hist_data) {
777 hist_field = hist_data->fields[i];
778 hist_val = hist_field->fn(hist_field, rec);
779 tracing_map_update_sum(elt, i, hist_val);
780 }
781}
782
783static void event_hist_trigger(struct event_trigger_data *data, void *rec)
784{
785 struct hist_trigger_data *hist_data = data->private_data;
76a3b0c8 786 char compound_key[HIST_KEY_SIZE_MAX];
7ef224d1
TZ		 787 struct hist_field *key_field;
788 struct tracing_map_elt *elt;
789 u64 field_contents;
790 void *key = NULL;
791 unsigned int i;
792
76a3b0c8
TZ		 793 if (hist_data->n_keys > 1)
794 memset(compound_key, 0, hist_data->key_size);
795
7ef224d1
TZ		 796 for_each_hist_key_field(i, hist_data) {
797 key_field = hist_data->fields[i];
798
799 field_contents = key_field->fn(key_field, rec);
800 if (key_field->flags & HIST_FIELD_FL_STRING)
801 key = (void *)(unsigned long)field_contents;
802 else
803 key = (void *)&field_contents;
76a3b0c8
TZ		 804
805 if (hist_data->n_keys > 1) {
806 memcpy(compound_key + key_field->offset, key,
807 key_field->size);
808 }
7ef224d1
TZ		 809 }
810
76a3b0c8
TZ		 811 if (hist_data->n_keys > 1)
812 key = compound_key;
813
7ef224d1
TZ		 814 elt = tracing_map_insert(hist_data->map, key);
815 if (elt)
816 hist_trigger_elt_update(hist_data, elt, rec);
817}
818
819static void
820hist_trigger_entry_print(struct seq_file *m,
821 struct hist_trigger_data *hist_data, void *key,
822 struct tracing_map_elt *elt)
823{
824 struct hist_field *key_field;
c6afad49 825 char str[KSYM_SYMBOL_LEN];
7ef224d1
TZ		 826 unsigned int i;
827 u64 uval;
828
829 seq_puts(m, "{ ");
830
831 for_each_hist_key_field(i, hist_data) {
832 key_field = hist_data->fields[i];
833
834 if (i > hist_data->n_vals)
835 seq_puts(m, ", ");
836
0c4a6b46
TZ		 837 if (key_field->flags & HIST_FIELD_FL_HEX) {
838 uval = *(u64 *)(key + key_field->offset);
839 seq_printf(m, "%s: %llx",
840 key_field->field->name, uval);
c6afad49
TZ		 841 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
842 uval = *(u64 *)(key + key_field->offset);
843 sprint_symbol_no_offset(str, uval);
844 seq_printf(m, "%s: [%llx] %-45s",
845 key_field->field->name, uval, str);
846 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
847 uval = *(u64 *)(key + key_field->offset);
848 sprint_symbol(str, uval);
849 seq_printf(m, "%s: [%llx] %-55s",
850 key_field->field->name, uval, str);
6b4827ad
TZ		 851 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
852 char *comm = elt->private_data;
853
854 uval = *(u64 *)(key + key_field->offset);
855 seq_printf(m, "%s: %-16s[%10llu]",
856 key_field->field->name, comm, uval);
0c4a6b46 857 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
7ef224d1 858 seq_printf(m, "%s: %-50s", key_field->field->name,
76a3b0c8 859 (char *)(key + key_field->offset));
7ef224d1 860 } else {
76a3b0c8
TZ		 861 uval = *(u64 *)(key + key_field->offset);
862 seq_printf(m, "%s: %10llu", key_field->field->name,
863 uval);
7ef224d1
TZ		 864 }
865 }
866
867 seq_puts(m, " }");
868
869 seq_printf(m, " hitcount: %10llu",
870 tracing_map_read_sum(elt, HITCOUNT_IDX));
871
f2606835 872 for (i = 1; i < hist_data->n_vals; i++) {
0c4a6b46
TZ		 873 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
874 seq_printf(m, " %s: %10llx",
875 hist_data->fields[i]->field->name,
876 tracing_map_read_sum(elt, i));
877 } else {
878 seq_printf(m, " %s: %10llu",
879 hist_data->fields[i]->field->name,
880 tracing_map_read_sum(elt, i));
881 }
f2606835
TZ		 882 }
883
7ef224d1
TZ		 884 seq_puts(m, "\n");
885}
886
887static int print_entries(struct seq_file *m,
888 struct hist_trigger_data *hist_data)
889{
890 struct tracing_map_sort_entry **sort_entries = NULL;
891 struct tracing_map *map = hist_data->map;
892 unsigned int i, n_entries;
893
894 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
895 hist_data->n_sort_keys,
896 &sort_entries);
897 if (n_entries < 0)
898 return n_entries;
899
900 for (i = 0; i < n_entries; i++)
901 hist_trigger_entry_print(m, hist_data,
902 sort_entries[i]->key,
903 sort_entries[i]->elt);
904
905 tracing_map_destroy_sort_entries(sort_entries, n_entries);
906
907 return n_entries;
908}
909
910static int hist_show(struct seq_file *m, void *v)
911{
912 struct event_trigger_data *test, *data = NULL;
913 struct trace_event_file *event_file;
914 struct hist_trigger_data *hist_data;
915 int n_entries, ret = 0;
916
917 mutex_lock(&event_mutex);
918
919 event_file = event_file_data(m->private);
920 if (unlikely(!event_file)) {
921 ret = -ENODEV;
922 goto out_unlock;
923 }
924
925 list_for_each_entry_rcu(test, &event_file->triggers, list) {
926 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
927 data = test;
928 break;
929 }
930 }
931 if (!data)
932 goto out_unlock;
933
934 seq_puts(m, "# event histogram\n#\n# trigger info: ");
935 data->ops->print(m, data->ops, data);
936 seq_puts(m, "\n");
937
938 hist_data = data->private_data;
939 n_entries = print_entries(m, hist_data);
940 if (n_entries < 0) {
941 ret = n_entries;
942 n_entries = 0;
943 }
944
945 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
946 (u64)atomic64_read(&hist_data->map->hits),
947 n_entries, (u64)atomic64_read(&hist_data->map->drops));
948 out_unlock:
949 mutex_unlock(&event_mutex);
950
951 return ret;
952}
953
954static int event_hist_open(struct inode *inode, struct file *file)
955{
956 return single_open(file, hist_show, file);
957}
958
959const struct file_operations event_hist_fops = {
960 .open = event_hist_open,
961 .read = seq_read,
962 .llseek = seq_lseek,
963 .release = single_release,
964};
965
0c4a6b46
TZ		 966static const char *get_hist_field_flags(struct hist_field *hist_field)
967{
968 const char *flags_str = NULL;
969
970 if (hist_field->flags & HIST_FIELD_FL_HEX)
971 flags_str = "hex";
c6afad49
TZ		 972 else if (hist_field->flags & HIST_FIELD_FL_SYM)
973 flags_str = "sym";
974 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
975 flags_str = "sym-offset";
6b4827ad
TZ		 976 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
977 flags_str = "execname";
0c4a6b46
TZ		 978
979 return flags_str;
980}
981
7ef224d1
TZ		 982static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
983{
984 seq_printf(m, "%s", hist_field->field->name);
0c4a6b46
TZ		 985 if (hist_field->flags) {
986 const char *flags_str = get_hist_field_flags(hist_field);
987
988 if (flags_str)
989 seq_printf(m, ".%s", flags_str);
990 }
7ef224d1
TZ		 991}
992
993static int event_hist_trigger_print(struct seq_file *m,
994 struct event_trigger_ops *ops,
995 struct event_trigger_data *data)
996{
997 struct hist_trigger_data *hist_data = data->private_data;
998 struct hist_field *key_field;
999 unsigned int i;
1000
1001 seq_puts(m, "hist:keys=");
1002
1003 for_each_hist_key_field(i, hist_data) {
1004 key_field = hist_data->fields[i];
1005
1006 if (i > hist_data->n_vals)
1007 seq_puts(m, ",");
1008
1009 hist_field_print(m, key_field);
1010 }
1011
1012 seq_puts(m, ":vals=");
f2606835
TZ		 1013
1014 for_each_hist_val_field(i, hist_data) {
1015 if (i == HITCOUNT_IDX)
1016 seq_puts(m, "hitcount");
1017 else {
1018 seq_puts(m, ",");
1019 hist_field_print(m, hist_data->fields[i]);
1020 }
1021 }
7ef224d1
TZ		 1022
1023 seq_puts(m, ":sort=");
e62347d2
TZ		 1024
1025 for (i = 0; i < hist_data->n_sort_keys; i++) {
1026 struct tracing_map_sort_key *sort_key;
1027
1028 sort_key = &hist_data->sort_keys[i];
1029
1030 if (i > 0)
1031 seq_puts(m, ",");
1032
1033 if (sort_key->field_idx == HITCOUNT_IDX)
1034 seq_puts(m, "hitcount");
1035 else {
1036 unsigned int idx = sort_key->field_idx;
1037
1038 if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1039 return -EINVAL;
1040
1041 hist_field_print(m, hist_data->fields[idx]);
1042 }
1043
1044 if (sort_key->descending)
1045 seq_puts(m, ".descending");
1046 }
7ef224d1
TZ		 1047
1048 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1049
1050 if (data->filter_str)
1051 seq_printf(m, " if %s", data->filter_str);
1052
83e99914
TZ		 1053 if (data->paused)
1054 seq_puts(m, " [paused]");
1055 else
1056 seq_puts(m, " [active]");
7ef224d1
TZ		 1057
1058 seq_putc(m, '\n');
1059
1060 return 0;
1061}
1062
1063static void event_hist_trigger_free(struct event_trigger_ops *ops,
1064 struct event_trigger_data *data)
1065{
1066 struct hist_trigger_data *hist_data = data->private_data;
1067
1068 if (WARN_ON_ONCE(data->ref <= 0))
1069 return;
1070
1071 data->ref--;
1072 if (!data->ref) {
1073 trigger_data_free(data);
1074 destroy_hist_data(hist_data);
1075 }
1076}
1077
1078static struct event_trigger_ops event_hist_trigger_ops = {
1079 .func = event_hist_trigger,
1080 .print = event_hist_trigger_print,
1081 .init = event_trigger_init,
1082 .free = event_hist_trigger_free,
1083};
1084
1085static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1086 char *param)
1087{
1088 return &event_hist_trigger_ops;
1089}
1090
e86ae9ba
TZ		 1091static void hist_clear(struct event_trigger_data *data)
1092{
1093 struct hist_trigger_data *hist_data = data->private_data;
1094 bool paused;
1095
1096 paused = data->paused;
1097 data->paused = true;
1098
1099 synchronize_sched();
1100
1101 tracing_map_clear(hist_data->map);
1102
1103 data->paused = paused;
1104}
1105
7ef224d1
TZ		 1106static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1107 struct event_trigger_data *data,
1108 struct trace_event_file *file)
1109{
83e99914 1110 struct hist_trigger_data *hist_data = data->private_data;
7ef224d1
TZ		 1111 struct event_trigger_data *test;
1112 int ret = 0;
1113
1114 list_for_each_entry_rcu(test, &file->triggers, list) {
1115 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
83e99914
TZ		 1116 if (hist_data->attrs->pause)
1117 test->paused = true;
1118 else if (hist_data->attrs->cont)
1119 test->paused = false;
e86ae9ba
TZ		 1120 else if (hist_data->attrs->clear)
1121 hist_clear(test);
83e99914
TZ		 1122 else
1123 ret = -EEXIST;
7ef224d1
TZ		 1124 goto out;
1125 }
1126 }
1127
e86ae9ba 1128 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 1129 ret = -ENOENT;
1130 goto out;
1131 }
1132
1133 if (hist_data->attrs->pause)
1134 data->paused = true;
1135
7ef224d1
TZ		 1136 if (data->ops->init) {
1137 ret = data->ops->init(data->ops, data);
1138 if (ret < 0)
1139 goto out;
1140 }
1141
1142 list_add_rcu(&data->list, &file->triggers);
1143 ret++;
1144
1145 update_cond_flag(file);
1146 if (trace_event_trigger_enable_disable(file, 1) < 0) {
1147 list_del_rcu(&data->list);
1148 update_cond_flag(file);
1149 ret--;
1150 }
1151 out:
1152 return ret;
1153}
1154
1155static int event_hist_trigger_func(struct event_command *cmd_ops,
1156 struct trace_event_file *file,
1157 char *glob, char *cmd, char *param)
1158{
1159 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1160 struct event_trigger_data *trigger_data;
1161 struct hist_trigger_attrs *attrs;
1162 struct event_trigger_ops *trigger_ops;
1163 struct hist_trigger_data *hist_data;
1164 char *trigger;
1165 int ret = 0;
1166
1167 if (!param)
1168 return -EINVAL;
1169
1170 /* separate the trigger from the filter (k:v [if filter]) */
1171 trigger = strsep(&param, " \t");
1172 if (!trigger)
1173 return -EINVAL;
1174
1175 attrs = parse_hist_trigger_attrs(trigger);
1176 if (IS_ERR(attrs))
1177 return PTR_ERR(attrs);
1178
1179 if (attrs->map_bits)
1180 hist_trigger_bits = attrs->map_bits;
1181
1182 hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1183 if (IS_ERR(hist_data)) {
1184 destroy_hist_trigger_attrs(attrs);
1185 return PTR_ERR(hist_data);
1186 }
1187
1188 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1189
1190 ret = -ENOMEM;
1191 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1192 if (!trigger_data)
1193 goto out_free;
1194
1195 trigger_data->count = -1;
1196 trigger_data->ops = trigger_ops;
1197 trigger_data->cmd_ops = cmd_ops;
1198
1199 INIT_LIST_HEAD(&trigger_data->list);
1200 RCU_INIT_POINTER(trigger_data->filter, NULL);
1201
1202 trigger_data->private_data = hist_data;
1203
1204 if (glob[0] == '!') {
1205 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1206 ret = 0;
1207 goto out_free;
1208 }
1209
1210 if (!param) /* if param is non-empty, it's supposed to be a filter */
1211 goto out_reg;
1212
1213 if (!cmd_ops->set_filter)
1214 goto out_reg;
1215
1216 ret = cmd_ops->set_filter(param, trigger_data, file);
1217 if (ret < 0)
1218 goto out_free;
1219 out_reg:
1220 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1221 /*
1222 * The above returns on success the # of triggers registered,
1223 * but if it didn't register any it returns zero. Consider no
1224 * triggers registered a failure too.
1225 */
1226 if (!ret) {
e86ae9ba 1227 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 1228 ret = -ENOENT;
7ef224d1
TZ		 1229 goto out_free;
1230 } else if (ret < 0)
1231 goto out_free;
1232 /* Just return zero, not the number of registered triggers */
1233 ret = 0;
1234 out:
1235 return ret;
1236 out_free:
1237 if (cmd_ops->set_filter)
1238 cmd_ops->set_filter(NULL, trigger_data, NULL);
1239
1240 kfree(trigger_data);
1241
1242 destroy_hist_data(hist_data);
1243 goto out;
1244}
1245
1246static struct event_command trigger_hist_cmd = {
1247 .name = "hist",
1248 .trigger_type = ETT_EVENT_HIST,
1249 .flags = EVENT_CMD_FL_NEEDS_REC,
1250 .func = event_hist_trigger_func,
1251 .reg = hist_register_trigger,
1252 .unreg = unregister_trigger,
1253 .get_trigger_ops = event_hist_get_trigger_ops,
1254 .set_filter = set_trigger_filter,
1255};
1256
1257__init int register_trigger_hist_cmd(void)
1258{
1259 int ret;
1260
1261 ret = register_event_command(&trigger_hist_cmd);
1262 WARN_ON(ret < 0);
1263
1264 return ret;
1265}
Linux 6.x block layer and io_uring tree(s)