projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Account for variables in named trigger compatibility
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
b2d09103 22#include <linux/rculist.h>
7ef224d1
TZ		 23
24#include "tracing_map.h"
25#include "trace.h"
26
27struct hist_field;
28
fbd302cb
TZ		 29typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event,
30 struct ring_buffer_event *rbe);
7ef224d1 31
5819eadd 32#define HIST_FIELD_OPERANDS_MAX 2
30350d65
TZ		 33#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
34
35struct hist_var {
36 char *name;
37 struct hist_trigger_data *hist_data;
38 unsigned int idx;
39};
5819eadd 40
7ef224d1
TZ		 41struct hist_field {
42 struct ftrace_event_field *field;
43 unsigned long flags;
44 hist_field_fn_t fn;
45 unsigned int size;
76a3b0c8 46 unsigned int offset;
5819eadd
TZ		 47 unsigned int is_signed;
48 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 49 struct hist_trigger_data *hist_data;
30350d65 50 struct hist_var var;
7ef224d1
TZ		 51};
52
fbd302cb
TZ		 53static u64 hist_field_none(struct hist_field *field, void *event,
54 struct ring_buffer_event *rbe)
69a0200c
TZ		 55{
56 return 0;
57}
58
fbd302cb
TZ		 59static u64 hist_field_counter(struct hist_field *field, void *event,
60 struct ring_buffer_event *rbe)
7ef224d1
TZ		 61{
62 return 1;
63}
64
fbd302cb
TZ		 65static u64 hist_field_string(struct hist_field *hist_field, void *event,
66 struct ring_buffer_event *rbe)
7ef224d1
TZ		 67{
68 char *addr = (char *)(event + hist_field->field->offset);
69
70 return (u64)(unsigned long)addr;
71}
72
fbd302cb
TZ		 73static u64 hist_field_dynstring(struct hist_field *hist_field, void *event,
74 struct ring_buffer_event *rbe)
79e577cb
NK		 75{
76 u32 str_item = *(u32 *)(event + hist_field->field->offset);
77 int str_loc = str_item & 0xffff;
78 char *addr = (char *)(event + str_loc);
79
80 return (u64)(unsigned long)addr;
81}
82
fbd302cb
TZ		 83static u64 hist_field_pstring(struct hist_field *hist_field, void *event,
84 struct ring_buffer_event *rbe)
79e577cb
NK		 85{
86 char **addr = (char **)(event + hist_field->field->offset);
87
88 return (u64)(unsigned long)*addr;
89}
90
fbd302cb
TZ		 91static u64 hist_field_log2(struct hist_field *hist_field, void *event,
92 struct ring_buffer_event *rbe)
4b94f5b7 93{
5819eadd
TZ		 94 struct hist_field *operand = hist_field->operands[0];
95
fbd302cb 96 u64 val = operand->fn(operand, event, rbe);
4b94f5b7
NK		 97
98 return (u64) ilog2(roundup_pow_of_two(val));
99}
100
7ef224d1 101#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb
TZ		 102 static u64 hist_field_##type(struct hist_field *hist_field, \
103 void *event, \
104 struct ring_buffer_event *rbe) \
7ef224d1
TZ		 105{ \
106 type *addr = (type *)(event + hist_field->field->offset); \
107 \
79e577cb 108 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 109}
110
111DEFINE_HIST_FIELD_FN(s64);
112DEFINE_HIST_FIELD_FN(u64);
113DEFINE_HIST_FIELD_FN(s32);
114DEFINE_HIST_FIELD_FN(u32);
115DEFINE_HIST_FIELD_FN(s16);
116DEFINE_HIST_FIELD_FN(u16);
117DEFINE_HIST_FIELD_FN(s8);
118DEFINE_HIST_FIELD_FN(u8);
119
120#define for_each_hist_field(i, hist_data) \
121 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
122
123#define for_each_hist_val_field(i, hist_data) \
124 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
125
126#define for_each_hist_key_field(i, hist_data) \
127 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
128
69a0200c
TZ		 129#define HIST_STACKTRACE_DEPTH 16
130#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
131#define HIST_STACKTRACE_SKIP 5
132
7ef224d1 133#define HITCOUNT_IDX 0
69a0200c 134#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 135
136enum hist_field_flags {
0d7a8325
TZ		 137 HIST_FIELD_FL_HITCOUNT = 1 << 0,
138 HIST_FIELD_FL_KEY = 1 << 1,
139 HIST_FIELD_FL_STRING = 1 << 2,
140 HIST_FIELD_FL_HEX = 1 << 3,
141 HIST_FIELD_FL_SYM = 1 << 4,
142 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
143 HIST_FIELD_FL_EXECNAME = 1 << 6,
144 HIST_FIELD_FL_SYSCALL = 1 << 7,
145 HIST_FIELD_FL_STACKTRACE = 1 << 8,
146 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 147 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 148 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65
TZ		 149 HIST_FIELD_FL_VAR = 1 << 12,
150};
151
152struct var_defs {
153 unsigned int n_vars;
154 char *name[TRACING_MAP_VARS_MAX];
155 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 156};
157
158struct hist_trigger_attrs {
159 char *keys_str;
f2606835 160 char *vals_str;
e62347d2 161 char *sort_key_str;
5463bfda 162 char *name;
83e99914
TZ		 163 bool pause;
164 bool cont;
e86ae9ba 165 bool clear;
860f9f6b 166 bool ts_in_usecs;
7ef224d1 167 unsigned int map_bits;
30350d65
TZ		 168
169 char *assignment_str[TRACING_MAP_VARS_MAX];
170 unsigned int n_assignments;
171
172 struct var_defs var_defs;
7ef224d1
TZ		 173};
174
175struct hist_trigger_data {
30350d65 176 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 177 unsigned int n_vals;
178 unsigned int n_keys;
179 unsigned int n_fields;
30350d65 180 unsigned int n_vars;
7ef224d1
TZ		 181 unsigned int key_size;
182 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
183 unsigned int n_sort_keys;
184 struct trace_event_file *event_file;
185 struct hist_trigger_attrs *attrs;
186 struct tracing_map *map;
ad42febe 187 bool enable_timestamps;
30350d65 188 bool remove;
7ef224d1
TZ		 189};
190
860f9f6b
TZ		 191static u64 hist_field_timestamp(struct hist_field *hist_field, void *event,
192 struct ring_buffer_event *rbe)
193{
194 struct hist_trigger_data *hist_data = hist_field->hist_data;
195 struct trace_array *tr = hist_data->event_file->tr;
196
197 u64 ts = ring_buffer_event_time_stamp(rbe);
198
199 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
200 ts = ns2usecs(ts);
201
202 return ts;
203}
204
30350d65
TZ		 205static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
206 const char *var_name)
207{
208 struct hist_field *hist_field, *found = NULL;
209 int i;
210
211 for_each_hist_field(i, hist_data) {
212 hist_field = hist_data->fields[i];
213 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
214 strcmp(hist_field->var.name, var_name) == 0) {
215 found = hist_field;
216 break;
217 }
218 }
219
220 return found;
221}
222
223static struct hist_field *find_var(struct hist_trigger_data *hist_data,
224 struct trace_event_file *file,
225 const char *var_name)
226{
227 struct hist_trigger_data *test_data;
228 struct event_trigger_data *test;
229 struct hist_field *hist_field;
230
231 hist_field = find_var_field(hist_data, var_name);
232 if (hist_field)
233 return hist_field;
234
235 list_for_each_entry_rcu(test, &file->triggers, list) {
236 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
237 test_data = test->private_data;
238 hist_field = find_var_field(test_data, var_name);
239 if (hist_field)
240 return hist_field;
241 }
242 }
243
244 return NULL;
245}
246
85013256
TZ		 247static const char *hist_field_name(struct hist_field *field,
248 unsigned int level)
249{
250 const char *field_name = "";
251
252 if (level > 1)
253 return field_name;
254
255 if (field->field)
256 field_name = field->field->name;
5819eadd
TZ		 257 else if (field->flags & HIST_FIELD_FL_LOG2)
258 field_name = hist_field_name(field->operands[0], ++level);
ad42febe
TZ		 259 else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
260 field_name = "common_timestamp";
85013256
TZ		 261
262 if (field_name == NULL)
263 field_name = "";
264
265 return field_name;
266}
267
7ef224d1
TZ		 268static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
269{
270 hist_field_fn_t fn = NULL;
271
272 switch (field_size) {
273 case 8:
274 if (field_is_signed)
275 fn = hist_field_s64;
276 else
277 fn = hist_field_u64;
278 break;
279 case 4:
280 if (field_is_signed)
281 fn = hist_field_s32;
282 else
283 fn = hist_field_u32;
284 break;
285 case 2:
286 if (field_is_signed)
287 fn = hist_field_s16;
288 else
289 fn = hist_field_u16;
290 break;
291 case 1:
292 if (field_is_signed)
293 fn = hist_field_s8;
294 else
295 fn = hist_field_u8;
296 break;
297 }
298
299 return fn;
300}
301
302static int parse_map_size(char *str)
303{
304 unsigned long size, map_bits;
305 int ret;
306
307 strsep(&str, "=");
308 if (!str) {
309 ret = -EINVAL;
310 goto out;
311 }
312
313 ret = kstrtoul(str, 0, &size);
314 if (ret)
315 goto out;
316
317 map_bits = ilog2(roundup_pow_of_two(size));
318 if (map_bits < TRACING_MAP_BITS_MIN ||
319 map_bits > TRACING_MAP_BITS_MAX)
320 ret = -EINVAL;
321 else
322 ret = map_bits;
323 out:
324 return ret;
325}
326
327static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
328{
30350d65
TZ		 329 unsigned int i;
330
7ef224d1
TZ		 331 if (!attrs)
332 return;
333
30350d65
TZ		 334 for (i = 0; i < attrs->n_assignments; i++)
335 kfree(attrs->assignment_str[i]);
336
5463bfda 337 kfree(attrs->name);
e62347d2 338 kfree(attrs->sort_key_str);
7ef224d1 339 kfree(attrs->keys_str);
f2606835 340 kfree(attrs->vals_str);
7ef224d1
TZ		 341 kfree(attrs);
342}
343
9b1ae035
TZ		 344static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
345{
346 int ret = 0;
347
348 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
349 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
350 attrs->keys_str = kstrdup(str, GFP_KERNEL);
351 if (!attrs->keys_str) {
352 ret = -ENOMEM;
353 goto out;
354 }
355 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
356 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
357 (strncmp(str, "values=", strlen("values=")) == 0)) {
358 attrs->vals_str = kstrdup(str, GFP_KERNEL);
359 if (!attrs->vals_str) {
360 ret = -ENOMEM;
361 goto out;
362 }
363 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
364 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
365 if (!attrs->sort_key_str) {
366 ret = -ENOMEM;
367 goto out;
368 }
369 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
370 attrs->name = kstrdup(str, GFP_KERNEL);
371 if (!attrs->name) {
372 ret = -ENOMEM;
373 goto out;
374 }
375 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
376 int map_bits = parse_map_size(str);
377
378 if (map_bits < 0) {
379 ret = map_bits;
380 goto out;
381 }
382 attrs->map_bits = map_bits;
30350d65
TZ		 383 } else {
384 char *assignment;
385
386 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
387 ret = -EINVAL;
388 goto out;
389 }
390
391 assignment = kstrdup(str, GFP_KERNEL);
392 if (!assignment) {
393 ret = -ENOMEM;
394 goto out;
395 }
396
397 attrs->assignment_str[attrs->n_assignments++] = assignment;
398 }
9b1ae035
TZ		 399 out:
400 return ret;
401}
402
7ef224d1
TZ		 403static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
404{
405 struct hist_trigger_attrs *attrs;
406 int ret = 0;
407
408 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
409 if (!attrs)
410 return ERR_PTR(-ENOMEM);
411
412 while (trigger_str) {
413 char *str = strsep(&trigger_str, ":");
414
9b1ae035
TZ		 415 if (strchr(str, '=')) {
416 ret = parse_assignment(str, attrs);
417 if (ret)
418 goto free;
419 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 420 attrs->pause = true;
421 else if ((strcmp(str, "cont") == 0) ||
422 (strcmp(str, "continue") == 0))
423 attrs->cont = true;
e86ae9ba
TZ		 424 else if (strcmp(str, "clear") == 0)
425 attrs->clear = true;
9b1ae035 426 else {
7ef224d1
TZ		 427 ret = -EINVAL;
428 goto free;
429 }
430 }
431
432 if (!attrs->keys_str) {
433 ret = -EINVAL;
434 goto free;
435 }
436
437 return attrs;
438 free:
439 destroy_hist_trigger_attrs(attrs);
440
441 return ERR_PTR(ret);
442}
443
6b4827ad
TZ		 444static inline void save_comm(char *comm, struct task_struct *task)
445{
446 if (!task->pid) {
447 strcpy(comm, "<idle>");
448 return;
449 }
450
451 if (WARN_ON_ONCE(task->pid < 0)) {
452 strcpy(comm, "<XXX>");
453 return;
454 }
455
456 memcpy(comm, task->comm, TASK_COMM_LEN);
457}
458
459static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
460{
461 kfree((char *)elt->private_data);
462}
463
464static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
465{
466 struct hist_trigger_data *hist_data = elt->map->private_data;
467 struct hist_field *key_field;
468 unsigned int i;
469
470 for_each_hist_key_field(i, hist_data) {
471 key_field = hist_data->fields[i];
472
473 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
474 unsigned int size = TASK_COMM_LEN + 1;
475
476 elt->private_data = kzalloc(size, GFP_KERNEL);
477 if (!elt->private_data)
478 return -ENOMEM;
479 break;
480 }
481 }
482
483 return 0;
484}
485
6b4827ad
TZ		 486static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
487{
488 char *comm = elt->private_data;
489
490 if (comm)
491 save_comm(comm, current);
492}
493
494static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
495 .elt_alloc = hist_trigger_elt_comm_alloc,
6b4827ad
TZ		 496 .elt_free = hist_trigger_elt_comm_free,
497 .elt_init = hist_trigger_elt_comm_init,
498};
499
5819eadd
TZ		 500static void destroy_hist_field(struct hist_field *hist_field,
501 unsigned int level)
7ef224d1 502{
5819eadd
TZ		 503 unsigned int i;
504
505 if (level > 2)
506 return;
507
508 if (!hist_field)
509 return;
510
511 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
512 destroy_hist_field(hist_field->operands[i], level + 1);
513
30350d65
TZ		 514 kfree(hist_field->var.name);
515
7ef224d1
TZ		 516 kfree(hist_field);
517}
518
b559d003
TZ		 519static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
520 struct ftrace_event_field *field,
30350d65
TZ		 521 unsigned long flags,
522 char *var_name)
7ef224d1
TZ		 523{
524 struct hist_field *hist_field;
525
526 if (field && is_function_field(field))
527 return NULL;
528
529 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
530 if (!hist_field)
531 return NULL;
532
b559d003
TZ		 533 hist_field->hist_data = hist_data;
534
7ef224d1
TZ		 535 if (flags & HIST_FIELD_FL_HITCOUNT) {
536 hist_field->fn = hist_field_counter;
537 goto out;
538 }
539
69a0200c
TZ		 540 if (flags & HIST_FIELD_FL_STACKTRACE) {
541 hist_field->fn = hist_field_none;
542 goto out;
543 }
544
4b94f5b7 545 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 546 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 547 hist_field->fn = hist_field_log2;
30350d65 548 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 549 hist_field->size = hist_field->operands[0]->size;
4b94f5b7
NK		 550 goto out;
551 }
552
ad42febe
TZ		 553 if (flags & HIST_FIELD_FL_TIMESTAMP) {
554 hist_field->fn = hist_field_timestamp;
555 hist_field->size = sizeof(u64);
556 goto out;
557 }
558
432480c5
TZ		 559 if (WARN_ON_ONCE(!field))
560 goto out;
561
7ef224d1
TZ		 562 if (is_string_field(field)) {
563 flags |= HIST_FIELD_FL_STRING;
79e577cb
NK		 564
565 if (field->filter_type == FILTER_STATIC_STRING)
566 hist_field->fn = hist_field_string;
567 else if (field->filter_type == FILTER_DYN_STRING)
568 hist_field->fn = hist_field_dynstring;
569 else
570 hist_field->fn = hist_field_pstring;
7ef224d1
TZ		 571 } else {
572 hist_field->fn = select_value_fn(field->size,
573 field->is_signed);
574 if (!hist_field->fn) {
5819eadd 575 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 576 return NULL;
577 }
578 }
579 out:
580 hist_field->field = field;
581 hist_field->flags = flags;
582
30350d65
TZ		 583 if (var_name) {
584 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
585 if (!hist_field->var.name)
586 goto free;
587 }
588
7ef224d1 589 return hist_field;
30350d65
TZ		 590 free:
591 destroy_hist_field(hist_field, 0);
592 return NULL;
7ef224d1
TZ		 593}
594
595static void destroy_hist_fields(struct hist_trigger_data *hist_data)
596{
597 unsigned int i;
598
30350d65 599 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 600 if (hist_data->fields[i]) {
5819eadd 601 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 602 hist_data->fields[i] = NULL;
603 }
604 }
605}
606
607static int create_hitcount_val(struct hist_trigger_data *hist_data)
608{
609 hist_data->fields[HITCOUNT_IDX] =
30350d65 610 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 611 if (!hist_data->fields[HITCOUNT_IDX])
612 return -ENOMEM;
613
614 hist_data->n_vals++;
30350d65 615 hist_data->n_fields++;
7ef224d1
TZ		 616
617 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
618 return -EINVAL;
619
620 return 0;
621}
622
30350d65
TZ		 623static int __create_val_field(struct hist_trigger_data *hist_data,
624 unsigned int val_idx,
625 struct trace_event_file *file,
626 char *var_name, char *field_str,
627 unsigned long flags)
f2606835
TZ		 628{
629 struct ftrace_event_field *field = NULL;
0c4a6b46 630 char *field_name;
f2606835
TZ		 631 int ret = 0;
632
0c4a6b46
TZ		 633 field_name = strsep(&field_str, ".");
634 if (field_str) {
635 if (strcmp(field_str, "hex") == 0)
636 flags |= HIST_FIELD_FL_HEX;
637 else {
638 ret = -EINVAL;
639 goto out;
640 }
641 }
642
ad42febe
TZ		 643 if (strcmp(field_name, "common_timestamp") == 0) {
644 flags |= HIST_FIELD_FL_TIMESTAMP;
645 hist_data->enable_timestamps = true;
646 } else {
647 field = trace_find_event_field(file->event_call, field_name);
648 if (!field || !field->size) {
649 ret = -EINVAL;
650 goto out;
651 }
f2606835
TZ		 652 }
653
30350d65 654 hist_data->fields[val_idx] = create_hist_field(hist_data, field, flags, var_name);
f2606835
TZ		 655 if (!hist_data->fields[val_idx]) {
656 ret = -ENOMEM;
657 goto out;
658 }
659
660 ++hist_data->n_vals;
30350d65 661 ++hist_data->n_fields;
f2606835 662
30350d65 663 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 664 ret = -EINVAL;
665 out:
666 return ret;
667}
668
30350d65
TZ		 669static int create_val_field(struct hist_trigger_data *hist_data,
670 unsigned int val_idx,
671 struct trace_event_file *file,
672 char *field_str)
673{
674 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
675 return -EINVAL;
676
677 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
678}
679
680static int create_var_field(struct hist_trigger_data *hist_data,
681 unsigned int val_idx,
682 struct trace_event_file *file,
683 char *var_name, char *expr_str)
684{
685 unsigned long flags = 0;
686
687 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
688 return -EINVAL;
689 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
690 return -EINVAL;
691 }
692
693 flags |= HIST_FIELD_FL_VAR;
694 hist_data->n_vars++;
695 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
696 return -EINVAL;
697
698 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
699}
700
7ef224d1
TZ		 701static int create_val_fields(struct hist_trigger_data *hist_data,
702 struct trace_event_file *file)
703{
f2606835 704 char *fields_str, *field_str;
30350d65 705 unsigned int i, j = 1;
7ef224d1
TZ		 706 int ret;
707
708 ret = create_hitcount_val(hist_data);
f2606835
TZ		 709 if (ret)
710 goto out;
7ef224d1 711
f2606835
TZ		 712 fields_str = hist_data->attrs->vals_str;
713 if (!fields_str)
714 goto out;
715
716 strsep(&fields_str, "=");
717 if (!fields_str)
718 goto out;
719
720 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
721 j < TRACING_MAP_VALS_MAX; i++) {
722 field_str = strsep(&fields_str, ",");
723 if (!field_str)
724 break;
30350d65 725
f2606835
TZ		 726 if (strcmp(field_str, "hitcount") == 0)
727 continue;
30350d65 728
f2606835
TZ		 729 ret = create_val_field(hist_data, j++, file, field_str);
730 if (ret)
731 goto out;
732 }
30350d65 733
f2606835
TZ		 734 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
735 ret = -EINVAL;
736 out:
7ef224d1
TZ		 737 return ret;
738}
739
740static int create_key_field(struct hist_trigger_data *hist_data,
741 unsigned int key_idx,
76a3b0c8 742 unsigned int key_offset,
7ef224d1
TZ		 743 struct trace_event_file *file,
744 char *field_str)
745{
746 struct ftrace_event_field *field = NULL;
30350d65 747 struct hist_field *hist_field = NULL;
7ef224d1
TZ		 748 unsigned long flags = 0;
749 unsigned int key_size;
750 int ret = 0;
751
30350d65 752 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 753 return -EINVAL;
754
755 flags |= HIST_FIELD_FL_KEY;
756
69a0200c
TZ		 757 if (strcmp(field_str, "stacktrace") == 0) {
758 flags |= HIST_FIELD_FL_STACKTRACE;
759 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 760 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c
TZ		 761 } else {
762 char *field_name = strsep(&field_str, ".");
763
764 if (field_str) {
765 if (strcmp(field_str, "hex") == 0)
766 flags |= HIST_FIELD_FL_HEX;
767 else if (strcmp(field_str, "sym") == 0)
768 flags |= HIST_FIELD_FL_SYM;
769 else if (strcmp(field_str, "sym-offset") == 0)
770 flags |= HIST_FIELD_FL_SYM_OFFSET;
771 else if ((strcmp(field_str, "execname") == 0) &&
772 (strcmp(field_name, "common_pid") == 0))
773 flags |= HIST_FIELD_FL_EXECNAME;
774 else if (strcmp(field_str, "syscall") == 0)
775 flags |= HIST_FIELD_FL_SYSCALL;
4b94f5b7
NK		 776 else if (strcmp(field_str, "log2") == 0)
777 flags |= HIST_FIELD_FL_LOG2;
860f9f6b
TZ		 778 else if (strcmp(field_str, "usecs") == 0)
779 flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
69a0200c
TZ		 780 else {
781 ret = -EINVAL;
782 goto out;
783 }
784 }
785
ad42febe
TZ		 786 if (strcmp(field_name, "common_timestamp") == 0) {
787 flags |= HIST_FIELD_FL_TIMESTAMP;
788 hist_data->enable_timestamps = true;
860f9f6b
TZ		 789 if (flags & HIST_FIELD_FL_TIMESTAMP_USECS)
790 hist_data->attrs->ts_in_usecs = true;
ad42febe
TZ		 791 key_size = sizeof(u64);
792 } else {
793 field = trace_find_event_field(file->event_call, field_name);
794 if (!field || !field->size) {
795 ret = -EINVAL;
796 goto out;
797 }
0c4a6b46 798
ad42febe
TZ		 799 if (is_string_field(field))
800 key_size = MAX_FILTER_STR_VAL;
801 else
802 key_size = field->size;
803 }
7ef224d1
TZ		 804 }
805
30350d65 806 hist_data->fields[key_idx] = create_hist_field(hist_data, field, flags, NULL);
7ef224d1
TZ		 807 if (!hist_data->fields[key_idx]) {
808 ret = -ENOMEM;
809 goto out;
810 }
811
812 key_size = ALIGN(key_size, sizeof(u64));
813 hist_data->fields[key_idx]->size = key_size;
76a3b0c8
TZ		 814 hist_data->fields[key_idx]->offset = key_offset;
815 hist_data->key_size += key_size;
7ef224d1
TZ		 816 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
817 ret = -EINVAL;
818 goto out;
819 }
820
821 hist_data->n_keys++;
30350d65 822 hist_data->n_fields++;
7ef224d1
TZ		 823
824 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
825 return -EINVAL;
826
827 ret = key_size;
828 out:
829 return ret;
830}
831
832static int create_key_fields(struct hist_trigger_data *hist_data,
833 struct trace_event_file *file)
834{
76a3b0c8 835 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 836 char *fields_str, *field_str;
837 int ret = -EINVAL;
838
839 fields_str = hist_data->attrs->keys_str;
840 if (!fields_str)
841 goto out;
842
843 strsep(&fields_str, "=");
844 if (!fields_str)
845 goto out;
846
76a3b0c8 847 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 848 field_str = strsep(&fields_str, ",");
849 if (!field_str)
850 break;
76a3b0c8
TZ		 851 ret = create_key_field(hist_data, i, key_offset,
852 file, field_str);
7ef224d1
TZ		 853 if (ret < 0)
854 goto out;
76a3b0c8 855 key_offset += ret;
7ef224d1
TZ		 856 }
857 if (fields_str) {
858 ret = -EINVAL;
859 goto out;
860 }
861 ret = 0;
862 out:
863 return ret;
864}
865
30350d65
TZ		 866static int create_var_fields(struct hist_trigger_data *hist_data,
867 struct trace_event_file *file)
868{
869 unsigned int i, j = hist_data->n_vals;
870 int ret = 0;
871
872 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
873
874 for (i = 0; i < n_vars; i++) {
875 char *var_name = hist_data->attrs->var_defs.name[i];
876 char *expr = hist_data->attrs->var_defs.expr[i];
877
878 ret = create_var_field(hist_data, j++, file, var_name, expr);
879 if (ret)
880 goto out;
881 }
882 out:
883 return ret;
884}
885
886static void free_var_defs(struct hist_trigger_data *hist_data)
887{
888 unsigned int i;
889
890 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
891 kfree(hist_data->attrs->var_defs.name[i]);
892 kfree(hist_data->attrs->var_defs.expr[i]);
893 }
894
895 hist_data->attrs->var_defs.n_vars = 0;
896}
897
898static int parse_var_defs(struct hist_trigger_data *hist_data)
899{
900 char *s, *str, *var_name, *field_str;
901 unsigned int i, j, n_vars = 0;
902 int ret = 0;
903
904 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
905 str = hist_data->attrs->assignment_str[i];
906 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
907 field_str = strsep(&str, ",");
908 if (!field_str)
909 break;
910
911 var_name = strsep(&field_str, "=");
912 if (!var_name || !field_str) {
913 ret = -EINVAL;
914 goto free;
915 }
916
917 if (n_vars == TRACING_MAP_VARS_MAX) {
918 ret = -EINVAL;
919 goto free;
920 }
921
922 s = kstrdup(var_name, GFP_KERNEL);
923 if (!s) {
924 ret = -ENOMEM;
925 goto free;
926 }
927 hist_data->attrs->var_defs.name[n_vars] = s;
928
929 s = kstrdup(field_str, GFP_KERNEL);
930 if (!s) {
931 kfree(hist_data->attrs->var_defs.name[n_vars]);
932 ret = -ENOMEM;
933 goto free;
934 }
935 hist_data->attrs->var_defs.expr[n_vars++] = s;
936
937 hist_data->attrs->var_defs.n_vars = n_vars;
938 }
939 }
940
941 return ret;
942 free:
943 free_var_defs(hist_data);
944
945 return ret;
946}
947
7ef224d1
TZ		 948static int create_hist_fields(struct hist_trigger_data *hist_data,
949 struct trace_event_file *file)
950{
951 int ret;
952
30350d65
TZ		 953 ret = parse_var_defs(hist_data);
954 if (ret)
955 goto out;
956
7ef224d1
TZ		 957 ret = create_val_fields(hist_data, file);
958 if (ret)
959 goto out;
960
30350d65 961 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 962 if (ret)
963 goto out;
964
30350d65
TZ		 965 ret = create_key_fields(hist_data, file);
966 if (ret)
967 goto out;
7ef224d1 968 out:
30350d65
TZ		 969 free_var_defs(hist_data);
970
7ef224d1
TZ		 971 return ret;
972}
973
e62347d2
TZ		 974static int is_descending(const char *str)
975{
976 if (!str)
977 return 0;
978
979 if (strcmp(str, "descending") == 0)
980 return 1;
981
982 if (strcmp(str, "ascending") == 0)
983 return 0;
984
985 return -EINVAL;
986}
987
7ef224d1
TZ		 988static int create_sort_keys(struct hist_trigger_data *hist_data)
989{
e62347d2 990 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 991 struct tracing_map_sort_key *sort_key;
992 int descending, ret = 0;
30350d65 993 unsigned int i, j, k;
e62347d2
TZ		 994
995 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
996
997 if (!fields_str)
998 goto out;
999
1000 strsep(&fields_str, "=");
1001 if (!fields_str) {
1002 ret = -EINVAL;
1003 goto out;
1004 }
1005
1006 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 1007 struct hist_field *hist_field;
e62347d2 1008 char *field_str, *field_name;
85013256 1009 const char *test_name;
e62347d2
TZ		 1010
1011 sort_key = &hist_data->sort_keys[i];
1012
1013 field_str = strsep(&fields_str, ",");
1014 if (!field_str) {
1015 if (i == 0)
1016 ret = -EINVAL;
1017 break;
1018 }
1019
1020 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
1021 ret = -EINVAL;
1022 break;
1023 }
7ef224d1 1024
e62347d2
TZ		 1025 field_name = strsep(&field_str, ".");
1026 if (!field_name) {
1027 ret = -EINVAL;
1028 break;
1029 }
1030
1031 if (strcmp(field_name, "hitcount") == 0) {
1032 descending = is_descending(field_str);
1033 if (descending < 0) {
1034 ret = descending;
1035 break;
1036 }
1037 sort_key->descending = descending;
1038 continue;
1039 }
7ef224d1 1040
30350d65
TZ		 1041 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
1042 unsigned int idx;
1043
85013256 1044 hist_field = hist_data->fields[j];
30350d65
TZ		 1045 if (hist_field->flags & HIST_FIELD_FL_VAR)
1046 continue;
1047
1048 idx = k++;
1049
85013256
TZ		 1050 test_name = hist_field_name(hist_field, 0);
1051
1052 if (strcmp(field_name, test_name) == 0) {
30350d65 1053 sort_key->field_idx = idx;
e62347d2
TZ		 1054 descending = is_descending(field_str);
1055 if (descending < 0) {
1056 ret = descending;
1057 goto out;
1058 }
1059 sort_key->descending = descending;
1060 break;
1061 }
1062 }
1063 if (j == hist_data->n_fields) {
1064 ret = -EINVAL;
1065 break;
1066 }
1067 }
30350d65 1068
e62347d2
TZ		 1069 hist_data->n_sort_keys = i;
1070 out:
7ef224d1
TZ		 1071 return ret;
1072}
1073
1074static void destroy_hist_data(struct hist_trigger_data *hist_data)
1075{
1076 destroy_hist_trigger_attrs(hist_data->attrs);
1077 destroy_hist_fields(hist_data);
1078 tracing_map_destroy(hist_data->map);
1079 kfree(hist_data);
1080}
1081
1082static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
1083{
1084 struct tracing_map *map = hist_data->map;
1085 struct ftrace_event_field *field;
1086 struct hist_field *hist_field;
d50c744e 1087 int i, idx;
7ef224d1
TZ		 1088
1089 for_each_hist_field(i, hist_data) {
1090 hist_field = hist_data->fields[i];
1091 if (hist_field->flags & HIST_FIELD_FL_KEY) {
1092 tracing_map_cmp_fn_t cmp_fn;
1093
1094 field = hist_field->field;
1095
69a0200c
TZ		 1096 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1097 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 1098 else if (!field)
1099 cmp_fn = tracing_map_cmp_num(hist_field->size,
1100 hist_field->is_signed);
69a0200c 1101 else if (is_string_field(field))
7ef224d1
TZ		 1102 cmp_fn = tracing_map_cmp_string;
1103 else
1104 cmp_fn = tracing_map_cmp_num(field->size,
1105 field->is_signed);
76a3b0c8
TZ		 1106 idx = tracing_map_add_key_field(map,
1107 hist_field->offset,
1108 cmp_fn);
30350d65 1109 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 1110 idx = tracing_map_add_sum_field(map);
1111
1112 if (idx < 0)
1113 return idx;
30350d65
TZ		 1114
1115 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1116 idx = tracing_map_add_var(map);
1117 if (idx < 0)
1118 return idx;
1119 hist_field->var.idx = idx;
1120 hist_field->var.hist_data = hist_data;
1121 }
7ef224d1
TZ		 1122 }
1123
1124 return 0;
1125}
1126
6b4827ad
TZ		 1127static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
1128{
1129 struct hist_field *key_field;
1130 unsigned int i;
1131
1132 for_each_hist_key_field(i, hist_data) {
1133 key_field = hist_data->fields[i];
1134
1135 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
1136 return true;
1137 }
1138
1139 return false;
1140}
1141
7ef224d1
TZ		 1142static struct hist_trigger_data *
1143create_hist_data(unsigned int map_bits,
1144 struct hist_trigger_attrs *attrs,
30350d65
TZ		 1145 struct trace_event_file *file,
1146 bool remove)
7ef224d1 1147{
6b4827ad 1148 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 1149 struct hist_trigger_data *hist_data;
1150 int ret = 0;
1151
1152 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
1153 if (!hist_data)
1154 return ERR_PTR(-ENOMEM);
1155
1156 hist_data->attrs = attrs;
30350d65 1157 hist_data->remove = remove;
7ef224d1
TZ		 1158
1159 ret = create_hist_fields(hist_data, file);
1160 if (ret)
1161 goto free;
1162
1163 ret = create_sort_keys(hist_data);
1164 if (ret)
1165 goto free;
1166
6b4827ad
TZ		 1167 if (need_tracing_map_ops(hist_data))
1168 map_ops = &hist_trigger_elt_comm_ops;
1169
7ef224d1 1170 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 1171 map_ops, hist_data);
7ef224d1
TZ		 1172 if (IS_ERR(hist_data->map)) {
1173 ret = PTR_ERR(hist_data->map);
1174 hist_data->map = NULL;
1175 goto free;
1176 }
1177
1178 ret = create_tracing_map_fields(hist_data);
1179 if (ret)
1180 goto free;
1181
1182 ret = tracing_map_init(hist_data->map);
1183 if (ret)
1184 goto free;
1185
1186 hist_data->event_file = file;
1187 out:
1188 return hist_data;
1189 free:
1190 hist_data->attrs = NULL;
1191
1192 destroy_hist_data(hist_data);
1193
1194 hist_data = ERR_PTR(ret);
1195
1196 goto out;
1197}
1198
1199static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb
TZ		 1200 struct tracing_map_elt *elt, void *rec,
1201 struct ring_buffer_event *rbe)
7ef224d1
TZ		 1202{
1203 struct hist_field *hist_field;
30350d65 1204 unsigned int i, var_idx;
7ef224d1
TZ		 1205 u64 hist_val;
1206
1207 for_each_hist_val_field(i, hist_data) {
1208 hist_field = hist_data->fields[i];
fbd302cb 1209 hist_val = hist_field->fn(hist_field, rec, rbe);
30350d65
TZ		 1210 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1211 var_idx = hist_field->var.idx;
1212 tracing_map_set_var(elt, var_idx, hist_val);
1213 continue;
1214 }
7ef224d1
TZ		 1215 tracing_map_update_sum(elt, i, hist_val);
1216 }
30350d65
TZ		 1217
1218 for_each_hist_key_field(i, hist_data) {
1219 hist_field = hist_data->fields[i];
1220 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1221 hist_val = hist_field->fn(hist_field, rec, rbe);
1222 var_idx = hist_field->var.idx;
1223 tracing_map_set_var(elt, var_idx, hist_val);
1224 }
1225 }
7ef224d1
TZ		 1226}
1227
6a475cb1
TZ		 1228static inline void add_to_key(char *compound_key, void *key,
1229 struct hist_field *key_field, void *rec)
1230{
1231 size_t size = key_field->size;
1232
1233 if (key_field->flags & HIST_FIELD_FL_STRING) {
1234 struct ftrace_event_field *field;
1235
1236 field = key_field->field;
1237 if (field->filter_type == FILTER_DYN_STRING)
1238 size = *(u32 *)(rec + field->offset) >> 16;
1239 else if (field->filter_type == FILTER_PTR_STRING)
1240 size = strlen(key);
1241 else if (field->filter_type == FILTER_STATIC_STRING)
1242 size = field->size;
1243
1244 /* ensure NULL-termination */
1245 if (size > key_field->size - 1)
1246 size = key_field->size - 1;
1247 }
1248
1249 memcpy(compound_key + key_field->offset, key, size);
1250}
1251
1ac4f51c 1252static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 1253 struct ring_buffer_event *rbe)
7ef224d1
TZ		 1254{
1255 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 1256 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 1257 unsigned long entries[HIST_STACKTRACE_DEPTH];
76a3b0c8 1258 char compound_key[HIST_KEY_SIZE_MAX];
69a0200c 1259 struct stack_trace stacktrace;
7ef224d1
TZ		 1260 struct hist_field *key_field;
1261 struct tracing_map_elt *elt;
1262 u64 field_contents;
1263 void *key = NULL;
1264 unsigned int i;
1265
6a475cb1 1266 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 1267
7ef224d1
TZ		 1268 for_each_hist_key_field(i, hist_data) {
1269 key_field = hist_data->fields[i];
1270
69a0200c
TZ		 1271 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1272 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
1273 stacktrace.entries = entries;
1274 stacktrace.nr_entries = 0;
1275 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 1276
69a0200c
TZ		 1277 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
1278 save_stack_trace(&stacktrace);
1279
1280 key = entries;
1281 } else {
fbd302cb 1282 field_contents = key_field->fn(key_field, rec, rbe);
6a475cb1 1283 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 1284 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 1285 use_compound_key = true;
1286 } else
69a0200c 1287 key = (void *)&field_contents;
76a3b0c8 1288 }
6a475cb1
TZ		 1289
1290 if (use_compound_key)
1291 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 1292 }
1293
6a475cb1 1294 if (use_compound_key)
76a3b0c8
TZ		 1295 key = compound_key;
1296
7ef224d1
TZ		 1297 elt = tracing_map_insert(hist_data->map, key);
1298 if (elt)
fbd302cb 1299 hist_trigger_elt_update(hist_data, elt, rec, rbe);
7ef224d1
TZ		 1300}
1301
69a0200c
TZ		 1302static void hist_trigger_stacktrace_print(struct seq_file *m,
1303 unsigned long *stacktrace_entries,
1304 unsigned int max_entries)
1305{
1306 char str[KSYM_SYMBOL_LEN];
1307 unsigned int spaces = 8;
1308 unsigned int i;
1309
1310 for (i = 0; i < max_entries; i++) {
1311 if (stacktrace_entries[i] == ULONG_MAX)
1312 return;
1313
1314 seq_printf(m, "%*c", 1 + spaces, ' ');
1315 sprint_symbol(str, stacktrace_entries[i]);
1316 seq_printf(m, "%s\n", str);
1317 }
1318}
1319
7ef224d1
TZ		 1320static void
1321hist_trigger_entry_print(struct seq_file *m,
1322 struct hist_trigger_data *hist_data, void *key,
1323 struct tracing_map_elt *elt)
1324{
1325 struct hist_field *key_field;
c6afad49 1326 char str[KSYM_SYMBOL_LEN];
69a0200c 1327 bool multiline = false;
85013256 1328 const char *field_name;
7ef224d1
TZ		 1329 unsigned int i;
1330 u64 uval;
1331
1332 seq_puts(m, "{ ");
1333
1334 for_each_hist_key_field(i, hist_data) {
1335 key_field = hist_data->fields[i];
1336
1337 if (i > hist_data->n_vals)
1338 seq_puts(m, ", ");
1339
85013256
TZ		 1340 field_name = hist_field_name(key_field, 0);
1341
0c4a6b46
TZ		 1342 if (key_field->flags & HIST_FIELD_FL_HEX) {
1343 uval = *(u64 *)(key + key_field->offset);
85013256 1344 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 1345 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
1346 uval = *(u64 *)(key + key_field->offset);
1347 sprint_symbol_no_offset(str, uval);
85013256
TZ		 1348 seq_printf(m, "%s: [%llx] %-45s", field_name,
1349 uval, str);
c6afad49
TZ		 1350 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
1351 uval = *(u64 *)(key + key_field->offset);
1352 sprint_symbol(str, uval);
85013256
TZ		 1353 seq_printf(m, "%s: [%llx] %-55s", field_name,
1354 uval, str);
6b4827ad
TZ		 1355 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1356 char *comm = elt->private_data;
1357
1358 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 1359 seq_printf(m, "%s: %-16s[%10llu]", field_name,
1360 comm, uval);
31696198
TZ		 1361 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
1362 const char *syscall_name;
1363
1364 uval = *(u64 *)(key + key_field->offset);
1365 syscall_name = get_syscall_name(uval);
1366 if (!syscall_name)
1367 syscall_name = "unknown_syscall";
1368
85013256
TZ		 1369 seq_printf(m, "%s: %-30s[%3llu]", field_name,
1370 syscall_name, uval);
69a0200c
TZ		 1371 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1372 seq_puts(m, "stacktrace:\n");
1373 hist_trigger_stacktrace_print(m,
1374 key + key_field->offset,
1375 HIST_STACKTRACE_DEPTH);
1376 multiline = true;
4b94f5b7 1377 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 1378 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 1379 *(u64 *)(key + key_field->offset));
0c4a6b46 1380 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 1381 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 1382 (char *)(key + key_field->offset));
7ef224d1 1383 } else {
76a3b0c8 1384 uval = *(u64 *)(key + key_field->offset);
85013256 1385 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 1386 }
1387 }
1388
69a0200c
TZ		 1389 if (!multiline)
1390 seq_puts(m, " ");
1391
1392 seq_puts(m, "}");
7ef224d1
TZ		 1393
1394 seq_printf(m, " hitcount: %10llu",
1395 tracing_map_read_sum(elt, HITCOUNT_IDX));
1396
f2606835 1397 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 1398 field_name = hist_field_name(hist_data->fields[i], 0);
1399
30350d65
TZ		 1400 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR)
1401 continue;
1402
0c4a6b46 1403 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 1404 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 1405 tracing_map_read_sum(elt, i));
1406 } else {
85013256 1407 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 1408 tracing_map_read_sum(elt, i));
1409 }
f2606835
TZ		 1410 }
1411
7ef224d1
TZ		 1412 seq_puts(m, "\n");
1413}
1414
1415static int print_entries(struct seq_file *m,
1416 struct hist_trigger_data *hist_data)
1417{
1418 struct tracing_map_sort_entry **sort_entries = NULL;
1419 struct tracing_map *map = hist_data->map;
d50c744e 1420 int i, n_entries;
7ef224d1
TZ		 1421
1422 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1423 hist_data->n_sort_keys,
1424 &sort_entries);
1425 if (n_entries < 0)
1426 return n_entries;
1427
1428 for (i = 0; i < n_entries; i++)
1429 hist_trigger_entry_print(m, hist_data,
1430 sort_entries[i]->key,
1431 sort_entries[i]->elt);
1432
1433 tracing_map_destroy_sort_entries(sort_entries, n_entries);
1434
1435 return n_entries;
1436}
1437
52a7f16d
TZ		 1438static void hist_trigger_show(struct seq_file *m,
1439 struct event_trigger_data *data, int n)
7ef224d1 1440{
7ef224d1 1441 struct hist_trigger_data *hist_data;
6e7a2398 1442 int n_entries;
7ef224d1 1443
52a7f16d
TZ		 1444 if (n > 0)
1445 seq_puts(m, "\n\n");
7ef224d1
TZ		 1446
1447 seq_puts(m, "# event histogram\n#\n# trigger info: ");
1448 data->ops->print(m, data->ops, data);
52a7f16d 1449 seq_puts(m, "#\n\n");
7ef224d1
TZ		 1450
1451 hist_data = data->private_data;
1452 n_entries = print_entries(m, hist_data);
6e7a2398 1453 if (n_entries < 0)
7ef224d1 1454 n_entries = 0;
7ef224d1
TZ		 1455
1456 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
1457 (u64)atomic64_read(&hist_data->map->hits),
1458 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 1459}
1460
1461static int hist_show(struct seq_file *m, void *v)
1462{
1463 struct event_trigger_data *data;
1464 struct trace_event_file *event_file;
1465 int n = 0, ret = 0;
1466
1467 mutex_lock(&event_mutex);
1468
1469 event_file = event_file_data(m->private);
1470 if (unlikely(!event_file)) {
1471 ret = -ENODEV;
1472 goto out_unlock;
1473 }
1474
1475 list_for_each_entry_rcu(data, &event_file->triggers, list) {
1476 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1477 hist_trigger_show(m, data, n++);
1478 }
1479
7ef224d1
TZ		 1480 out_unlock:
1481 mutex_unlock(&event_mutex);
1482
1483 return ret;
1484}
1485
1486static int event_hist_open(struct inode *inode, struct file *file)
1487{
1488 return single_open(file, hist_show, file);
1489}
1490
1491const struct file_operations event_hist_fops = {
1492 .open = event_hist_open,
1493 .read = seq_read,
1494 .llseek = seq_lseek,
1495 .release = single_release,
1496};
1497
0c4a6b46
TZ		 1498static const char *get_hist_field_flags(struct hist_field *hist_field)
1499{
1500 const char *flags_str = NULL;
1501
1502 if (hist_field->flags & HIST_FIELD_FL_HEX)
1503 flags_str = "hex";
c6afad49
TZ		 1504 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1505 flags_str = "sym";
1506 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1507 flags_str = "sym-offset";
6b4827ad
TZ		 1508 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1509 flags_str = "execname";
31696198
TZ		 1510 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1511 flags_str = "syscall";
4b94f5b7
NK		 1512 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1513 flags_str = "log2";
860f9f6b
TZ		 1514 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1515 flags_str = "usecs";
0c4a6b46
TZ		 1516
1517 return flags_str;
1518}
1519
7ef224d1
TZ		 1520static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1521{
85013256
TZ		 1522 const char *field_name = hist_field_name(hist_field, 0);
1523
30350d65
TZ		 1524 if (hist_field->var.name)
1525 seq_printf(m, "%s=", hist_field->var.name);
1526
ad42febe
TZ		 1527 if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
1528 seq_puts(m, "common_timestamp");
1529 else if (field_name)
1530 seq_printf(m, "%s", field_name);
1531
0c4a6b46
TZ		 1532 if (hist_field->flags) {
1533 const char *flags_str = get_hist_field_flags(hist_field);
1534
1535 if (flags_str)
1536 seq_printf(m, ".%s", flags_str);
1537 }
7ef224d1
TZ		 1538}
1539
1540static int event_hist_trigger_print(struct seq_file *m,
1541 struct event_trigger_ops *ops,
1542 struct event_trigger_data *data)
1543{
1544 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 1545 struct hist_field *field;
1546 bool have_var = false;
7ef224d1
TZ		 1547 unsigned int i;
1548
5463bfda
TZ		 1549 seq_puts(m, "hist:");
1550
1551 if (data->name)
1552 seq_printf(m, "%s:", data->name);
1553
1554 seq_puts(m, "keys=");
7ef224d1
TZ		 1555
1556 for_each_hist_key_field(i, hist_data) {
30350d65 1557 field = hist_data->fields[i];
7ef224d1
TZ		 1558
1559 if (i > hist_data->n_vals)
1560 seq_puts(m, ",");
1561
30350d65 1562 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 1563 seq_puts(m, "stacktrace");
1564 else
30350d65 1565 hist_field_print(m, field);
7ef224d1
TZ		 1566 }
1567
1568 seq_puts(m, ":vals=");
f2606835
TZ		 1569
1570 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 1571 field = hist_data->fields[i];
1572 if (field->flags & HIST_FIELD_FL_VAR) {
1573 have_var = true;
1574 continue;
1575 }
1576
f2606835
TZ		 1577 if (i == HITCOUNT_IDX)
1578 seq_puts(m, "hitcount");
1579 else {
1580 seq_puts(m, ",");
30350d65
TZ		 1581 hist_field_print(m, field);
1582 }
1583 }
1584
1585 if (have_var) {
1586 unsigned int n = 0;
1587
1588 seq_puts(m, ":");
1589
1590 for_each_hist_val_field(i, hist_data) {
1591 field = hist_data->fields[i];
1592
1593 if (field->flags & HIST_FIELD_FL_VAR) {
1594 if (n++)
1595 seq_puts(m, ",");
1596 hist_field_print(m, field);
1597 }
f2606835
TZ		 1598 }
1599 }
7ef224d1
TZ		 1600
1601 seq_puts(m, ":sort=");
e62347d2
TZ		 1602
1603 for (i = 0; i < hist_data->n_sort_keys; i++) {
1604 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 1605 unsigned int idx, first_key_idx;
1606
1607 /* skip VAR vals */
1608 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 1609
1610 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 1611 idx = sort_key->field_idx;
1612
1a361dfc 1613 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 1614 return -EINVAL;
e62347d2
TZ		 1615
1616 if (i > 0)
1617 seq_puts(m, ",");
1618
ad42febe 1619 if (idx == HITCOUNT_IDX)
e62347d2 1620 seq_puts(m, "hitcount");
30350d65
TZ		 1621 else {
1622 if (idx >= first_key_idx)
1623 idx += hist_data->n_vars;
e62347d2 1624 hist_field_print(m, hist_data->fields[idx]);
30350d65 1625 }
e62347d2
TZ		 1626
1627 if (sort_key->descending)
1628 seq_puts(m, ".descending");
1629 }
7ef224d1
TZ		 1630 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1631
1632 if (data->filter_str)
1633 seq_printf(m, " if %s", data->filter_str);
1634
83e99914
TZ		 1635 if (data->paused)
1636 seq_puts(m, " [paused]");
1637 else
1638 seq_puts(m, " [active]");
7ef224d1
TZ		 1639
1640 seq_putc(m, '\n');
1641
1642 return 0;
1643}
1644
5463bfda
TZ		 1645static int event_hist_trigger_init(struct event_trigger_ops *ops,
1646 struct event_trigger_data *data)
1647{
1648 struct hist_trigger_data *hist_data = data->private_data;
1649
1650 if (!data->ref && hist_data->attrs->name)
1651 save_named_trigger(hist_data->attrs->name, data);
1652
1653 data->ref++;
1654
1655 return 0;
1656}
1657
7ef224d1
TZ		 1658static void event_hist_trigger_free(struct event_trigger_ops *ops,
1659 struct event_trigger_data *data)
1660{
1661 struct hist_trigger_data *hist_data = data->private_data;
1662
1663 if (WARN_ON_ONCE(data->ref <= 0))
1664 return;
1665
1666 data->ref--;
1667 if (!data->ref) {
5463bfda
TZ		 1668 if (data->name)
1669 del_named_trigger(data);
7ef224d1
TZ		 1670 trigger_data_free(data);
1671 destroy_hist_data(hist_data);
1672 }
1673}
1674
1675static struct event_trigger_ops event_hist_trigger_ops = {
1676 .func = event_hist_trigger,
1677 .print = event_hist_trigger_print,
5463bfda 1678 .init = event_hist_trigger_init,
7ef224d1
TZ		 1679 .free = event_hist_trigger_free,
1680};
1681
5463bfda
TZ		 1682static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
1683 struct event_trigger_data *data)
1684{
1685 data->ref++;
1686
1687 save_named_trigger(data->named_data->name, data);
1688
1689 event_hist_trigger_init(ops, data->named_data);
1690
1691 return 0;
1692}
1693
1694static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
1695 struct event_trigger_data *data)
1696{
1697 if (WARN_ON_ONCE(data->ref <= 0))
1698 return;
1699
1700 event_hist_trigger_free(ops, data->named_data);
1701
1702 data->ref--;
1703 if (!data->ref) {
1704 del_named_trigger(data);
1705 trigger_data_free(data);
1706 }
1707}
1708
1709static struct event_trigger_ops event_hist_trigger_named_ops = {
1710 .func = event_hist_trigger,
1711 .print = event_hist_trigger_print,
1712 .init = event_hist_trigger_named_init,
1713 .free = event_hist_trigger_named_free,
1714};
1715
7ef224d1
TZ		 1716static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1717 char *param)
1718{
1719 return &event_hist_trigger_ops;
1720}
1721
e86ae9ba
TZ		 1722static void hist_clear(struct event_trigger_data *data)
1723{
1724 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 1725
5463bfda
TZ		 1726 if (data->name)
1727 pause_named_trigger(data);
e86ae9ba
TZ		 1728
1729 synchronize_sched();
1730
1731 tracing_map_clear(hist_data->map);
1732
5463bfda
TZ		 1733 if (data->name)
1734 unpause_named_trigger(data);
1735}
1736
1737static bool compatible_field(struct ftrace_event_field *field,
1738 struct ftrace_event_field *test_field)
1739{
1740 if (field == test_field)
1741 return true;
1742 if (field == NULL || test_field == NULL)
1743 return false;
1744 if (strcmp(field->name, test_field->name) != 0)
1745 return false;
1746 if (strcmp(field->type, test_field->type) != 0)
1747 return false;
1748 if (field->size != test_field->size)
1749 return false;
1750 if (field->is_signed != test_field->is_signed)
1751 return false;
1752
1753 return true;
e86ae9ba
TZ		 1754}
1755
52a7f16d 1756static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 1757 struct event_trigger_data *data_test,
1758 struct event_trigger_data *named_data,
1759 bool ignore_filter)
52a7f16d
TZ		 1760{
1761 struct tracing_map_sort_key *sort_key, *sort_key_test;
1762 struct hist_trigger_data *hist_data, *hist_data_test;
1763 struct hist_field *key_field, *key_field_test;
1764 unsigned int i;
1765
5463bfda
TZ		 1766 if (named_data && (named_data != data_test) &&
1767 (named_data != data_test->named_data))
1768 return false;
1769
1770 if (!named_data && is_named_trigger(data_test))
1771 return false;
1772
52a7f16d
TZ		 1773 hist_data = data->private_data;
1774 hist_data_test = data_test->private_data;
1775
1776 if (hist_data->n_vals != hist_data_test->n_vals ||
1777 hist_data->n_fields != hist_data_test->n_fields ||
1778 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
1779 return false;
1780
5463bfda
TZ		 1781 if (!ignore_filter) {
1782 if ((data->filter_str && !data_test->filter_str) ||
1783 (!data->filter_str && data_test->filter_str))
1784 return false;
1785 }
52a7f16d
TZ		 1786
1787 for_each_hist_field(i, hist_data) {
1788 key_field = hist_data->fields[i];
1789 key_field_test = hist_data_test->fields[i];
1790
1791 if (key_field->flags != key_field_test->flags)
1792 return false;
5463bfda 1793 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 1794 return false;
1795 if (key_field->offset != key_field_test->offset)
1796 return false;
ad42febe
TZ		 1797 if (key_field->size != key_field_test->size)
1798 return false;
1799 if (key_field->is_signed != key_field_test->is_signed)
1800 return false;
1a361dfc
TZ		 1801 if (!!key_field->var.name != !!key_field_test->var.name)
1802 return false;
1803 if (key_field->var.name &&
1804 strcmp(key_field->var.name, key_field_test->var.name) != 0)
1805 return false;
52a7f16d
TZ		 1806 }
1807
1808 for (i = 0; i < hist_data->n_sort_keys; i++) {
1809 sort_key = &hist_data->sort_keys[i];
1810 sort_key_test = &hist_data_test->sort_keys[i];
1811
1812 if (sort_key->field_idx != sort_key_test->field_idx ||
1813 sort_key->descending != sort_key_test->descending)
1814 return false;
1815 }
1816
5463bfda 1817 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 1818 (strcmp(data->filter_str, data_test->filter_str) != 0))
1819 return false;
1820
1821 return true;
1822}
1823
7ef224d1
TZ		 1824static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1825 struct event_trigger_data *data,
1826 struct trace_event_file *file)
1827{
83e99914 1828 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 1829 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 1830 int ret = 0;
1831
5463bfda
TZ		 1832 if (hist_data->attrs->name) {
1833 named_data = find_named_trigger(hist_data->attrs->name);
1834 if (named_data) {
1835 if (!hist_trigger_match(data, named_data, named_data,
1836 true)) {
1837 ret = -EINVAL;
1838 goto out;
1839 }
1840 }
1841 }
1842
1843 if (hist_data->attrs->name && !named_data)
1844 goto new;
1845
7ef224d1
TZ		 1846 list_for_each_entry_rcu(test, &file->triggers, list) {
1847 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 1848 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 1849 continue;
83e99914
TZ		 1850 if (hist_data->attrs->pause)
1851 test->paused = true;
1852 else if (hist_data->attrs->cont)
1853 test->paused = false;
e86ae9ba
TZ		 1854 else if (hist_data->attrs->clear)
1855 hist_clear(test);
83e99914
TZ		 1856 else
1857 ret = -EEXIST;
7ef224d1
TZ		 1858 goto out;
1859 }
1860 }
5463bfda 1861 new:
e86ae9ba 1862 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 1863 ret = -ENOENT;
1864 goto out;
1865 }
1866
7522c03a
TZ		 1867 if (hist_data->attrs->pause)
1868 data->paused = true;
1869
5463bfda
TZ		 1870 if (named_data) {
1871 destroy_hist_data(data->private_data);
1872 data->private_data = named_data->private_data;
1873 set_named_trigger_data(data, named_data);
1874 data->ops = &event_hist_trigger_named_ops;
1875 }
1876
7ef224d1
TZ		 1877 if (data->ops->init) {
1878 ret = data->ops->init(data->ops, data);
1879 if (ret < 0)
1880 goto out;
1881 }
1882
1883 list_add_rcu(&data->list, &file->triggers);
1884 ret++;
1885
1886 update_cond_flag(file);
5463bfda 1887
ad42febe
TZ		 1888 if (hist_data->enable_timestamps)
1889 tracing_set_time_stamp_abs(file->tr, true);
1890
7ef224d1
TZ		 1891 if (trace_event_trigger_enable_disable(file, 1) < 0) {
1892 list_del_rcu(&data->list);
1893 update_cond_flag(file);
1894 ret--;
1895 }
1896 out:
1897 return ret;
1898}
1899
52a7f16d
TZ		 1900static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
1901 struct event_trigger_data *data,
1902 struct trace_event_file *file)
1903{
5463bfda
TZ		 1904 struct hist_trigger_data *hist_data = data->private_data;
1905 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 1906 bool unregistered = false;
1907
5463bfda
TZ		 1908 if (hist_data->attrs->name)
1909 named_data = find_named_trigger(hist_data->attrs->name);
1910
52a7f16d
TZ		 1911 list_for_each_entry_rcu(test, &file->triggers, list) {
1912 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 1913 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 1914 continue;
1915 unregistered = true;
1916 list_del_rcu(&test->list);
1917 trace_event_trigger_enable_disable(file, 0);
1918 update_cond_flag(file);
1919 break;
1920 }
1921 }
1922
1923 if (unregistered && test->ops->free)
1924 test->ops->free(test->ops, test);
ad42febe
TZ		 1925
1926 if (hist_data->enable_timestamps) {
30350d65 1927 if (!hist_data->remove || unregistered)
ad42febe
TZ		 1928 tracing_set_time_stamp_abs(file->tr, false);
1929 }
52a7f16d
TZ		 1930}
1931
1932static void hist_unreg_all(struct trace_event_file *file)
1933{
47c18569 1934 struct event_trigger_data *test, *n;
ad42febe 1935 struct hist_trigger_data *hist_data;
52a7f16d 1936
47c18569 1937 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 1938 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 1939 hist_data = test->private_data;
52a7f16d
TZ		 1940 list_del_rcu(&test->list);
1941 trace_event_trigger_enable_disable(file, 0);
1942 update_cond_flag(file);
ad42febe
TZ		 1943 if (hist_data->enable_timestamps)
1944 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 1945 if (test->ops->free)
1946 test->ops->free(test->ops, test);
1947 }
1948 }
1949}
1950
7ef224d1
TZ		 1951static int event_hist_trigger_func(struct event_command *cmd_ops,
1952 struct trace_event_file *file,
1953 char *glob, char *cmd, char *param)
1954{
1955 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1956 struct event_trigger_data *trigger_data;
1957 struct hist_trigger_attrs *attrs;
1958 struct event_trigger_ops *trigger_ops;
1959 struct hist_trigger_data *hist_data;
30350d65 1960 bool remove = false;
7ef224d1
TZ		 1961 char *trigger;
1962 int ret = 0;
1963
1964 if (!param)
1965 return -EINVAL;
1966
30350d65
TZ		 1967 if (glob[0] == '!')
1968 remove = true;
1969
7ef224d1
TZ		 1970 /* separate the trigger from the filter (k:v [if filter]) */
1971 trigger = strsep(&param, " \t");
1972 if (!trigger)
1973 return -EINVAL;
1974
1975 attrs = parse_hist_trigger_attrs(trigger);
1976 if (IS_ERR(attrs))
1977 return PTR_ERR(attrs);
1978
1979 if (attrs->map_bits)
1980 hist_trigger_bits = attrs->map_bits;
1981
30350d65 1982 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 1983 if (IS_ERR(hist_data)) {
1984 destroy_hist_trigger_attrs(attrs);
1985 return PTR_ERR(hist_data);
1986 }
1987
1988 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1989
1990 ret = -ENOMEM;
1991 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1992 if (!trigger_data)
1993 goto out_free;
1994
1995 trigger_data->count = -1;
1996 trigger_data->ops = trigger_ops;
1997 trigger_data->cmd_ops = cmd_ops;
1998
1999 INIT_LIST_HEAD(&trigger_data->list);
2000 RCU_INIT_POINTER(trigger_data->filter, NULL);
2001
2002 trigger_data->private_data = hist_data;
2003
52a7f16d
TZ		 2004 /* if param is non-empty, it's supposed to be a filter */
2005 if (param && cmd_ops->set_filter) {
2006 ret = cmd_ops->set_filter(param, trigger_data, file);
2007 if (ret < 0)
2008 goto out_free;
2009 }
2010
30350d65 2011 if (remove) {
7ef224d1
TZ		 2012 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
2013 ret = 0;
2014 goto out_free;
2015 }
2016
7ef224d1
TZ		 2017 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
2018 /*
2019 * The above returns on success the # of triggers registered,
2020 * but if it didn't register any it returns zero. Consider no
2021 * triggers registered a failure too.
2022 */
2023 if (!ret) {
e86ae9ba 2024 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 2025 ret = -ENOENT;
7ef224d1
TZ		 2026 goto out_free;
2027 } else if (ret < 0)
2028 goto out_free;
2029 /* Just return zero, not the number of registered triggers */
2030 ret = 0;
2031 out:
2032 return ret;
2033 out_free:
2034 if (cmd_ops->set_filter)
2035 cmd_ops->set_filter(NULL, trigger_data, NULL);
2036
2037 kfree(trigger_data);
2038
2039 destroy_hist_data(hist_data);
2040 goto out;
2041}
2042
2043static struct event_command trigger_hist_cmd = {
2044 .name = "hist",
2045 .trigger_type = ETT_EVENT_HIST,
2046 .flags = EVENT_CMD_FL_NEEDS_REC,
2047 .func = event_hist_trigger_func,
2048 .reg = hist_register_trigger,
52a7f16d
TZ		 2049 .unreg = hist_unregister_trigger,
2050 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 2051 .get_trigger_ops = event_hist_get_trigger_ops,
2052 .set_filter = set_trigger_filter,
2053};
2054
2055__init int register_trigger_hist_cmd(void)
2056{
2057 int ret;
2058
2059 ret = register_event_command(&trigger_hist_cmd);
2060 WARN_ON(ret < 0);
2061
2062 return ret;
2063}
d0bad49b
TZ		 2064
2065static void
1ac4f51c
TZ		 2066hist_enable_trigger(struct event_trigger_data *data, void *rec,
2067 struct ring_buffer_event *event)
d0bad49b
TZ		 2068{
2069 struct enable_trigger_data *enable_data = data->private_data;
2070 struct event_trigger_data *test;
2071
2072 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
2073 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2074 if (enable_data->enable)
2075 test->paused = false;
2076 else
2077 test->paused = true;
d0bad49b
TZ		 2078 }
2079 }
2080}
2081
2082static void
1ac4f51c
TZ		 2083hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
2084 struct ring_buffer_event *event)
d0bad49b
TZ		 2085{
2086 if (!data->count)
2087 return;
2088
2089 if (data->count != -1)
2090 (data->count)--;
2091
1ac4f51c 2092 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 2093}
2094
2095static struct event_trigger_ops hist_enable_trigger_ops = {
2096 .func = hist_enable_trigger,
2097 .print = event_enable_trigger_print,
2098 .init = event_trigger_init,
2099 .free = event_enable_trigger_free,
2100};
2101
2102static struct event_trigger_ops hist_enable_count_trigger_ops = {
2103 .func = hist_enable_count_trigger,
2104 .print = event_enable_trigger_print,
2105 .init = event_trigger_init,
2106 .free = event_enable_trigger_free,
2107};
2108
2109static struct event_trigger_ops hist_disable_trigger_ops = {
2110 .func = hist_enable_trigger,
2111 .print = event_enable_trigger_print,
2112 .init = event_trigger_init,
2113 .free = event_enable_trigger_free,
2114};
2115
2116static struct event_trigger_ops hist_disable_count_trigger_ops = {
2117 .func = hist_enable_count_trigger,
2118 .print = event_enable_trigger_print,
2119 .init = event_trigger_init,
2120 .free = event_enable_trigger_free,
2121};
2122
2123static struct event_trigger_ops *
2124hist_enable_get_trigger_ops(char *cmd, char *param)
2125{
2126 struct event_trigger_ops *ops;
2127 bool enable;
2128
2129 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
2130
2131 if (enable)
2132 ops = param ? &hist_enable_count_trigger_ops :
2133 &hist_enable_trigger_ops;
2134 else
2135 ops = param ? &hist_disable_count_trigger_ops :
2136 &hist_disable_trigger_ops;
2137
2138 return ops;
2139}
2140
52a7f16d
TZ		 2141static void hist_enable_unreg_all(struct trace_event_file *file)
2142{
47c18569 2143 struct event_trigger_data *test, *n;
52a7f16d 2144
47c18569 2145 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 2146 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
2147 list_del_rcu(&test->list);
2148 update_cond_flag(file);
2149 trace_event_trigger_enable_disable(file, 0);
2150 if (test->ops->free)
2151 test->ops->free(test->ops, test);
2152 }
2153 }
2154}
2155
d0bad49b
TZ		 2156static struct event_command trigger_hist_enable_cmd = {
2157 .name = ENABLE_HIST_STR,
2158 .trigger_type = ETT_HIST_ENABLE,
2159 .func = event_enable_trigger_func,
2160 .reg = event_enable_register_trigger,
2161 .unreg = event_enable_unregister_trigger,
52a7f16d 2162 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 2163 .get_trigger_ops = hist_enable_get_trigger_ops,
2164 .set_filter = set_trigger_filter,
2165};
2166
2167static struct event_command trigger_hist_disable_cmd = {
2168 .name = DISABLE_HIST_STR,
2169 .trigger_type = ETT_HIST_ENABLE,
2170 .func = event_enable_trigger_func,
2171 .reg = event_enable_register_trigger,
2172 .unreg = event_enable_unregister_trigger,
52a7f16d 2173 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 2174 .get_trigger_ops = hist_enable_get_trigger_ops,
2175 .set_filter = set_trigger_filter,
2176};
2177
2178static __init void unregister_trigger_hist_enable_disable_cmds(void)
2179{
2180 unregister_event_command(&trigger_hist_enable_cmd);
2181 unregister_event_command(&trigger_hist_disable_cmd);
2182}
2183
2184__init int register_trigger_hist_enable_disable_cmds(void)
2185{
2186 int ret;
2187
2188 ret = register_event_command(&trigger_hist_enable_cmd);
2189 if (WARN_ON(ret < 0))
2190 return ret;
2191 ret = register_event_command(&trigger_hist_disable_cmd);
2192 if (WARN_ON(ret < 0))
2193 unregister_trigger_hist_enable_disable_cmds();
2194
2195 return ret;
2196}
Linux 6.x block layer and io_uring tree(s)