projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tracing: Add hist trigger support for variable reference aliases
[linux-2.6-block.git] / kernel / trace / trace_events_hist.c
CommitLineData
7ef224d1
TZ		 1/*
2 * trace_events_hist - trace event hist triggers
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
15 */
16
17#include <linux/module.h>
18#include <linux/kallsyms.h>
19#include <linux/mutex.h>
20#include <linux/slab.h>
21#include <linux/stacktrace.h>
b2d09103 22#include <linux/rculist.h>
4b147936 23#include <linux/tracefs.h>
7ef224d1
TZ		 24
25#include "tracing_map.h"
26#include "trace.h"
27
4b147936
TZ		 28#define SYNTH_SYSTEM "synthetic"
29#define SYNTH_FIELDS_MAX 16
30
31#define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */
32
7ef224d1
TZ		 33struct hist_field;
34
df35d93b
TZ		 35typedef u64 (*hist_field_fn_t) (struct hist_field *field,
36 struct tracing_map_elt *elt,
37 struct ring_buffer_event *rbe,
38 void *event);
7ef224d1 39
5819eadd 40#define HIST_FIELD_OPERANDS_MAX 2
30350d65 41#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
0212e2aa 42#define HIST_ACTIONS_MAX 8
30350d65 43
100719dc
TZ		 44enum field_op_id {
45 FIELD_OP_NONE,
46 FIELD_OP_PLUS,
47 FIELD_OP_MINUS,
48 FIELD_OP_UNARY_MINUS,
49};
50
30350d65
TZ		 51struct hist_var {
52 char *name;
53 struct hist_trigger_data *hist_data;
54 unsigned int idx;
55};
5819eadd 56
7ef224d1
TZ		 57struct hist_field {
58 struct ftrace_event_field *field;
59 unsigned long flags;
60 hist_field_fn_t fn;
61 unsigned int size;
76a3b0c8 62 unsigned int offset;
5819eadd 63 unsigned int is_signed;
19a9facd 64 const char *type;
5819eadd 65 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
b559d003 66 struct hist_trigger_data *hist_data;
30350d65 67 struct hist_var var;
100719dc 68 enum field_op_id operator;
067fe038
TZ		 69 char *system;
70 char *event_name;
100719dc 71 char *name;
067fe038
TZ		 72 unsigned int var_idx;
73 unsigned int var_ref_idx;
74 bool read_once;
7ef224d1
TZ		 75};
76
df35d93b
TZ		 77static u64 hist_field_none(struct hist_field *field,
78 struct tracing_map_elt *elt,
79 struct ring_buffer_event *rbe,
80 void *event)
69a0200c
TZ		 81{
82 return 0;
83}
84
df35d93b
TZ		 85static u64 hist_field_counter(struct hist_field *field,
86 struct tracing_map_elt *elt,
87 struct ring_buffer_event *rbe,
88 void *event)
7ef224d1
TZ		 89{
90 return 1;
91}
92
df35d93b
TZ		 93static u64 hist_field_string(struct hist_field *hist_field,
94 struct tracing_map_elt *elt,
95 struct ring_buffer_event *rbe,
96 void *event)
7ef224d1
TZ		 97{
98 char *addr = (char *)(event + hist_field->field->offset);
99
100 return (u64)(unsigned long)addr;
101}
102
df35d93b
TZ		 103static u64 hist_field_dynstring(struct hist_field *hist_field,
104 struct tracing_map_elt *elt,
105 struct ring_buffer_event *rbe,
106 void *event)
79e577cb
NK		 107{
108 u32 str_item = *(u32 *)(event + hist_field->field->offset);
109 int str_loc = str_item & 0xffff;
110 char *addr = (char *)(event + str_loc);
111
112 return (u64)(unsigned long)addr;
113}
114
df35d93b
TZ		 115static u64 hist_field_pstring(struct hist_field *hist_field,
116 struct tracing_map_elt *elt,
117 struct ring_buffer_event *rbe,
118 void *event)
79e577cb
NK		 119{
120 char **addr = (char **)(event + hist_field->field->offset);
121
122 return (u64)(unsigned long)*addr;
123}
124
df35d93b
TZ		 125static u64 hist_field_log2(struct hist_field *hist_field,
126 struct tracing_map_elt *elt,
127 struct ring_buffer_event *rbe,
128 void *event)
4b94f5b7 129{
5819eadd
TZ		 130 struct hist_field *operand = hist_field->operands[0];
131
df35d93b 132 u64 val = operand->fn(operand, elt, rbe, event);
4b94f5b7
NK		 133
134 return (u64) ilog2(roundup_pow_of_two(val));
135}
136
df35d93b
TZ		 137static u64 hist_field_plus(struct hist_field *hist_field,
138 struct tracing_map_elt *elt,
139 struct ring_buffer_event *rbe,
140 void *event)
100719dc
TZ		 141{
142 struct hist_field *operand1 = hist_field->operands[0];
143 struct hist_field *operand2 = hist_field->operands[1];
144
df35d93b
TZ		 145 u64 val1 = operand1->fn(operand1, elt, rbe, event);
146 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 147
148 return val1 + val2;
149}
150
df35d93b
TZ		 151static u64 hist_field_minus(struct hist_field *hist_field,
152 struct tracing_map_elt *elt,
153 struct ring_buffer_event *rbe,
154 void *event)
100719dc
TZ		 155{
156 struct hist_field *operand1 = hist_field->operands[0];
157 struct hist_field *operand2 = hist_field->operands[1];
158
df35d93b
TZ		 159 u64 val1 = operand1->fn(operand1, elt, rbe, event);
160 u64 val2 = operand2->fn(operand2, elt, rbe, event);
100719dc
TZ		 161
162 return val1 - val2;
163}
164
df35d93b
TZ		 165static u64 hist_field_unary_minus(struct hist_field *hist_field,
166 struct tracing_map_elt *elt,
167 struct ring_buffer_event *rbe,
168 void *event)
100719dc
TZ		 169{
170 struct hist_field *operand = hist_field->operands[0];
171
df35d93b 172 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
100719dc
TZ		 173 u64 val = (u64)-sval;
174
175 return val;
176}
177
7ef224d1 178#define DEFINE_HIST_FIELD_FN(type) \
fbd302cb 179 static u64 hist_field_##type(struct hist_field *hist_field, \
df35d93b
TZ		 180 struct tracing_map_elt *elt, \
181 struct ring_buffer_event *rbe, \
182 void *event) \
7ef224d1
TZ		 183{ \
184 type *addr = (type *)(event + hist_field->field->offset); \
185 \
79e577cb 186 return (u64)(unsigned long)*addr; \
7ef224d1
TZ		 187}
188
189DEFINE_HIST_FIELD_FN(s64);
190DEFINE_HIST_FIELD_FN(u64);
191DEFINE_HIST_FIELD_FN(s32);
192DEFINE_HIST_FIELD_FN(u32);
193DEFINE_HIST_FIELD_FN(s16);
194DEFINE_HIST_FIELD_FN(u16);
195DEFINE_HIST_FIELD_FN(s8);
196DEFINE_HIST_FIELD_FN(u8);
197
198#define for_each_hist_field(i, hist_data) \
199 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
200
201#define for_each_hist_val_field(i, hist_data) \
202 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
203
204#define for_each_hist_key_field(i, hist_data) \
205 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
206
69a0200c
TZ		 207#define HIST_STACKTRACE_DEPTH 16
208#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
209#define HIST_STACKTRACE_SKIP 5
210
7ef224d1 211#define HITCOUNT_IDX 0
69a0200c 212#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
7ef224d1
TZ		 213
214enum hist_field_flags {
0d7a8325
TZ		 215 HIST_FIELD_FL_HITCOUNT = 1 << 0,
216 HIST_FIELD_FL_KEY = 1 << 1,
217 HIST_FIELD_FL_STRING = 1 << 2,
218 HIST_FIELD_FL_HEX = 1 << 3,
219 HIST_FIELD_FL_SYM = 1 << 4,
220 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
221 HIST_FIELD_FL_EXECNAME = 1 << 6,
222 HIST_FIELD_FL_SYSCALL = 1 << 7,
223 HIST_FIELD_FL_STACKTRACE = 1 << 8,
224 HIST_FIELD_FL_LOG2 = 1 << 9,
ad42febe 225 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
860f9f6b 226 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
30350d65 227 HIST_FIELD_FL_VAR = 1 << 12,
100719dc 228 HIST_FIELD_FL_EXPR = 1 << 13,
067fe038 229 HIST_FIELD_FL_VAR_REF = 1 << 14,
8b7622bf 230 HIST_FIELD_FL_CPU = 1 << 15,
7e8b88a3 231 HIST_FIELD_FL_ALIAS = 1 << 16,
30350d65
TZ		 232};
233
234struct var_defs {
235 unsigned int n_vars;
236 char *name[TRACING_MAP_VARS_MAX];
237 char *expr[TRACING_MAP_VARS_MAX];
7ef224d1
TZ		 238};
239
240struct hist_trigger_attrs {
241 char *keys_str;
f2606835 242 char *vals_str;
e62347d2 243 char *sort_key_str;
5463bfda 244 char *name;
83e99914
TZ		 245 bool pause;
246 bool cont;
e86ae9ba 247 bool clear;
860f9f6b 248 bool ts_in_usecs;
7ef224d1 249 unsigned int map_bits;
30350d65
TZ		 250
251 char *assignment_str[TRACING_MAP_VARS_MAX];
252 unsigned int n_assignments;
253
0212e2aa
TZ		 254 char *action_str[HIST_ACTIONS_MAX];
255 unsigned int n_actions;
256
30350d65 257 struct var_defs var_defs;
7ef224d1
TZ		 258};
259
02205a67
TZ		 260struct field_var {
261 struct hist_field *var;
262 struct hist_field *val;
263};
264
265struct field_var_hist {
266 struct hist_trigger_data *hist_data;
267 char *cmd;
268};
269
7ef224d1 270struct hist_trigger_data {
30350d65 271 struct hist_field *fields[HIST_FIELDS_MAX];
7ef224d1
TZ		 272 unsigned int n_vals;
273 unsigned int n_keys;
274 unsigned int n_fields;
30350d65 275 unsigned int n_vars;
7ef224d1
TZ		 276 unsigned int key_size;
277 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
278 unsigned int n_sort_keys;
279 struct trace_event_file *event_file;
280 struct hist_trigger_attrs *attrs;
281 struct tracing_map *map;
ad42febe 282 bool enable_timestamps;
30350d65 283 bool remove;
067fe038
TZ		 284 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
285 unsigned int n_var_refs;
0212e2aa
TZ		 286
287 struct action_data *actions[HIST_ACTIONS_MAX];
288 unsigned int n_actions;
02205a67 289
c282a386
TZ		 290 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX];
291 unsigned int n_synth_var_refs;
02205a67
TZ		 292 struct field_var *field_vars[SYNTH_FIELDS_MAX];
293 unsigned int n_field_vars;
294 unsigned int n_field_var_str;
295 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
296 unsigned int n_field_var_hists;
50450603
TZ		 297
298 struct field_var *max_vars[SYNTH_FIELDS_MAX];
299 unsigned int n_max_vars;
300 unsigned int n_max_var_str;
0212e2aa
TZ		 301};
302
4b147936
TZ		 303struct synth_field {
304 char *type;
305 char *name;
306 size_t size;
307 bool is_signed;
308 bool is_string;
309};
310
311struct synth_event {
312 struct list_head list;
313 int ref;
314 char *name;
315 struct synth_field **fields;
316 unsigned int n_fields;
317 unsigned int n_u64;
318 struct trace_event_class class;
319 struct trace_event_call call;
320 struct tracepoint *tp;
321};
322
0212e2aa
TZ		 323struct action_data;
324
325typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
326 struct tracing_map_elt *elt, void *rec,
327 struct ring_buffer_event *rbe,
328 struct action_data *data, u64 *var_ref_vals);
329
330struct action_data {
331 action_fn_t fn;
c282a386
TZ		 332 unsigned int n_params;
333 char *params[SYNTH_FIELDS_MAX];
334
335 union {
336 struct {
337 unsigned int var_ref_idx;
338 char *match_event;
339 char *match_event_system;
340 char *synth_event_name;
341 struct synth_event *synth_event;
342 } onmatch;
50450603
TZ		 343
344 struct {
345 char *var_str;
346 char *fn_name;
347 unsigned int max_var_ref_idx;
348 struct hist_field *max_var;
349 struct hist_field *var;
350 } onmax;
c282a386 351 };
7ef224d1
TZ		 352};
353
4b147936
TZ		 354static LIST_HEAD(synth_event_list);
355static DEFINE_MUTEX(synth_event_mutex);
356
357struct synth_trace_event {
358 struct trace_entry ent;
359 u64 fields[];
360};
361
362static int synth_event_define_fields(struct trace_event_call *call)
363{
364 struct synth_trace_event trace;
365 int offset = offsetof(typeof(trace), fields);
366 struct synth_event *event = call->data;
367 unsigned int i, size, n_u64;
368 char *name, *type;
369 bool is_signed;
370 int ret = 0;
371
372 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
373 size = event->fields[i]->size;
374 is_signed = event->fields[i]->is_signed;
375 type = event->fields[i]->type;
376 name = event->fields[i]->name;
377 ret = trace_define_field(call, type, name, offset, size,
378 is_signed, FILTER_OTHER);
379 if (ret)
380 break;
381
382 if (event->fields[i]->is_string) {
383 offset += STR_VAR_LEN_MAX;
384 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
385 } else {
386 offset += sizeof(u64);
387 n_u64++;
388 }
389 }
390
391 event->n_u64 = n_u64;
392
393 return ret;
394}
395
396static bool synth_field_signed(char *type)
397{
398 if (strncmp(type, "u", 1) == 0)
399 return false;
400
401 return true;
402}
403
404static int synth_field_is_string(char *type)
405{
406 if (strstr(type, "char[") != NULL)
407 return true;
408
409 return false;
410}
411
412static int synth_field_string_size(char *type)
413{
414 char buf[4], *end, *start;
415 unsigned int len;
416 int size, err;
417
418 start = strstr(type, "char[");
419 if (start == NULL)
420 return -EINVAL;
421 start += strlen("char[");
422
423 end = strchr(type, ']');
424 if (!end || end < start)
425 return -EINVAL;
426
427 len = end - start;
428 if (len > 3)
429 return -EINVAL;
430
431 strncpy(buf, start, len);
432 buf[len] = '\0';
433
434 err = kstrtouint(buf, 0, &size);
435 if (err)
436 return err;
437
438 if (size > STR_VAR_LEN_MAX)
439 return -EINVAL;
440
441 return size;
442}
443
444static int synth_field_size(char *type)
445{
446 int size = 0;
447
448 if (strcmp(type, "s64") == 0)
449 size = sizeof(s64);
450 else if (strcmp(type, "u64") == 0)
451 size = sizeof(u64);
452 else if (strcmp(type, "s32") == 0)
453 size = sizeof(s32);
454 else if (strcmp(type, "u32") == 0)
455 size = sizeof(u32);
456 else if (strcmp(type, "s16") == 0)
457 size = sizeof(s16);
458 else if (strcmp(type, "u16") == 0)
459 size = sizeof(u16);
460 else if (strcmp(type, "s8") == 0)
461 size = sizeof(s8);
462 else if (strcmp(type, "u8") == 0)
463 size = sizeof(u8);
464 else if (strcmp(type, "char") == 0)
465 size = sizeof(char);
466 else if (strcmp(type, "unsigned char") == 0)
467 size = sizeof(unsigned char);
468 else if (strcmp(type, "int") == 0)
469 size = sizeof(int);
470 else if (strcmp(type, "unsigned int") == 0)
471 size = sizeof(unsigned int);
472 else if (strcmp(type, "long") == 0)
473 size = sizeof(long);
474 else if (strcmp(type, "unsigned long") == 0)
475 size = sizeof(unsigned long);
476 else if (strcmp(type, "pid_t") == 0)
477 size = sizeof(pid_t);
478 else if (synth_field_is_string(type))
479 size = synth_field_string_size(type);
480
481 return size;
482}
483
484static const char *synth_field_fmt(char *type)
485{
486 const char *fmt = "%llu";
487
488 if (strcmp(type, "s64") == 0)
489 fmt = "%lld";
490 else if (strcmp(type, "u64") == 0)
491 fmt = "%llu";
492 else if (strcmp(type, "s32") == 0)
493 fmt = "%d";
494 else if (strcmp(type, "u32") == 0)
495 fmt = "%u";
496 else if (strcmp(type, "s16") == 0)
497 fmt = "%d";
498 else if (strcmp(type, "u16") == 0)
499 fmt = "%u";
500 else if (strcmp(type, "s8") == 0)
501 fmt = "%d";
502 else if (strcmp(type, "u8") == 0)
503 fmt = "%u";
504 else if (strcmp(type, "char") == 0)
505 fmt = "%d";
506 else if (strcmp(type, "unsigned char") == 0)
507 fmt = "%u";
508 else if (strcmp(type, "int") == 0)
509 fmt = "%d";
510 else if (strcmp(type, "unsigned int") == 0)
511 fmt = "%u";
512 else if (strcmp(type, "long") == 0)
513 fmt = "%ld";
514 else if (strcmp(type, "unsigned long") == 0)
515 fmt = "%lu";
516 else if (strcmp(type, "pid_t") == 0)
517 fmt = "%d";
518 else if (synth_field_is_string(type))
519 fmt = "%s";
520
521 return fmt;
522}
523
524static enum print_line_t print_synth_event(struct trace_iterator *iter,
525 int flags,
526 struct trace_event *event)
527{
528 struct trace_array *tr = iter->tr;
529 struct trace_seq *s = &iter->seq;
530 struct synth_trace_event *entry;
531 struct synth_event *se;
532 unsigned int i, n_u64;
533 char print_fmt[32];
534 const char *fmt;
535
536 entry = (struct synth_trace_event *)iter->ent;
537 se = container_of(event, struct synth_event, call.event);
538
539 trace_seq_printf(s, "%s: ", se->name);
540
541 for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
542 if (trace_seq_has_overflowed(s))
543 goto end;
544
545 fmt = synth_field_fmt(se->fields[i]->type);
546
547 /* parameter types */
548 if (tr->trace_flags & TRACE_ITER_VERBOSE)
549 trace_seq_printf(s, "%s ", fmt);
550
551 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
552
553 /* parameter values */
554 if (se->fields[i]->is_string) {
555 trace_seq_printf(s, print_fmt, se->fields[i]->name,
556 (char *)&entry->fields[n_u64],
557 i == se->n_fields - 1 ? "" : " ");
558 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
559 } else {
560 trace_seq_printf(s, print_fmt, se->fields[i]->name,
561 entry->fields[n_u64],
562 i == se->n_fields - 1 ? "" : " ");
563 n_u64++;
564 }
565 }
566end:
567 trace_seq_putc(s, '\n');
568
569 return trace_handle_return(s);
570}
571
572static struct trace_event_functions synth_event_funcs = {
573 .trace = print_synth_event
574};
575
576static notrace void trace_event_raw_event_synth(void *__data,
577 u64 *var_ref_vals,
578 unsigned int var_ref_idx)
579{
580 struct trace_event_file *trace_file = __data;
581 struct synth_trace_event *entry;
582 struct trace_event_buffer fbuffer;
583 struct synth_event *event;
584 unsigned int i, n_u64;
585 int fields_size = 0;
586
587 event = trace_file->event_call->data;
588
589 if (trace_trigger_soft_disabled(trace_file))
590 return;
591
592 fields_size = event->n_u64 * sizeof(u64);
593
594 entry = trace_event_buffer_reserve(&fbuffer, trace_file,
595 sizeof(*entry) + fields_size);
596 if (!entry)
597 return;
598
599 for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
600 if (event->fields[i]->is_string) {
601 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
602 char *str_field = (char *)&entry->fields[n_u64];
603
604 strncpy(str_field, str_val, STR_VAR_LEN_MAX);
605 n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
606 } else {
607 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
608 n_u64++;
609 }
610 }
611
612 trace_event_buffer_commit(&fbuffer);
613}
614
615static void free_synth_event_print_fmt(struct trace_event_call *call)
616{
617 if (call) {
618 kfree(call->print_fmt);
619 call->print_fmt = NULL;
620 }
621}
622
623static int __set_synth_event_print_fmt(struct synth_event *event,
624 char *buf, int len)
625{
626 const char *fmt;
627 int pos = 0;
628 int i;
629
630 /* When len=0, we just calculate the needed length */
631#define LEN_OR_ZERO (len ? len - pos : 0)
632
633 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
634 for (i = 0; i < event->n_fields; i++) {
635 fmt = synth_field_fmt(event->fields[i]->type);
636 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
637 event->fields[i]->name, fmt,
638 i == event->n_fields - 1 ? "" : ", ");
639 }
640 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
641
642 for (i = 0; i < event->n_fields; i++) {
643 pos += snprintf(buf + pos, LEN_OR_ZERO,
644 ", REC->%s", event->fields[i]->name);
645 }
646
647#undef LEN_OR_ZERO
648
649 /* return the length of print_fmt */
650 return pos;
651}
652
653static int set_synth_event_print_fmt(struct trace_event_call *call)
654{
655 struct synth_event *event = call->data;
656 char *print_fmt;
657 int len;
658
659 /* First: called with 0 length to calculate the needed length */
660 len = __set_synth_event_print_fmt(event, NULL, 0);
661
662 print_fmt = kmalloc(len + 1, GFP_KERNEL);
663 if (!print_fmt)
664 return -ENOMEM;
665
666 /* Second: actually write the @print_fmt */
667 __set_synth_event_print_fmt(event, print_fmt, len + 1);
668 call->print_fmt = print_fmt;
669
670 return 0;
671}
672
673static void free_synth_field(struct synth_field *field)
674{
675 kfree(field->type);
676 kfree(field->name);
677 kfree(field);
678}
679
680static struct synth_field *parse_synth_field(char *field_type,
681 char *field_name)
682{
683 struct synth_field *field;
684 int len, ret = 0;
685 char *array;
686
687 if (field_type[0] == ';')
688 field_type++;
689
690 len = strlen(field_name);
691 if (field_name[len - 1] == ';')
692 field_name[len - 1] = '\0';
693
694 field = kzalloc(sizeof(*field), GFP_KERNEL);
695 if (!field)
696 return ERR_PTR(-ENOMEM);
697
698 len = strlen(field_type) + 1;
699 array = strchr(field_name, '[');
700 if (array)
701 len += strlen(array);
702 field->type = kzalloc(len, GFP_KERNEL);
703 if (!field->type) {
704 ret = -ENOMEM;
705 goto free;
706 }
707 strcat(field->type, field_type);
708 if (array) {
709 strcat(field->type, array);
710 *array = '\0';
711 }
712
713 field->size = synth_field_size(field->type);
714 if (!field->size) {
715 ret = -EINVAL;
716 goto free;
717 }
718
719 if (synth_field_is_string(field->type))
720 field->is_string = true;
721
722 field->is_signed = synth_field_signed(field->type);
723
724 field->name = kstrdup(field_name, GFP_KERNEL);
725 if (!field->name) {
726 ret = -ENOMEM;
727 goto free;
728 }
729 out:
730 return field;
731 free:
732 free_synth_field(field);
733 field = ERR_PTR(ret);
734 goto out;
735}
736
737static void free_synth_tracepoint(struct tracepoint *tp)
738{
739 if (!tp)
740 return;
741
742 kfree(tp->name);
743 kfree(tp);
744}
745
746static struct tracepoint *alloc_synth_tracepoint(char *name)
747{
748 struct tracepoint *tp;
749
750 tp = kzalloc(sizeof(*tp), GFP_KERNEL);
751 if (!tp)
752 return ERR_PTR(-ENOMEM);
753
754 tp->name = kstrdup(name, GFP_KERNEL);
755 if (!tp->name) {
756 kfree(tp);
757 return ERR_PTR(-ENOMEM);
758 }
759
760 return tp;
761}
762
763typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
764 unsigned int var_ref_idx);
765
766static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
767 unsigned int var_ref_idx)
768{
769 struct tracepoint *tp = event->tp;
770
771 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
772 struct tracepoint_func *probe_func_ptr;
773 synth_probe_func_t probe_func;
774 void *__data;
775
776 if (!(cpu_online(raw_smp_processor_id())))
777 return;
778
779 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
780 if (probe_func_ptr) {
781 do {
782 probe_func = probe_func_ptr->func;
783 __data = probe_func_ptr->data;
784 probe_func(__data, var_ref_vals, var_ref_idx);
785 } while ((++probe_func_ptr)->func);
786 }
787 }
788}
789
790static struct synth_event *find_synth_event(const char *name)
791{
792 struct synth_event *event;
793
794 list_for_each_entry(event, &synth_event_list, list) {
795 if (strcmp(event->name, name) == 0)
796 return event;
797 }
798
799 return NULL;
800}
801
802static int register_synth_event(struct synth_event *event)
803{
804 struct trace_event_call *call = &event->call;
805 int ret = 0;
806
807 event->call.class = &event->class;
808 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
809 if (!event->class.system) {
810 ret = -ENOMEM;
811 goto out;
812 }
813
814 event->tp = alloc_synth_tracepoint(event->name);
815 if (IS_ERR(event->tp)) {
816 ret = PTR_ERR(event->tp);
817 event->tp = NULL;
818 goto out;
819 }
820
821 INIT_LIST_HEAD(&call->class->fields);
822 call->event.funcs = &synth_event_funcs;
823 call->class->define_fields = synth_event_define_fields;
824
825 ret = register_trace_event(&call->event);
826 if (!ret) {
827 ret = -ENODEV;
828 goto out;
829 }
830 call->flags = TRACE_EVENT_FL_TRACEPOINT;
831 call->class->reg = trace_event_reg;
832 call->class->probe = trace_event_raw_event_synth;
833 call->data = event;
834 call->tp = event->tp;
835
836 ret = trace_add_event_call(call);
837 if (ret) {
838 pr_warn("Failed to register synthetic event: %s\n",
839 trace_event_name(call));
840 goto err;
841 }
842
843 ret = set_synth_event_print_fmt(call);
844 if (ret < 0) {
845 trace_remove_event_call(call);
846 goto err;
847 }
848 out:
849 return ret;
850 err:
851 unregister_trace_event(&call->event);
852 goto out;
853}
854
855static int unregister_synth_event(struct synth_event *event)
856{
857 struct trace_event_call *call = &event->call;
858 int ret;
859
860 ret = trace_remove_event_call(call);
861
862 return ret;
863}
864
865static void free_synth_event(struct synth_event *event)
866{
867 unsigned int i;
868
869 if (!event)
870 return;
871
872 for (i = 0; i < event->n_fields; i++)
873 free_synth_field(event->fields[i]);
874
875 kfree(event->fields);
876 kfree(event->name);
877 kfree(event->class.system);
878 free_synth_tracepoint(event->tp);
879 free_synth_event_print_fmt(&event->call);
880 kfree(event);
881}
882
883static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
884 struct synth_field **fields)
885{
886 struct synth_event *event;
887 unsigned int i;
888
889 event = kzalloc(sizeof(*event), GFP_KERNEL);
890 if (!event) {
891 event = ERR_PTR(-ENOMEM);
892 goto out;
893 }
894
895 event->name = kstrdup(event_name, GFP_KERNEL);
896 if (!event->name) {
897 kfree(event);
898 event = ERR_PTR(-ENOMEM);
899 goto out;
900 }
901
902 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
903 if (!event->fields) {
904 free_synth_event(event);
905 event = ERR_PTR(-ENOMEM);
906 goto out;
907 }
908
909 for (i = 0; i < n_fields; i++)
910 event->fields[i] = fields[i];
911
912 event->n_fields = n_fields;
913 out:
914 return event;
915}
916
c282a386
TZ		 917static void action_trace(struct hist_trigger_data *hist_data,
918 struct tracing_map_elt *elt, void *rec,
919 struct ring_buffer_event *rbe,
920 struct action_data *data, u64 *var_ref_vals)
921{
922 struct synth_event *event = data->onmatch.synth_event;
923
924 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
925}
926
927struct hist_var_data {
928 struct list_head list;
929 struct hist_trigger_data *hist_data;
930};
931
4b147936
TZ		 932static void add_or_delete_synth_event(struct synth_event *event, int delete)
933{
934 if (delete)
935 free_synth_event(event);
936 else {
937 mutex_lock(&synth_event_mutex);
938 if (!find_synth_event(event->name))
939 list_add(&event->list, &synth_event_list);
940 else
941 free_synth_event(event);
942 mutex_unlock(&synth_event_mutex);
943 }
944}
945
946static int create_synth_event(int argc, char **argv)
947{
948 struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
949 struct synth_event *event = NULL;
950 bool delete_event = false;
951 int i, n_fields = 0, ret = 0;
952 char *name;
953
954 mutex_lock(&synth_event_mutex);
955
956 /*
957 * Argument syntax:
958 * - Add synthetic event: <event_name> field[;field] ...
959 * - Remove synthetic event: !<event_name> field[;field] ...
960 * where 'field' = type field_name
961 */
962 if (argc < 1) {
963 ret = -EINVAL;
964 goto out;
965 }
966
967 name = argv[0];
968 if (name[0] == '!') {
969 delete_event = true;
970 name++;
971 }
972
973 event = find_synth_event(name);
974 if (event) {
975 if (delete_event) {
976 if (event->ref) {
977 event = NULL;
978 ret = -EBUSY;
979 goto out;
980 }
981 list_del(&event->list);
982 goto out;
983 }
984 event = NULL;
985 ret = -EEXIST;
986 goto out;
987 } else if (delete_event)
988 goto out;
989
990 if (argc < 2) {
991 ret = -EINVAL;
992 goto out;
993 }
994
995 for (i = 1; i < argc - 1; i++) {
996 if (strcmp(argv[i], ";") == 0)
997 continue;
998 if (n_fields == SYNTH_FIELDS_MAX) {
999 ret = -EINVAL;
1000 goto err;
1001 }
1002
1003 field = parse_synth_field(argv[i], argv[i + 1]);
1004 if (IS_ERR(field)) {
1005 ret = PTR_ERR(field);
1006 goto err;
1007 }
1008 fields[n_fields] = field;
1009 i++; n_fields++;
1010 }
1011
1012 if (i < argc) {
1013 ret = -EINVAL;
1014 goto err;
1015 }
1016
1017 event = alloc_synth_event(name, n_fields, fields);
1018 if (IS_ERR(event)) {
1019 ret = PTR_ERR(event);
1020 event = NULL;
1021 goto err;
1022 }
1023 out:
1024 mutex_unlock(&synth_event_mutex);
1025
1026 if (event) {
1027 if (delete_event) {
1028 ret = unregister_synth_event(event);
1029 add_or_delete_synth_event(event, !ret);
1030 } else {
1031 ret = register_synth_event(event);
1032 add_or_delete_synth_event(event, ret);
1033 }
1034 }
1035
1036 return ret;
1037 err:
1038 mutex_unlock(&synth_event_mutex);
1039
1040 for (i = 0; i < n_fields; i++)
1041 free_synth_field(fields[i]);
1042 free_synth_event(event);
1043
1044 return ret;
1045}
1046
1047static int release_all_synth_events(void)
1048{
1049 struct list_head release_events;
1050 struct synth_event *event, *e;
1051 int ret = 0;
1052
1053 INIT_LIST_HEAD(&release_events);
1054
1055 mutex_lock(&synth_event_mutex);
1056
1057 list_for_each_entry(event, &synth_event_list, list) {
1058 if (event->ref) {
1059 mutex_unlock(&synth_event_mutex);
1060 return -EBUSY;
1061 }
1062 }
1063
1064 list_splice_init(&event->list, &release_events);
1065
1066 mutex_unlock(&synth_event_mutex);
1067
1068 list_for_each_entry_safe(event, e, &release_events, list) {
1069 list_del(&event->list);
1070
1071 ret = unregister_synth_event(event);
1072 add_or_delete_synth_event(event, !ret);
1073 }
1074
1075 return ret;
1076}
1077
1078
1079static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1080{
1081 mutex_lock(&synth_event_mutex);
1082
1083 return seq_list_start(&synth_event_list, *pos);
1084}
1085
1086static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1087{
1088 return seq_list_next(v, &synth_event_list, pos);
1089}
1090
1091static void synth_events_seq_stop(struct seq_file *m, void *v)
1092{
1093 mutex_unlock(&synth_event_mutex);
1094}
1095
1096static int synth_events_seq_show(struct seq_file *m, void *v)
1097{
1098 struct synth_field *field;
1099 struct synth_event *event = v;
1100 unsigned int i;
1101
1102 seq_printf(m, "%s\t", event->name);
1103
1104 for (i = 0; i < event->n_fields; i++) {
1105 field = event->fields[i];
1106
1107 /* parameter values */
1108 seq_printf(m, "%s %s%s", field->type, field->name,
1109 i == event->n_fields - 1 ? "" : "; ");
1110 }
1111
1112 seq_putc(m, '\n');
1113
1114 return 0;
1115}
1116
1117static const struct seq_operations synth_events_seq_op = {
1118 .start = synth_events_seq_start,
1119 .next = synth_events_seq_next,
1120 .stop = synth_events_seq_stop,
1121 .show = synth_events_seq_show
1122};
1123
1124static int synth_events_open(struct inode *inode, struct file *file)
1125{
1126 int ret;
1127
1128 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1129 ret = release_all_synth_events();
1130 if (ret < 0)
1131 return ret;
1132 }
1133
1134 return seq_open(file, &synth_events_seq_op);
1135}
1136
1137static ssize_t synth_events_write(struct file *file,
1138 const char __user *buffer,
1139 size_t count, loff_t *ppos)
1140{
1141 return trace_parse_run_command(file, buffer, count, ppos,
1142 create_synth_event);
1143}
1144
1145static const struct file_operations synth_events_fops = {
1146 .open = synth_events_open,
1147 .write = synth_events_write,
1148 .read = seq_read,
1149 .llseek = seq_lseek,
1150 .release = seq_release,
1151};
1152
df35d93b
TZ		 1153static u64 hist_field_timestamp(struct hist_field *hist_field,
1154 struct tracing_map_elt *elt,
1155 struct ring_buffer_event *rbe,
1156 void *event)
860f9f6b
TZ		 1157{
1158 struct hist_trigger_data *hist_data = hist_field->hist_data;
1159 struct trace_array *tr = hist_data->event_file->tr;
1160
1161 u64 ts = ring_buffer_event_time_stamp(rbe);
1162
1163 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1164 ts = ns2usecs(ts);
1165
1166 return ts;
1167}
1168
8b7622bf
TZ		 1169static u64 hist_field_cpu(struct hist_field *hist_field,
1170 struct tracing_map_elt *elt,
1171 struct ring_buffer_event *rbe,
1172 void *event)
1173{
1174 int cpu = smp_processor_id();
1175
1176 return cpu;
1177}
1178
067fe038
TZ		 1179static struct hist_field *
1180check_field_for_var_ref(struct hist_field *hist_field,
1181 struct hist_trigger_data *var_data,
1182 unsigned int var_idx)
1183{
1184 struct hist_field *found = NULL;
1185
1186 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1187 if (hist_field->var.idx == var_idx &&
1188 hist_field->var.hist_data == var_data) {
1189 found = hist_field;
1190 }
1191 }
1192
1193 return found;
1194}
1195
1196static struct hist_field *
1197check_field_for_var_refs(struct hist_trigger_data *hist_data,
1198 struct hist_field *hist_field,
1199 struct hist_trigger_data *var_data,
1200 unsigned int var_idx,
1201 unsigned int level)
1202{
1203 struct hist_field *found = NULL;
1204 unsigned int i;
1205
1206 if (level > 3)
1207 return found;
1208
1209 if (!hist_field)
1210 return found;
1211
1212 found = check_field_for_var_ref(hist_field, var_data, var_idx);
1213 if (found)
1214 return found;
1215
1216 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1217 struct hist_field *operand;
1218
1219 operand = hist_field->operands[i];
1220 found = check_field_for_var_refs(hist_data, operand, var_data,
1221 var_idx, level + 1);
1222 if (found)
1223 return found;
1224 }
1225
1226 return found;
1227}
1228
1229static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1230 struct hist_trigger_data *var_data,
1231 unsigned int var_idx)
1232{
1233 struct hist_field *hist_field, *found = NULL;
1234 unsigned int i;
1235
1236 for_each_hist_field(i, hist_data) {
1237 hist_field = hist_data->fields[i];
1238 found = check_field_for_var_refs(hist_data, hist_field,
1239 var_data, var_idx, 0);
1240 if (found)
1241 return found;
1242 }
1243
c282a386
TZ		 1244 for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1245 hist_field = hist_data->synth_var_refs[i];
1246 found = check_field_for_var_refs(hist_data, hist_field,
1247 var_data, var_idx, 0);
1248 if (found)
1249 return found;
1250 }
1251
067fe038
TZ		 1252 return found;
1253}
1254
1255static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1256 unsigned int var_idx)
1257{
1258 struct trace_array *tr = hist_data->event_file->tr;
1259 struct hist_field *found = NULL;
1260 struct hist_var_data *var_data;
1261
1262 list_for_each_entry(var_data, &tr->hist_vars, list) {
1263 if (var_data->hist_data == hist_data)
1264 continue;
1265 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1266 if (found)
1267 break;
1268 }
1269
1270 return found;
1271}
1272
1273static bool check_var_refs(struct hist_trigger_data *hist_data)
1274{
1275 struct hist_field *field;
1276 bool found = false;
1277 int i;
1278
1279 for_each_hist_field(i, hist_data) {
1280 field = hist_data->fields[i];
1281 if (field && field->flags & HIST_FIELD_FL_VAR) {
1282 if (find_any_var_ref(hist_data, field->var.idx)) {
1283 found = true;
1284 break;
1285 }
1286 }
1287 }
1288
1289 return found;
1290}
1291
1292static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1293{
1294 struct trace_array *tr = hist_data->event_file->tr;
1295 struct hist_var_data *var_data, *found = NULL;
1296
1297 list_for_each_entry(var_data, &tr->hist_vars, list) {
1298 if (var_data->hist_data == hist_data) {
1299 found = var_data;
1300 break;
1301 }
1302 }
1303
1304 return found;
1305}
1306
1307static bool field_has_hist_vars(struct hist_field *hist_field,
1308 unsigned int level)
1309{
1310 int i;
1311
1312 if (level > 3)
1313 return false;
1314
1315 if (!hist_field)
1316 return false;
1317
1318 if (hist_field->flags & HIST_FIELD_FL_VAR ||
1319 hist_field->flags & HIST_FIELD_FL_VAR_REF)
1320 return true;
1321
1322 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1323 struct hist_field *operand;
1324
1325 operand = hist_field->operands[i];
1326 if (field_has_hist_vars(operand, level + 1))
1327 return true;
1328 }
1329
1330 return false;
1331}
1332
1333static bool has_hist_vars(struct hist_trigger_data *hist_data)
1334{
1335 struct hist_field *hist_field;
1336 int i;
1337
1338 for_each_hist_field(i, hist_data) {
1339 hist_field = hist_data->fields[i];
1340 if (field_has_hist_vars(hist_field, 0))
1341 return true;
1342 }
1343
1344 return false;
1345}
1346
1347static int save_hist_vars(struct hist_trigger_data *hist_data)
1348{
1349 struct trace_array *tr = hist_data->event_file->tr;
1350 struct hist_var_data *var_data;
1351
1352 var_data = find_hist_vars(hist_data);
1353 if (var_data)
1354 return 0;
1355
1356 if (trace_array_get(tr) < 0)
1357 return -ENODEV;
1358
1359 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1360 if (!var_data) {
1361 trace_array_put(tr);
1362 return -ENOMEM;
1363 }
1364
1365 var_data->hist_data = hist_data;
1366 list_add(&var_data->list, &tr->hist_vars);
1367
1368 return 0;
1369}
1370
1371static void remove_hist_vars(struct hist_trigger_data *hist_data)
1372{
1373 struct trace_array *tr = hist_data->event_file->tr;
1374 struct hist_var_data *var_data;
1375
1376 var_data = find_hist_vars(hist_data);
1377 if (!var_data)
1378 return;
1379
1380 if (WARN_ON(check_var_refs(hist_data)))
1381 return;
1382
1383 list_del(&var_data->list);
1384
1385 kfree(var_data);
1386
1387 trace_array_put(tr);
1388}
1389
30350d65
TZ		 1390static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1391 const char *var_name)
1392{
1393 struct hist_field *hist_field, *found = NULL;
1394 int i;
1395
1396 for_each_hist_field(i, hist_data) {
1397 hist_field = hist_data->fields[i];
1398 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1399 strcmp(hist_field->var.name, var_name) == 0) {
1400 found = hist_field;
1401 break;
1402 }
1403 }
1404
1405 return found;
1406}
1407
1408static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1409 struct trace_event_file *file,
1410 const char *var_name)
1411{
1412 struct hist_trigger_data *test_data;
1413 struct event_trigger_data *test;
1414 struct hist_field *hist_field;
1415
1416 hist_field = find_var_field(hist_data, var_name);
1417 if (hist_field)
1418 return hist_field;
1419
1420 list_for_each_entry_rcu(test, &file->triggers, list) {
1421 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1422 test_data = test->private_data;
1423 hist_field = find_var_field(test_data, var_name);
1424 if (hist_field)
1425 return hist_field;
1426 }
1427 }
1428
1429 return NULL;
1430}
1431
067fe038
TZ		 1432static struct trace_event_file *find_var_file(struct trace_array *tr,
1433 char *system,
1434 char *event_name,
1435 char *var_name)
1436{
1437 struct hist_trigger_data *var_hist_data;
1438 struct hist_var_data *var_data;
1439 struct trace_event_file *file, *found = NULL;
1440
1441 if (system)
1442 return find_event_file(tr, system, event_name);
1443
1444 list_for_each_entry(var_data, &tr->hist_vars, list) {
1445 var_hist_data = var_data->hist_data;
1446 file = var_hist_data->event_file;
1447 if (file == found)
1448 continue;
1449
1450 if (find_var_field(var_hist_data, var_name)) {
1451 if (found)
1452 return NULL;
1453
1454 found = file;
1455 }
1456 }
1457
1458 return found;
1459}
1460
1461static struct hist_field *find_file_var(struct trace_event_file *file,
1462 const char *var_name)
1463{
1464 struct hist_trigger_data *test_data;
1465 struct event_trigger_data *test;
1466 struct hist_field *hist_field;
1467
1468 list_for_each_entry_rcu(test, &file->triggers, list) {
1469 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1470 test_data = test->private_data;
1471 hist_field = find_var_field(test_data, var_name);
1472 if (hist_field)
1473 return hist_field;
1474 }
1475 }
1476
1477 return NULL;
1478}
1479
c282a386
TZ		 1480static struct hist_field *
1481find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1482{
1483 struct trace_array *tr = hist_data->event_file->tr;
1484 struct hist_field *hist_field, *found = NULL;
1485 struct trace_event_file *file;
1486 unsigned int i;
1487
1488 for (i = 0; i < hist_data->n_actions; i++) {
1489 struct action_data *data = hist_data->actions[i];
1490
1491 if (data->fn == action_trace) {
1492 char *system = data->onmatch.match_event_system;
1493 char *event_name = data->onmatch.match_event;
1494
1495 file = find_var_file(tr, system, event_name, var_name);
1496 if (!file)
1497 continue;
1498 hist_field = find_file_var(file, var_name);
1499 if (hist_field) {
1500 if (found) {
1501 return ERR_PTR(-EINVAL);
1502 }
1503
1504 found = hist_field;
1505 }
1506 }
1507 }
1508 return found;
1509}
1510
067fe038
TZ		 1511static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1512 char *system,
1513 char *event_name,
1514 char *var_name)
1515{
1516 struct trace_array *tr = hist_data->event_file->tr;
1517 struct hist_field *hist_field = NULL;
1518 struct trace_event_file *file;
1519
c282a386
TZ		 1520 if (!system || !event_name) {
1521 hist_field = find_match_var(hist_data, var_name);
1522 if (IS_ERR(hist_field))
1523 return NULL;
1524 if (hist_field)
1525 return hist_field;
1526 }
1527
067fe038
TZ		 1528 file = find_var_file(tr, system, event_name, var_name);
1529 if (!file)
1530 return NULL;
1531
1532 hist_field = find_file_var(file, var_name);
1533
1534 return hist_field;
1535}
1536
af6a29bc
TZ		 1537struct hist_elt_data {
1538 char *comm;
067fe038 1539 u64 *var_ref_vals;
02205a67 1540 char *field_var_str[SYNTH_FIELDS_MAX];
af6a29bc
TZ		 1541};
1542
067fe038
TZ		 1543static u64 hist_field_var_ref(struct hist_field *hist_field,
1544 struct tracing_map_elt *elt,
1545 struct ring_buffer_event *rbe,
1546 void *event)
1547{
1548 struct hist_elt_data *elt_data;
1549 u64 var_val = 0;
1550
1551 elt_data = elt->private_data;
1552 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1553
1554 return var_val;
1555}
1556
1557static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1558 u64 *var_ref_vals, bool self)
1559{
1560 struct hist_trigger_data *var_data;
1561 struct tracing_map_elt *var_elt;
1562 struct hist_field *hist_field;
1563 unsigned int i, var_idx;
1564 bool resolved = true;
1565 u64 var_val = 0;
1566
1567 for (i = 0; i < hist_data->n_var_refs; i++) {
1568 hist_field = hist_data->var_refs[i];
1569 var_idx = hist_field->var.idx;
1570 var_data = hist_field->var.hist_data;
1571
1572 if (var_data == NULL) {
1573 resolved = false;
1574 break;
1575 }
1576
1577 if ((self && var_data != hist_data) ||
1578 (!self && var_data == hist_data))
1579 continue;
1580
1581 var_elt = tracing_map_lookup(var_data->map, key);
1582 if (!var_elt) {
1583 resolved = false;
1584 break;
1585 }
1586
1587 if (!tracing_map_var_set(var_elt, var_idx)) {
1588 resolved = false;
1589 break;
1590 }
1591
1592 if (self || !hist_field->read_once)
1593 var_val = tracing_map_read_var(var_elt, var_idx);
1594 else
1595 var_val = tracing_map_read_var_once(var_elt, var_idx);
1596
1597 var_ref_vals[i] = var_val;
1598 }
1599
1600 return resolved;
1601}
1602
85013256
TZ		 1603static const char *hist_field_name(struct hist_field *field,
1604 unsigned int level)
1605{
1606 const char *field_name = "";
1607
1608 if (level > 1)
1609 return field_name;
1610
1611 if (field->field)
1612 field_name = field->field->name;
7e8b88a3
TZ		 1613 else if (field->flags & HIST_FIELD_FL_LOG2 ||
1614 field->flags & HIST_FIELD_FL_ALIAS)
5819eadd 1615 field_name = hist_field_name(field->operands[0], ++level);
ad42febe
TZ		 1616 else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1617 field_name = "common_timestamp";
8b7622bf
TZ		 1618 else if (field->flags & HIST_FIELD_FL_CPU)
1619 field_name = "cpu";
067fe038
TZ		 1620 else if (field->flags & HIST_FIELD_FL_EXPR ||
1621 field->flags & HIST_FIELD_FL_VAR_REF) {
1622 if (field->system) {
1623 static char full_name[MAX_FILTER_STR_VAL];
1624
1625 strcat(full_name, field->system);
1626 strcat(full_name, ".");
1627 strcat(full_name, field->event_name);
1628 strcat(full_name, ".");
1629 strcat(full_name, field->name);
1630 field_name = full_name;
1631 } else
1632 field_name = field->name;
1633 }
85013256
TZ		 1634
1635 if (field_name == NULL)
1636 field_name = "";
1637
1638 return field_name;
1639}
1640
7ef224d1
TZ		 1641static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1642{
1643 hist_field_fn_t fn = NULL;
1644
1645 switch (field_size) {
1646 case 8:
1647 if (field_is_signed)
1648 fn = hist_field_s64;
1649 else
1650 fn = hist_field_u64;
1651 break;
1652 case 4:
1653 if (field_is_signed)
1654 fn = hist_field_s32;
1655 else
1656 fn = hist_field_u32;
1657 break;
1658 case 2:
1659 if (field_is_signed)
1660 fn = hist_field_s16;
1661 else
1662 fn = hist_field_u16;
1663 break;
1664 case 1:
1665 if (field_is_signed)
1666 fn = hist_field_s8;
1667 else
1668 fn = hist_field_u8;
1669 break;
1670 }
1671
1672 return fn;
1673}
1674
1675static int parse_map_size(char *str)
1676{
1677 unsigned long size, map_bits;
1678 int ret;
1679
1680 strsep(&str, "=");
1681 if (!str) {
1682 ret = -EINVAL;
1683 goto out;
1684 }
1685
1686 ret = kstrtoul(str, 0, &size);
1687 if (ret)
1688 goto out;
1689
1690 map_bits = ilog2(roundup_pow_of_two(size));
1691 if (map_bits < TRACING_MAP_BITS_MIN ||
1692 map_bits > TRACING_MAP_BITS_MAX)
1693 ret = -EINVAL;
1694 else
1695 ret = map_bits;
1696 out:
1697 return ret;
1698}
1699
1700static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1701{
30350d65
TZ		 1702 unsigned int i;
1703
7ef224d1
TZ		 1704 if (!attrs)
1705 return;
1706
30350d65
TZ		 1707 for (i = 0; i < attrs->n_assignments; i++)
1708 kfree(attrs->assignment_str[i]);
1709
0212e2aa
TZ		 1710 for (i = 0; i < attrs->n_actions; i++)
1711 kfree(attrs->action_str[i]);
1712
5463bfda 1713 kfree(attrs->name);
e62347d2 1714 kfree(attrs->sort_key_str);
7ef224d1 1715 kfree(attrs->keys_str);
f2606835 1716 kfree(attrs->vals_str);
7ef224d1
TZ		 1717 kfree(attrs);
1718}
1719
0212e2aa
TZ		 1720static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1721{
c282a386 1722 int ret = -EINVAL;
0212e2aa
TZ		 1723
1724 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1725 return ret;
1726
50450603
TZ		 1727 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1728 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
c282a386
TZ		 1729 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1730 if (!attrs->action_str[attrs->n_actions]) {
1731 ret = -ENOMEM;
1732 return ret;
1733 }
1734 attrs->n_actions++;
1735 ret = 0;
1736 }
1737
0212e2aa
TZ		 1738 return ret;
1739}
1740
9b1ae035
TZ		 1741static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1742{
1743 int ret = 0;
1744
1745 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1746 (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1747 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1748 if (!attrs->keys_str) {
1749 ret = -ENOMEM;
1750 goto out;
1751 }
1752 } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1753 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1754 (strncmp(str, "values=", strlen("values=")) == 0)) {
1755 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1756 if (!attrs->vals_str) {
1757 ret = -ENOMEM;
1758 goto out;
1759 }
1760 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1761 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1762 if (!attrs->sort_key_str) {
1763 ret = -ENOMEM;
1764 goto out;
1765 }
1766 } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1767 attrs->name = kstrdup(str, GFP_KERNEL);
1768 if (!attrs->name) {
1769 ret = -ENOMEM;
1770 goto out;
1771 }
1772 } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1773 int map_bits = parse_map_size(str);
1774
1775 if (map_bits < 0) {
1776 ret = map_bits;
1777 goto out;
1778 }
1779 attrs->map_bits = map_bits;
30350d65
TZ		 1780 } else {
1781 char *assignment;
1782
1783 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1784 ret = -EINVAL;
1785 goto out;
1786 }
1787
1788 assignment = kstrdup(str, GFP_KERNEL);
1789 if (!assignment) {
1790 ret = -ENOMEM;
1791 goto out;
1792 }
1793
1794 attrs->assignment_str[attrs->n_assignments++] = assignment;
1795 }
9b1ae035
TZ		 1796 out:
1797 return ret;
1798}
1799
7ef224d1
TZ		 1800static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1801{
1802 struct hist_trigger_attrs *attrs;
1803 int ret = 0;
1804
1805 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1806 if (!attrs)
1807 return ERR_PTR(-ENOMEM);
1808
1809 while (trigger_str) {
1810 char *str = strsep(&trigger_str, ":");
1811
9b1ae035
TZ		 1812 if (strchr(str, '=')) {
1813 ret = parse_assignment(str, attrs);
1814 if (ret)
1815 goto free;
1816 } else if (strcmp(str, "pause") == 0)
83e99914
TZ		 1817 attrs->pause = true;
1818 else if ((strcmp(str, "cont") == 0) ||
1819 (strcmp(str, "continue") == 0))
1820 attrs->cont = true;
e86ae9ba
TZ		 1821 else if (strcmp(str, "clear") == 0)
1822 attrs->clear = true;
9b1ae035 1823 else {
0212e2aa
TZ		 1824 ret = parse_action(str, attrs);
1825 if (ret)
1826 goto free;
7ef224d1
TZ		 1827 }
1828 }
1829
1830 if (!attrs->keys_str) {
1831 ret = -EINVAL;
1832 goto free;
1833 }
1834
1835 return attrs;
1836 free:
1837 destroy_hist_trigger_attrs(attrs);
1838
1839 return ERR_PTR(ret);
1840}
1841
6b4827ad
TZ		 1842static inline void save_comm(char *comm, struct task_struct *task)
1843{
1844 if (!task->pid) {
1845 strcpy(comm, "<idle>");
1846 return;
1847 }
1848
1849 if (WARN_ON_ONCE(task->pid < 0)) {
1850 strcpy(comm, "<XXX>");
1851 return;
1852 }
1853
1854 memcpy(comm, task->comm, TASK_COMM_LEN);
1855}
1856
af6a29bc
TZ		 1857static void hist_elt_data_free(struct hist_elt_data *elt_data)
1858{
02205a67
TZ		 1859 unsigned int i;
1860
1861 for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1862 kfree(elt_data->field_var_str[i]);
1863
af6a29bc
TZ		 1864 kfree(elt_data->comm);
1865 kfree(elt_data);
1866}
1867
1868static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
6b4827ad 1869{
af6a29bc
TZ		 1870 struct hist_elt_data *elt_data = elt->private_data;
1871
1872 hist_elt_data_free(elt_data);
6b4827ad
TZ		 1873}
1874
af6a29bc 1875static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
6b4827ad
TZ		 1876{
1877 struct hist_trigger_data *hist_data = elt->map->private_data;
af6a29bc
TZ		 1878 unsigned int size = TASK_COMM_LEN;
1879 struct hist_elt_data *elt_data;
6b4827ad 1880 struct hist_field *key_field;
02205a67 1881 unsigned int i, n_str;
6b4827ad 1882
af6a29bc
TZ		 1883 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1884 if (!elt_data)
1885 return -ENOMEM;
1886
6b4827ad
TZ		 1887 for_each_hist_key_field(i, hist_data) {
1888 key_field = hist_data->fields[i];
1889
1890 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 1891 elt_data->comm = kzalloc(size, GFP_KERNEL);
1892 if (!elt_data->comm) {
1893 kfree(elt_data);
6b4827ad 1894 return -ENOMEM;
af6a29bc 1895 }
6b4827ad
TZ		 1896 break;
1897 }
1898 }
1899
50450603 1900 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
02205a67
TZ		 1901
1902 size = STR_VAR_LEN_MAX;
1903
1904 for (i = 0; i < n_str; i++) {
1905 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1906 if (!elt_data->field_var_str[i]) {
1907 hist_elt_data_free(elt_data);
1908 return -ENOMEM;
1909 }
1910 }
1911
af6a29bc
TZ		 1912 elt->private_data = elt_data;
1913
6b4827ad
TZ		 1914 return 0;
1915}
1916
af6a29bc 1917static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
6b4827ad 1918{
af6a29bc 1919 struct hist_elt_data *elt_data = elt->private_data;
6b4827ad 1920
af6a29bc
TZ		 1921 if (elt_data->comm)
1922 save_comm(elt_data->comm, current);
6b4827ad
TZ		 1923}
1924
af6a29bc
TZ		 1925static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1926 .elt_alloc = hist_trigger_elt_data_alloc,
1927 .elt_free = hist_trigger_elt_data_free,
1928 .elt_init = hist_trigger_elt_data_init,
6b4827ad
TZ		 1929};
1930
2ece94fb
TZ		 1931static const char *get_hist_field_flags(struct hist_field *hist_field)
1932{
1933 const char *flags_str = NULL;
1934
1935 if (hist_field->flags & HIST_FIELD_FL_HEX)
1936 flags_str = "hex";
1937 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1938 flags_str = "sym";
1939 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1940 flags_str = "sym-offset";
1941 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1942 flags_str = "execname";
1943 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1944 flags_str = "syscall";
1945 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1946 flags_str = "log2";
1947 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1948 flags_str = "usecs";
1949
1950 return flags_str;
1951}
1952
100719dc
TZ		 1953static void expr_field_str(struct hist_field *field, char *expr)
1954{
067fe038
TZ		 1955 if (field->flags & HIST_FIELD_FL_VAR_REF)
1956 strcat(expr, "$");
1957
100719dc
TZ		 1958 strcat(expr, hist_field_name(field, 0));
1959
1960 if (field->flags) {
1961 const char *flags_str = get_hist_field_flags(field);
1962
1963 if (flags_str) {
1964 strcat(expr, ".");
1965 strcat(expr, flags_str);
1966 }
1967 }
1968}
1969
1970static char *expr_str(struct hist_field *field, unsigned int level)
1971{
1972 char *expr;
1973
1974 if (level > 1)
1975 return NULL;
1976
1977 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1978 if (!expr)
1979 return NULL;
1980
1981 if (!field->operands[0]) {
1982 expr_field_str(field, expr);
1983 return expr;
1984 }
1985
1986 if (field->operator == FIELD_OP_UNARY_MINUS) {
1987 char *subexpr;
1988
1989 strcat(expr, "-(");
1990 subexpr = expr_str(field->operands[0], ++level);
1991 if (!subexpr) {
1992 kfree(expr);
1993 return NULL;
1994 }
1995 strcat(expr, subexpr);
1996 strcat(expr, ")");
1997
1998 kfree(subexpr);
1999
2000 return expr;
2001 }
2002
2003 expr_field_str(field->operands[0], expr);
2004
2005 switch (field->operator) {
2006 case FIELD_OP_MINUS:
2007 strcat(expr, "-");
2008 break;
2009 case FIELD_OP_PLUS:
2010 strcat(expr, "+");
2011 break;
2012 default:
2013 kfree(expr);
2014 return NULL;
2015 }
2016
2017 expr_field_str(field->operands[1], expr);
2018
2019 return expr;
2020}
2021
2022static int contains_operator(char *str)
2023{
2024 enum field_op_id field_op = FIELD_OP_NONE;
2025 char *op;
2026
2027 op = strpbrk(str, "+-");
2028 if (!op)
2029 return FIELD_OP_NONE;
2030
2031 switch (*op) {
2032 case '-':
2033 if (*str == '-')
2034 field_op = FIELD_OP_UNARY_MINUS;
2035 else
2036 field_op = FIELD_OP_MINUS;
2037 break;
2038 case '+':
2039 field_op = FIELD_OP_PLUS;
2040 break;
2041 default:
2042 break;
2043 }
2044
2045 return field_op;
2046}
2047
5819eadd
TZ		 2048static void destroy_hist_field(struct hist_field *hist_field,
2049 unsigned int level)
7ef224d1 2050{
5819eadd
TZ		 2051 unsigned int i;
2052
100719dc 2053 if (level > 3)
5819eadd
TZ		 2054 return;
2055
2056 if (!hist_field)
2057 return;
2058
2059 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2060 destroy_hist_field(hist_field->operands[i], level + 1);
2061
30350d65 2062 kfree(hist_field->var.name);
100719dc 2063 kfree(hist_field->name);
19a9facd 2064 kfree(hist_field->type);
30350d65 2065
7ef224d1
TZ		 2066 kfree(hist_field);
2067}
2068
b559d003
TZ		 2069static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2070 struct ftrace_event_field *field,
30350d65
TZ		 2071 unsigned long flags,
2072 char *var_name)
7ef224d1
TZ		 2073{
2074 struct hist_field *hist_field;
2075
2076 if (field && is_function_field(field))
2077 return NULL;
2078
2079 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2080 if (!hist_field)
2081 return NULL;
2082
b559d003
TZ		 2083 hist_field->hist_data = hist_data;
2084
7e8b88a3 2085 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
100719dc
TZ		 2086 goto out; /* caller will populate */
2087
067fe038
TZ		 2088 if (flags & HIST_FIELD_FL_VAR_REF) {
2089 hist_field->fn = hist_field_var_ref;
2090 goto out;
2091 }
2092
7ef224d1
TZ		 2093 if (flags & HIST_FIELD_FL_HITCOUNT) {
2094 hist_field->fn = hist_field_counter;
19a9facd
TZ		 2095 hist_field->size = sizeof(u64);
2096 hist_field->type = kstrdup("u64", GFP_KERNEL);
2097 if (!hist_field->type)
2098 goto free;
7ef224d1
TZ		 2099 goto out;
2100 }
2101
69a0200c
TZ		 2102 if (flags & HIST_FIELD_FL_STACKTRACE) {
2103 hist_field->fn = hist_field_none;
2104 goto out;
2105 }
2106
4b94f5b7 2107 if (flags & HIST_FIELD_FL_LOG2) {
5819eadd 2108 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
4b94f5b7 2109 hist_field->fn = hist_field_log2;
30350d65 2110 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
5819eadd 2111 hist_field->size = hist_field->operands[0]->size;
19a9facd
TZ		 2112 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2113 if (!hist_field->type)
2114 goto free;
4b94f5b7
NK		 2115 goto out;
2116 }
2117
ad42febe
TZ		 2118 if (flags & HIST_FIELD_FL_TIMESTAMP) {
2119 hist_field->fn = hist_field_timestamp;
2120 hist_field->size = sizeof(u64);
19a9facd
TZ		 2121 hist_field->type = kstrdup("u64", GFP_KERNEL);
2122 if (!hist_field->type)
2123 goto free;
ad42febe
TZ		 2124 goto out;
2125 }
2126
8b7622bf
TZ		 2127 if (flags & HIST_FIELD_FL_CPU) {
2128 hist_field->fn = hist_field_cpu;
2129 hist_field->size = sizeof(int);
2130 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2131 if (!hist_field->type)
2132 goto free;
2133 goto out;
2134 }
2135
432480c5
TZ		 2136 if (WARN_ON_ONCE(!field))
2137 goto out;
2138
7ef224d1
TZ		 2139 if (is_string_field(field)) {
2140 flags |= HIST_FIELD_FL_STRING;
79e577cb 2141
19a9facd
TZ		 2142 hist_field->size = MAX_FILTER_STR_VAL;
2143 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2144 if (!hist_field->type)
2145 goto free;
2146
79e577cb
NK		 2147 if (field->filter_type == FILTER_STATIC_STRING)
2148 hist_field->fn = hist_field_string;
2149 else if (field->filter_type == FILTER_DYN_STRING)
2150 hist_field->fn = hist_field_dynstring;
2151 else
2152 hist_field->fn = hist_field_pstring;
7ef224d1 2153 } else {
19a9facd
TZ		 2154 hist_field->size = field->size;
2155 hist_field->is_signed = field->is_signed;
2156 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2157 if (!hist_field->type)
2158 goto free;
2159
7ef224d1
TZ		 2160 hist_field->fn = select_value_fn(field->size,
2161 field->is_signed);
2162 if (!hist_field->fn) {
5819eadd 2163 destroy_hist_field(hist_field, 0);
7ef224d1
TZ		 2164 return NULL;
2165 }
2166 }
2167 out:
2168 hist_field->field = field;
2169 hist_field->flags = flags;
2170
30350d65
TZ		 2171 if (var_name) {
2172 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2173 if (!hist_field->var.name)
2174 goto free;
2175 }
2176
7ef224d1 2177 return hist_field;
30350d65
TZ		 2178 free:
2179 destroy_hist_field(hist_field, 0);
2180 return NULL;
7ef224d1
TZ		 2181}
2182
2183static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2184{
2185 unsigned int i;
2186
30350d65 2187 for (i = 0; i < HIST_FIELDS_MAX; i++) {
7ef224d1 2188 if (hist_data->fields[i]) {
5819eadd 2189 destroy_hist_field(hist_data->fields[i], 0);
7ef224d1
TZ		 2190 hist_data->fields[i] = NULL;
2191 }
2192 }
2193}
2194
067fe038
TZ		 2195static int init_var_ref(struct hist_field *ref_field,
2196 struct hist_field *var_field,
2197 char *system, char *event_name)
2198{
2199 int err = 0;
2200
2201 ref_field->var.idx = var_field->var.idx;
2202 ref_field->var.hist_data = var_field->hist_data;
2203 ref_field->size = var_field->size;
2204 ref_field->is_signed = var_field->is_signed;
2205 ref_field->flags |= var_field->flags &
2206 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2207
2208 if (system) {
2209 ref_field->system = kstrdup(system, GFP_KERNEL);
2210 if (!ref_field->system)
2211 return -ENOMEM;
2212 }
2213
2214 if (event_name) {
2215 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2216 if (!ref_field->event_name) {
2217 err = -ENOMEM;
2218 goto free;
2219 }
2220 }
2221
7e8b88a3
TZ		 2222 if (var_field->var.name) {
2223 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2224 if (!ref_field->name) {
2225 err = -ENOMEM;
2226 goto free;
2227 }
2228 } else if (var_field->name) {
2229 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2230 if (!ref_field->name) {
2231 err = -ENOMEM;
2232 goto free;
2233 }
067fe038
TZ		 2234 }
2235
2236 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2237 if (!ref_field->type) {
2238 err = -ENOMEM;
2239 goto free;
2240 }
2241 out:
2242 return err;
2243 free:
2244 kfree(ref_field->system);
2245 kfree(ref_field->event_name);
2246 kfree(ref_field->name);
2247
2248 goto out;
2249}
2250
2251static struct hist_field *create_var_ref(struct hist_field *var_field,
2252 char *system, char *event_name)
2253{
2254 unsigned long flags = HIST_FIELD_FL_VAR_REF;
2255 struct hist_field *ref_field;
2256
2257 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2258 if (ref_field) {
2259 if (init_var_ref(ref_field, var_field, system, event_name)) {
2260 destroy_hist_field(ref_field, 0);
2261 return NULL;
2262 }
2263 }
2264
2265 return ref_field;
2266}
2267
2268static bool is_var_ref(char *var_name)
2269{
2270 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2271 return false;
2272
2273 return true;
2274}
2275
2276static char *field_name_from_var(struct hist_trigger_data *hist_data,
2277 char *var_name)
2278{
2279 char *name, *field;
2280 unsigned int i;
2281
2282 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2283 name = hist_data->attrs->var_defs.name[i];
2284
2285 if (strcmp(var_name, name) == 0) {
2286 field = hist_data->attrs->var_defs.expr[i];
2287 if (contains_operator(field) || is_var_ref(field))
2288 continue;
2289 return field;
2290 }
2291 }
2292
2293 return NULL;
2294}
2295
2296static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2297 char *system, char *event_name,
2298 char *var_name)
2299{
2300 struct trace_event_call *call;
2301
2302 if (system && event_name) {
2303 call = hist_data->event_file->event_call;
2304
2305 if (strcmp(system, call->class->system) != 0)
2306 return NULL;
2307
2308 if (strcmp(event_name, trace_event_name(call)) != 0)
2309 return NULL;
2310 }
2311
2312 if (!!system != !!event_name)
2313 return NULL;
2314
2315 if (!is_var_ref(var_name))
2316 return NULL;
2317
2318 var_name++;
2319
2320 return field_name_from_var(hist_data, var_name);
2321}
2322
2323static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2324 char *system, char *event_name,
2325 char *var_name)
2326{
2327 struct hist_field *var_field = NULL, *ref_field = NULL;
2328
2329 if (!is_var_ref(var_name))
2330 return NULL;
2331
2332 var_name++;
2333
2334 var_field = find_event_var(hist_data, system, event_name, var_name);
2335 if (var_field)
2336 ref_field = create_var_ref(var_field, system, event_name);
2337
2338 return ref_field;
2339}
2340
100719dc
TZ		 2341static struct ftrace_event_field *
2342parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2343 char *field_str, unsigned long *flags)
2344{
2345 struct ftrace_event_field *field = NULL;
2346 char *field_name, *modifier, *str;
2347
2348 modifier = str = kstrdup(field_str, GFP_KERNEL);
2349 if (!modifier)
2350 return ERR_PTR(-ENOMEM);
2351
2352 field_name = strsep(&modifier, ".");
2353 if (modifier) {
2354 if (strcmp(modifier, "hex") == 0)
2355 *flags |= HIST_FIELD_FL_HEX;
2356 else if (strcmp(modifier, "sym") == 0)
2357 *flags |= HIST_FIELD_FL_SYM;
2358 else if (strcmp(modifier, "sym-offset") == 0)
2359 *flags |= HIST_FIELD_FL_SYM_OFFSET;
2360 else if ((strcmp(modifier, "execname") == 0) &&
2361 (strcmp(field_name, "common_pid") == 0))
2362 *flags |= HIST_FIELD_FL_EXECNAME;
2363 else if (strcmp(modifier, "syscall") == 0)
2364 *flags |= HIST_FIELD_FL_SYSCALL;
2365 else if (strcmp(modifier, "log2") == 0)
2366 *flags |= HIST_FIELD_FL_LOG2;
2367 else if (strcmp(modifier, "usecs") == 0)
2368 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2369 else {
2370 field = ERR_PTR(-EINVAL);
2371 goto out;
2372 }
2373 }
2374
2375 if (strcmp(field_name, "common_timestamp") == 0) {
2376 *flags |= HIST_FIELD_FL_TIMESTAMP;
2377 hist_data->enable_timestamps = true;
2378 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2379 hist_data->attrs->ts_in_usecs = true;
8b7622bf
TZ		 2380 } else if (strcmp(field_name, "cpu") == 0)
2381 *flags |= HIST_FIELD_FL_CPU;
2382 else {
100719dc
TZ		 2383 field = trace_find_event_field(file->event_call, field_name);
2384 if (!field || !field->size) {
2385 field = ERR_PTR(-EINVAL);
2386 goto out;
2387 }
2388 }
2389 out:
2390 kfree(str);
2391
2392 return field;
2393}
2394
7e8b88a3
TZ		 2395static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2396 struct hist_field *var_ref,
2397 char *var_name)
2398{
2399 struct hist_field *alias = NULL;
2400 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2401
2402 alias = create_hist_field(hist_data, NULL, flags, var_name);
2403 if (!alias)
2404 return NULL;
2405
2406 alias->fn = var_ref->fn;
2407 alias->operands[0] = var_ref;
2408
2409 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2410 destroy_hist_field(alias, 0);
2411 return NULL;
2412 }
2413
2414 return alias;
2415}
2416
100719dc
TZ		 2417static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2418 struct trace_event_file *file, char *str,
2419 unsigned long *flags, char *var_name)
2420{
067fe038 2421 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
100719dc
TZ		 2422 struct ftrace_event_field *field = NULL;
2423 struct hist_field *hist_field = NULL;
2424 int ret = 0;
2425
067fe038
TZ		 2426 s = strchr(str, '.');
2427 if (s) {
2428 s = strchr(++s, '.');
2429 if (s) {
2430 ref_system = strsep(&str, ".");
2431 if (!str) {
2432 ret = -EINVAL;
2433 goto out;
2434 }
2435 ref_event = strsep(&str, ".");
2436 if (!str) {
2437 ret = -EINVAL;
2438 goto out;
2439 }
2440 ref_var = str;
2441 }
2442 }
2443
2444 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2445 if (!s) {
2446 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2447 if (hist_field) {
2448 hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2449 hist_field->var_ref_idx = hist_data->n_var_refs++;
7e8b88a3
TZ		 2450 if (var_name) {
2451 hist_field = create_alias(hist_data, hist_field, var_name);
2452 if (!hist_field) {
2453 ret = -ENOMEM;
2454 goto out;
2455 }
2456 }
067fe038
TZ		 2457 return hist_field;
2458 }
2459 } else
2460 str = s;
2461
100719dc
TZ		 2462 field = parse_field(hist_data, file, str, flags);
2463 if (IS_ERR(field)) {
2464 ret = PTR_ERR(field);
2465 goto out;
2466 }
2467
2468 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2469 if (!hist_field) {
2470 ret = -ENOMEM;
2471 goto out;
2472 }
2473
2474 return hist_field;
2475 out:
2476 return ERR_PTR(ret);
2477}
2478
2479static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2480 struct trace_event_file *file,
2481 char *str, unsigned long flags,
2482 char *var_name, unsigned int level);
2483
2484static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2485 struct trace_event_file *file,
2486 char *str, unsigned long flags,
2487 char *var_name, unsigned int level)
2488{
2489 struct hist_field *operand1, *expr = NULL;
2490 unsigned long operand_flags;
2491 int ret = 0;
2492 char *s;
2493
2494 /* we support only -(xxx) i.e. explicit parens required */
2495
2496 if (level > 3) {
2497 ret = -EINVAL;
2498 goto free;
2499 }
2500
2501 str++; /* skip leading '-' */
2502
2503 s = strchr(str, '(');
2504 if (s)
2505 str++;
2506 else {
2507 ret = -EINVAL;
2508 goto free;
2509 }
2510
2511 s = strrchr(str, ')');
2512 if (s)
2513 *s = '\0';
2514 else {
2515 ret = -EINVAL; /* no closing ')' */
2516 goto free;
2517 }
2518
2519 flags |= HIST_FIELD_FL_EXPR;
2520 expr = create_hist_field(hist_data, NULL, flags, var_name);
2521 if (!expr) {
2522 ret = -ENOMEM;
2523 goto free;
2524 }
2525
2526 operand_flags = 0;
2527 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2528 if (IS_ERR(operand1)) {
2529 ret = PTR_ERR(operand1);
2530 goto free;
2531 }
2532
2533 expr->flags |= operand1->flags &
2534 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2535 expr->fn = hist_field_unary_minus;
2536 expr->operands[0] = operand1;
2537 expr->operator = FIELD_OP_UNARY_MINUS;
2538 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2539 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2540 if (!expr->type) {
2541 ret = -ENOMEM;
2542 goto free;
2543 }
100719dc
TZ		 2544
2545 return expr;
2546 free:
2547 destroy_hist_field(expr, 0);
2548 return ERR_PTR(ret);
2549}
2550
2551static int check_expr_operands(struct hist_field *operand1,
2552 struct hist_field *operand2)
2553{
2554 unsigned long operand1_flags = operand1->flags;
2555 unsigned long operand2_flags = operand2->flags;
2556
7e8b88a3
TZ		 2557 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2558 (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2559 struct hist_field *var;
2560
2561 var = find_var_field(operand1->var.hist_data, operand1->name);
2562 if (!var)
2563 return -EINVAL;
2564 operand1_flags = var->flags;
2565 }
2566
2567 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2568 (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2569 struct hist_field *var;
2570
2571 var = find_var_field(operand2->var.hist_data, operand2->name);
2572 if (!var)
2573 return -EINVAL;
2574 operand2_flags = var->flags;
2575 }
2576
100719dc
TZ		 2577 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2578 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
2579 return -EINVAL;
2580
2581 return 0;
2582}
2583
2584static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2585 struct trace_event_file *file,
2586 char *str, unsigned long flags,
2587 char *var_name, unsigned int level)
2588{
2589 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2590 unsigned long operand_flags;
2591 int field_op, ret = -EINVAL;
2592 char *sep, *operand1_str;
2593
2594 if (level > 3)
2595 return ERR_PTR(-EINVAL);
2596
2597 field_op = contains_operator(str);
2598
2599 if (field_op == FIELD_OP_NONE)
2600 return parse_atom(hist_data, file, str, &flags, var_name);
2601
2602 if (field_op == FIELD_OP_UNARY_MINUS)
2603 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2604
2605 switch (field_op) {
2606 case FIELD_OP_MINUS:
2607 sep = "-";
2608 break;
2609 case FIELD_OP_PLUS:
2610 sep = "+";
2611 break;
2612 default:
2613 goto free;
2614 }
2615
2616 operand1_str = strsep(&str, sep);
2617 if (!operand1_str || !str)
2618 goto free;
2619
2620 operand_flags = 0;
2621 operand1 = parse_atom(hist_data, file, operand1_str,
2622 &operand_flags, NULL);
2623 if (IS_ERR(operand1)) {
2624 ret = PTR_ERR(operand1);
2625 operand1 = NULL;
2626 goto free;
2627 }
2628
2629 /* rest of string could be another expression e.g. b+c in a+b+c */
2630 operand_flags = 0;
2631 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2632 if (IS_ERR(operand2)) {
2633 ret = PTR_ERR(operand2);
2634 operand2 = NULL;
2635 goto free;
2636 }
2637
2638 ret = check_expr_operands(operand1, operand2);
2639 if (ret)
2640 goto free;
2641
2642 flags |= HIST_FIELD_FL_EXPR;
2643
2644 flags |= operand1->flags &
2645 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2646
2647 expr = create_hist_field(hist_data, NULL, flags, var_name);
2648 if (!expr) {
2649 ret = -ENOMEM;
2650 goto free;
2651 }
2652
067fe038
TZ		 2653 operand1->read_once = true;
2654 operand2->read_once = true;
2655
100719dc
TZ		 2656 expr->operands[0] = operand1;
2657 expr->operands[1] = operand2;
2658 expr->operator = field_op;
2659 expr->name = expr_str(expr, 0);
19a9facd
TZ		 2660 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2661 if (!expr->type) {
2662 ret = -ENOMEM;
2663 goto free;
2664 }
100719dc
TZ		 2665
2666 switch (field_op) {
2667 case FIELD_OP_MINUS:
2668 expr->fn = hist_field_minus;
2669 break;
2670 case FIELD_OP_PLUS:
2671 expr->fn = hist_field_plus;
2672 break;
2673 default:
2674 goto free;
2675 }
2676
2677 return expr;
2678 free:
2679 destroy_hist_field(operand1, 0);
2680 destroy_hist_field(operand2, 0);
2681 destroy_hist_field(expr, 0);
2682
2683 return ERR_PTR(ret);
2684}
2685
02205a67
TZ		 2686static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2687 struct trace_event_file *file)
2688{
2689 struct event_trigger_data *test;
2690
2691 list_for_each_entry_rcu(test, &file->triggers, list) {
2692 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2693 if (test->private_data == hist_data)
2694 return test->filter_str;
2695 }
2696 }
2697
2698 return NULL;
2699}
2700
2701static struct event_command trigger_hist_cmd;
2702static int event_hist_trigger_func(struct event_command *cmd_ops,
2703 struct trace_event_file *file,
2704 char *glob, char *cmd, char *param);
2705
2706static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2707 struct hist_trigger_data *hist_data,
2708 unsigned int n_keys)
2709{
2710 struct hist_field *target_hist_field, *hist_field;
2711 unsigned int n, i, j;
2712
2713 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2714 return false;
2715
2716 i = hist_data->n_vals;
2717 j = target_hist_data->n_vals;
2718
2719 for (n = 0; n < n_keys; n++) {
2720 hist_field = hist_data->fields[i + n];
2721 target_hist_field = target_hist_data->fields[j + n];
2722
2723 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2724 return false;
2725 if (hist_field->size != target_hist_field->size)
2726 return false;
2727 if (hist_field->is_signed != target_hist_field->is_signed)
2728 return false;
2729 }
2730
2731 return true;
2732}
2733
2734static struct hist_trigger_data *
2735find_compatible_hist(struct hist_trigger_data *target_hist_data,
2736 struct trace_event_file *file)
2737{
2738 struct hist_trigger_data *hist_data;
2739 struct event_trigger_data *test;
2740 unsigned int n_keys;
2741
2742 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2743
2744 list_for_each_entry_rcu(test, &file->triggers, list) {
2745 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2746 hist_data = test->private_data;
2747
2748 if (compatible_keys(target_hist_data, hist_data, n_keys))
2749 return hist_data;
2750 }
2751 }
2752
2753 return NULL;
2754}
2755
2756static struct trace_event_file *event_file(struct trace_array *tr,
2757 char *system, char *event_name)
2758{
2759 struct trace_event_file *file;
2760
2761 file = find_event_file(tr, system, event_name);
2762 if (!file)
2763 return ERR_PTR(-EINVAL);
2764
2765 return file;
2766}
2767
2768static struct hist_field *
2769find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2770 char *system, char *event_name, char *field_name)
2771{
2772 struct hist_field *event_var;
2773 char *synthetic_name;
2774
2775 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2776 if (!synthetic_name)
2777 return ERR_PTR(-ENOMEM);
2778
2779 strcpy(synthetic_name, "synthetic_");
2780 strcat(synthetic_name, field_name);
2781
2782 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2783
2784 kfree(synthetic_name);
2785
2786 return event_var;
2787}
2788
2789/**
2790 * create_field_var_hist - Automatically create a histogram and var for a field
2791 * @target_hist_data: The target hist trigger
2792 * @subsys_name: Optional subsystem name
2793 * @event_name: Optional event name
2794 * @field_name: The name of the field (and the resulting variable)
2795 *
2796 * Hist trigger actions fetch data from variables, not directly from
2797 * events. However, for convenience, users are allowed to directly
2798 * specify an event field in an action, which will be automatically
2799 * converted into a variable on their behalf.
2800
2801 * If a user specifies a field on an event that isn't the event the
2802 * histogram currently being defined (the target event histogram), the
2803 * only way that can be accomplished is if a new hist trigger is
2804 * created and the field variable defined on that.
2805 *
2806 * This function creates a new histogram compatible with the target
2807 * event (meaning a histogram with the same key as the target
2808 * histogram), and creates a variable for the specified field, but
2809 * with 'synthetic_' prepended to the variable name in order to avoid
2810 * collision with normal field variables.
2811 *
2812 * Return: The variable created for the field.
2813 */
c282a386 2814static struct hist_field *
02205a67
TZ		 2815create_field_var_hist(struct hist_trigger_data *target_hist_data,
2816 char *subsys_name, char *event_name, char *field_name)
2817{
2818 struct trace_array *tr = target_hist_data->event_file->tr;
2819 struct hist_field *event_var = ERR_PTR(-EINVAL);
2820 struct hist_trigger_data *hist_data;
2821 unsigned int i, n, first = true;
2822 struct field_var_hist *var_hist;
2823 struct trace_event_file *file;
2824 struct hist_field *key_field;
2825 char *saved_filter;
2826 char *cmd;
2827 int ret;
2828
2829 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX)
2830 return ERR_PTR(-EINVAL);
2831
2832 file = event_file(tr, subsys_name, event_name);
2833
2834 if (IS_ERR(file)) {
2835 ret = PTR_ERR(file);
2836 return ERR_PTR(ret);
2837 }
2838
2839 /*
2840 * Look for a histogram compatible with target. We'll use the
2841 * found histogram specification to create a new matching
2842 * histogram with our variable on it. target_hist_data is not
2843 * yet a registered histogram so we can't use that.
2844 */
2845 hist_data = find_compatible_hist(target_hist_data, file);
2846 if (!hist_data)
2847 return ERR_PTR(-EINVAL);
2848
2849 /* See if a synthetic field variable has already been created */
2850 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2851 event_name, field_name);
2852 if (!IS_ERR_OR_NULL(event_var))
2853 return event_var;
2854
2855 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2856 if (!var_hist)
2857 return ERR_PTR(-ENOMEM);
2858
2859 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2860 if (!cmd) {
2861 kfree(var_hist);
2862 return ERR_PTR(-ENOMEM);
2863 }
2864
2865 /* Use the same keys as the compatible histogram */
2866 strcat(cmd, "keys=");
2867
2868 for_each_hist_key_field(i, hist_data) {
2869 key_field = hist_data->fields[i];
2870 if (!first)
2871 strcat(cmd, ",");
2872 strcat(cmd, key_field->field->name);
2873 first = false;
2874 }
2875
2876 /* Create the synthetic field variable specification */
2877 strcat(cmd, ":synthetic_");
2878 strcat(cmd, field_name);
2879 strcat(cmd, "=");
2880 strcat(cmd, field_name);
2881
2882 /* Use the same filter as the compatible histogram */
2883 saved_filter = find_trigger_filter(hist_data, file);
2884 if (saved_filter) {
2885 strcat(cmd, " if ");
2886 strcat(cmd, saved_filter);
2887 }
2888
2889 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2890 if (!var_hist->cmd) {
2891 kfree(cmd);
2892 kfree(var_hist);
2893 return ERR_PTR(-ENOMEM);
2894 }
2895
2896 /* Save the compatible histogram information */
2897 var_hist->hist_data = hist_data;
2898
2899 /* Create the new histogram with our variable */
2900 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2901 "", "hist", cmd);
2902 if (ret) {
2903 kfree(cmd);
2904 kfree(var_hist->cmd);
2905 kfree(var_hist);
2906 return ERR_PTR(ret);
2907 }
2908
2909 kfree(cmd);
2910
2911 /* If we can't find the variable, something went wrong */
2912 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2913 event_name, field_name);
2914 if (IS_ERR_OR_NULL(event_var)) {
2915 kfree(var_hist->cmd);
2916 kfree(var_hist);
2917 return ERR_PTR(-EINVAL);
2918 }
2919
2920 n = target_hist_data->n_field_var_hists;
2921 target_hist_data->field_var_hists[n] = var_hist;
2922 target_hist_data->n_field_var_hists++;
2923
2924 return event_var;
2925}
2926
c282a386 2927static struct hist_field *
02205a67
TZ		 2928find_target_event_var(struct hist_trigger_data *hist_data,
2929 char *subsys_name, char *event_name, char *var_name)
2930{
2931 struct trace_event_file *file = hist_data->event_file;
2932 struct hist_field *hist_field = NULL;
2933
2934 if (subsys_name) {
2935 struct trace_event_call *call;
2936
2937 if (!event_name)
2938 return NULL;
2939
2940 call = file->event_call;
2941
2942 if (strcmp(subsys_name, call->class->system) != 0)
2943 return NULL;
2944
2945 if (strcmp(event_name, trace_event_name(call)) != 0)
2946 return NULL;
2947 }
2948
2949 hist_field = find_var_field(hist_data, var_name);
2950
2951 return hist_field;
2952}
2953
2954static inline void __update_field_vars(struct tracing_map_elt *elt,
2955 struct ring_buffer_event *rbe,
2956 void *rec,
2957 struct field_var **field_vars,
2958 unsigned int n_field_vars,
2959 unsigned int field_var_str_start)
2960{
2961 struct hist_elt_data *elt_data = elt->private_data;
2962 unsigned int i, j, var_idx;
2963 u64 var_val;
2964
2965 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2966 struct field_var *field_var = field_vars[i];
2967 struct hist_field *var = field_var->var;
2968 struct hist_field *val = field_var->val;
2969
2970 var_val = val->fn(val, elt, rbe, rec);
2971 var_idx = var->var.idx;
2972
2973 if (val->flags & HIST_FIELD_FL_STRING) {
2974 char *str = elt_data->field_var_str[j++];
2975 char *val_str = (char *)(uintptr_t)var_val;
2976
2977 strncpy(str, val_str, STR_VAR_LEN_MAX);
2978 var_val = (u64)(uintptr_t)str;
2979 }
2980 tracing_map_set_var(elt, var_idx, var_val);
2981 }
2982}
2983
2984static void update_field_vars(struct hist_trigger_data *hist_data,
2985 struct tracing_map_elt *elt,
2986 struct ring_buffer_event *rbe,
2987 void *rec)
2988{
2989 __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2990 hist_data->n_field_vars, 0);
2991}
2992
50450603
TZ		 2993static void update_max_vars(struct hist_trigger_data *hist_data,
2994 struct tracing_map_elt *elt,
2995 struct ring_buffer_event *rbe,
2996 void *rec)
2997{
2998 __update_field_vars(elt, rbe, rec, hist_data->max_vars,
2999 hist_data->n_max_vars, hist_data->n_field_var_str);
3000}
3001
02205a67
TZ		 3002static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3003 struct trace_event_file *file,
3004 char *name, int size, const char *type)
3005{
3006 struct hist_field *var;
3007 int idx;
3008
3009 if (find_var(hist_data, file, name) && !hist_data->remove) {
3010 var = ERR_PTR(-EINVAL);
3011 goto out;
3012 }
3013
3014 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3015 if (!var) {
3016 var = ERR_PTR(-ENOMEM);
3017 goto out;
3018 }
3019
3020 idx = tracing_map_add_var(hist_data->map);
3021 if (idx < 0) {
3022 kfree(var);
3023 var = ERR_PTR(-EINVAL);
3024 goto out;
3025 }
3026
3027 var->flags = HIST_FIELD_FL_VAR;
3028 var->var.idx = idx;
3029 var->var.hist_data = var->hist_data = hist_data;
3030 var->size = size;
3031 var->var.name = kstrdup(name, GFP_KERNEL);
3032 var->type = kstrdup(type, GFP_KERNEL);
3033 if (!var->var.name || !var->type) {
3034 kfree(var->var.name);
3035 kfree(var->type);
3036 kfree(var);
3037 var = ERR_PTR(-ENOMEM);
3038 }
3039 out:
3040 return var;
3041}
3042
3043static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3044 struct trace_event_file *file,
3045 char *field_name)
3046{
3047 struct hist_field *val = NULL, *var = NULL;
3048 unsigned long flags = HIST_FIELD_FL_VAR;
3049 struct field_var *field_var;
3050 int ret = 0;
3051
3052 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3053 ret = -EINVAL;
3054 goto err;
3055 }
3056
3057 val = parse_atom(hist_data, file, field_name, &flags, NULL);
3058 if (IS_ERR(val)) {
3059 ret = PTR_ERR(val);
3060 goto err;
3061 }
3062
3063 var = create_var(hist_data, file, field_name, val->size, val->type);
3064 if (IS_ERR(var)) {
3065 kfree(val);
3066 ret = PTR_ERR(var);
3067 goto err;
3068 }
3069
3070 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3071 if (!field_var) {
3072 kfree(val);
3073 kfree(var);
3074 ret = -ENOMEM;
3075 goto err;
3076 }
3077
3078 field_var->var = var;
3079 field_var->val = val;
3080 out:
3081 return field_var;
3082 err:
3083 field_var = ERR_PTR(ret);
3084 goto out;
3085}
3086
3087/**
3088 * create_target_field_var - Automatically create a variable for a field
3089 * @target_hist_data: The target hist trigger
3090 * @subsys_name: Optional subsystem name
3091 * @event_name: Optional event name
3092 * @var_name: The name of the field (and the resulting variable)
3093 *
3094 * Hist trigger actions fetch data from variables, not directly from
3095 * events. However, for convenience, users are allowed to directly
3096 * specify an event field in an action, which will be automatically
3097 * converted into a variable on their behalf.
3098
3099 * This function creates a field variable with the name var_name on
3100 * the hist trigger currently being defined on the target event. If
3101 * subsys_name and event_name are specified, this function simply
3102 * verifies that they do in fact match the target event subsystem and
3103 * event name.
3104 *
3105 * Return: The variable created for the field.
3106 */
c282a386 3107static struct field_var *
02205a67
TZ		 3108create_target_field_var(struct hist_trigger_data *target_hist_data,
3109 char *subsys_name, char *event_name, char *var_name)
3110{
3111 struct trace_event_file *file = target_hist_data->event_file;
3112
3113 if (subsys_name) {
3114 struct trace_event_call *call;
3115
3116 if (!event_name)
3117 return NULL;
3118
3119 call = file->event_call;
3120
3121 if (strcmp(subsys_name, call->class->system) != 0)
3122 return NULL;
3123
3124 if (strcmp(event_name, trace_event_name(call)) != 0)
3125 return NULL;
3126 }
3127
3128 return create_field_var(target_hist_data, file, var_name);
3129}
3130
50450603
TZ		 3131static void onmax_print(struct seq_file *m,
3132 struct hist_trigger_data *hist_data,
3133 struct tracing_map_elt *elt,
3134 struct action_data *data)
3135{
3136 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3137
3138 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3139
3140 for (i = 0; i < hist_data->n_max_vars; i++) {
3141 struct hist_field *save_val = hist_data->max_vars[i]->val;
3142 struct hist_field *save_var = hist_data->max_vars[i]->var;
3143 u64 val;
3144
3145 save_var_idx = save_var->var.idx;
3146
3147 val = tracing_map_read_var(elt, save_var_idx);
3148
3149 if (save_val->flags & HIST_FIELD_FL_STRING) {
3150 seq_printf(m, " %s: %-32s", save_var->var.name,
3151 (char *)(uintptr_t)(val));
3152 } else
3153 seq_printf(m, " %s: %10llu", save_var->var.name, val);
3154 }
3155}
3156
3157static void onmax_save(struct hist_trigger_data *hist_data,
3158 struct tracing_map_elt *elt, void *rec,
3159 struct ring_buffer_event *rbe,
3160 struct action_data *data, u64 *var_ref_vals)
3161{
3162 unsigned int max_idx = data->onmax.max_var->var.idx;
3163 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3164
3165 u64 var_val, max_val;
3166
3167 var_val = var_ref_vals[max_var_ref_idx];
3168 max_val = tracing_map_read_var(elt, max_idx);
3169
3170 if (var_val <= max_val)
3171 return;
3172
3173 tracing_map_set_var(elt, max_idx, var_val);
3174
3175 update_max_vars(hist_data, elt, rbe, rec);
3176}
3177
3178static void onmax_destroy(struct action_data *data)
3179{
3180 unsigned int i;
3181
3182 destroy_hist_field(data->onmax.max_var, 0);
3183 destroy_hist_field(data->onmax.var, 0);
3184
3185 kfree(data->onmax.var_str);
3186 kfree(data->onmax.fn_name);
3187
3188 for (i = 0; i < data->n_params; i++)
3189 kfree(data->params[i]);
3190
3191 kfree(data);
3192}
3193
3194static int onmax_create(struct hist_trigger_data *hist_data,
3195 struct action_data *data)
3196{
3197 struct trace_event_file *file = hist_data->event_file;
3198 struct hist_field *var_field, *ref_field, *max_var;
3199 unsigned int var_ref_idx = hist_data->n_var_refs;
3200 struct field_var *field_var;
3201 char *onmax_var_str, *param;
3202 unsigned long flags;
3203 unsigned int i;
3204 int ret = 0;
3205
3206 onmax_var_str = data->onmax.var_str;
3207 if (onmax_var_str[0] != '$')
3208 return -EINVAL;
3209 onmax_var_str++;
3210
3211 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3212 if (!var_field)
3213 return -EINVAL;
3214
3215 flags = HIST_FIELD_FL_VAR_REF;
3216 ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3217 if (!ref_field)
3218 return -ENOMEM;
3219
3220 if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3221 destroy_hist_field(ref_field, 0);
3222 ret = -ENOMEM;
3223 goto out;
3224 }
3225 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3226 ref_field->var_ref_idx = hist_data->n_var_refs++;
3227 data->onmax.var = ref_field;
3228
3229 data->fn = onmax_save;
3230 data->onmax.max_var_ref_idx = var_ref_idx;
3231 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3232 if (IS_ERR(max_var)) {
3233 ret = PTR_ERR(max_var);
3234 goto out;
3235 }
3236 data->onmax.max_var = max_var;
3237
3238 for (i = 0; i < data->n_params; i++) {
3239 param = kstrdup(data->params[i], GFP_KERNEL);
3240 if (!param) {
3241 ret = -ENOMEM;
3242 goto out;
3243 }
3244
3245 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3246 if (IS_ERR(field_var)) {
3247 ret = PTR_ERR(field_var);
3248 kfree(param);
3249 goto out;
3250 }
3251
3252 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3253 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3254 hist_data->n_max_var_str++;
3255
3256 kfree(param);
3257 }
3258 out:
3259 return ret;
3260}
3261
3262static int parse_action_params(char *params, struct action_data *data)
3263{
3264 char *param, *saved_param;
3265 int ret = 0;
3266
3267 while (params) {
3268 if (data->n_params >= SYNTH_FIELDS_MAX)
3269 goto out;
3270
3271 param = strsep(&params, ",");
3272 if (!param) {
3273 ret = -EINVAL;
3274 goto out;
3275 }
3276
3277 param = strstrip(param);
3278 if (strlen(param) < 2) {
3279 ret = -EINVAL;
3280 goto out;
3281 }
3282
3283 saved_param = kstrdup(param, GFP_KERNEL);
3284 if (!saved_param) {
3285 ret = -ENOMEM;
3286 goto out;
3287 }
3288
3289 data->params[data->n_params++] = saved_param;
3290 }
3291 out:
3292 return ret;
3293}
3294
3295static struct action_data *onmax_parse(char *str)
3296{
3297 char *onmax_fn_name, *onmax_var_str;
3298 struct action_data *data;
3299 int ret = -EINVAL;
3300
3301 data = kzalloc(sizeof(*data), GFP_KERNEL);
3302 if (!data)
3303 return ERR_PTR(-ENOMEM);
3304
3305 onmax_var_str = strsep(&str, ")");
3306 if (!onmax_var_str || !str) {
3307 ret = -EINVAL;
3308 goto free;
3309 }
3310
3311 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3312 if (!data->onmax.var_str) {
3313 ret = -ENOMEM;
3314 goto free;
3315 }
3316
3317 strsep(&str, ".");
3318 if (!str)
3319 goto free;
3320
3321 onmax_fn_name = strsep(&str, "(");
3322 if (!onmax_fn_name || !str)
3323 goto free;
3324
3325 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3326 char *params = strsep(&str, ")");
3327
3328 if (!params) {
3329 ret = -EINVAL;
3330 goto free;
3331 }
3332
3333 ret = parse_action_params(params, data);
3334 if (ret)
3335 goto free;
3336 } else
3337 goto free;
3338
3339 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3340 if (!data->onmax.fn_name) {
3341 ret = -ENOMEM;
3342 goto free;
3343 }
3344 out:
3345 return data;
3346 free:
3347 onmax_destroy(data);
3348 data = ERR_PTR(ret);
3349 goto out;
3350}
3351
c282a386
TZ		 3352static void onmatch_destroy(struct action_data *data)
3353{
3354 unsigned int i;
3355
3356 mutex_lock(&synth_event_mutex);
3357
3358 kfree(data->onmatch.match_event);
3359 kfree(data->onmatch.match_event_system);
3360 kfree(data->onmatch.synth_event_name);
3361
3362 for (i = 0; i < data->n_params; i++)
3363 kfree(data->params[i]);
3364
3365 if (data->onmatch.synth_event)
3366 data->onmatch.synth_event->ref--;
3367
3368 kfree(data);
3369
3370 mutex_unlock(&synth_event_mutex);
3371}
3372
02205a67
TZ		 3373static void destroy_field_var(struct field_var *field_var)
3374{
3375 if (!field_var)
3376 return;
3377
3378 destroy_hist_field(field_var->var, 0);
3379 destroy_hist_field(field_var->val, 0);
3380
3381 kfree(field_var);
3382}
3383
3384static void destroy_field_vars(struct hist_trigger_data *hist_data)
3385{
3386 unsigned int i;
3387
3388 for (i = 0; i < hist_data->n_field_vars; i++)
3389 destroy_field_var(hist_data->field_vars[i]);
3390}
3391
c282a386
TZ		 3392static void save_field_var(struct hist_trigger_data *hist_data,
3393 struct field_var *field_var)
02205a67
TZ		 3394{
3395 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3396
3397 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3398 hist_data->n_field_var_str++;
3399}
3400
c282a386
TZ		 3401
3402static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3403{
3404 unsigned int i;
3405
3406 for (i = 0; i < hist_data->n_synth_var_refs; i++)
3407 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3408}
3409
3410static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3411 struct hist_field *var_ref)
3412{
3413 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3414
3415 hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3416 var_ref->var_ref_idx = hist_data->n_var_refs++;
3417}
3418
3419static int check_synth_field(struct synth_event *event,
3420 struct hist_field *hist_field,
3421 unsigned int field_pos)
3422{
3423 struct synth_field *field;
3424
3425 if (field_pos >= event->n_fields)
3426 return -EINVAL;
3427
3428 field = event->fields[field_pos];
3429
3430 if (strcmp(field->type, hist_field->type) != 0)
3431 return -EINVAL;
3432
3433 return 0;
3434}
3435
c282a386
TZ		 3436static struct hist_field *
3437onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3438 char *system, char *event, char *var)
3439{
3440 struct hist_field *hist_field;
3441
3442 var++; /* skip '$' */
3443
3444 hist_field = find_target_event_var(hist_data, system, event, var);
3445 if (!hist_field) {
3446 if (!system) {
3447 system = data->onmatch.match_event_system;
3448 event = data->onmatch.match_event;
3449 }
3450
3451 hist_field = find_event_var(hist_data, system, event, var);
3452 }
3453
3454 return hist_field;
3455}
3456
3457static struct hist_field *
3458onmatch_create_field_var(struct hist_trigger_data *hist_data,
3459 struct action_data *data, char *system,
3460 char *event, char *var)
3461{
3462 struct hist_field *hist_field = NULL;
3463 struct field_var *field_var;
3464
3465 /*
3466 * First try to create a field var on the target event (the
3467 * currently being defined). This will create a variable for
3468 * unqualified fields on the target event, or if qualified,
3469 * target fields that have qualified names matching the target.
3470 */
3471 field_var = create_target_field_var(hist_data, system, event, var);
3472
3473 if (field_var && !IS_ERR(field_var)) {
3474 save_field_var(hist_data, field_var);
3475 hist_field = field_var->var;
3476 } else {
3477 field_var = NULL;
3478 /*
3479 * If no explicit system.event is specfied, default to
3480 * looking for fields on the onmatch(system.event.xxx)
3481 * event.
3482 */
3483 if (!system) {
3484 system = data->onmatch.match_event_system;
3485 event = data->onmatch.match_event;
3486 }
3487
3488 /*
3489 * At this point, we're looking at a field on another
3490 * event. Because we can't modify a hist trigger on
3491 * another event to add a variable for a field, we need
3492 * to create a new trigger on that event and create the
3493 * variable at the same time.
3494 */
3495 hist_field = create_field_var_hist(hist_data, system, event, var);
3496 if (IS_ERR(hist_field))
3497 goto free;
3498 }
3499 out:
3500 return hist_field;
3501 free:
3502 destroy_field_var(field_var);
3503 hist_field = NULL;
3504 goto out;
3505}
3506
3507static int onmatch_create(struct hist_trigger_data *hist_data,
3508 struct trace_event_file *file,
3509 struct action_data *data)
3510{
3511 char *event_name, *param, *system = NULL;
3512 struct hist_field *hist_field, *var_ref;
3513 unsigned int i, var_ref_idx;
3514 unsigned int field_pos = 0;
3515 struct synth_event *event;
3516 int ret = 0;
3517
3518 mutex_lock(&synth_event_mutex);
3519 event = find_synth_event(data->onmatch.synth_event_name);
3520 if (!event) {
3521 mutex_unlock(&synth_event_mutex);
3522 return -EINVAL;
3523 }
3524 event->ref++;
3525 mutex_unlock(&synth_event_mutex);
3526
3527 var_ref_idx = hist_data->n_var_refs;
3528
3529 for (i = 0; i < data->n_params; i++) {
3530 char *p;
3531
3532 p = param = kstrdup(data->params[i], GFP_KERNEL);
3533 if (!param) {
3534 ret = -ENOMEM;
3535 goto err;
3536 }
3537
3538 system = strsep(&param, ".");
3539 if (!param) {
3540 param = (char *)system;
3541 system = event_name = NULL;
3542 } else {
3543 event_name = strsep(&param, ".");
3544 if (!param) {
3545 kfree(p);
3546 ret = -EINVAL;
3547 goto err;
3548 }
3549 }
3550
3551 if (param[0] == '$')
3552 hist_field = onmatch_find_var(hist_data, data, system,
3553 event_name, param);
3554 else
3555 hist_field = onmatch_create_field_var(hist_data, data,
3556 system,
3557 event_name,
3558 param);
3559
3560 if (!hist_field) {
3561 kfree(p);
3562 ret = -EINVAL;
3563 goto err;
3564 }
3565
3566 if (check_synth_field(event, hist_field, field_pos) == 0) {
3567 var_ref = create_var_ref(hist_field, system, event_name);
3568 if (!var_ref) {
3569 kfree(p);
3570 ret = -ENOMEM;
3571 goto err;
3572 }
3573
3574 save_synth_var_ref(hist_data, var_ref);
3575 field_pos++;
3576 kfree(p);
3577 continue;
3578 }
3579
3580 kfree(p);
3581 ret = -EINVAL;
3582 goto err;
3583 }
3584
3585 if (field_pos != event->n_fields) {
3586 ret = -EINVAL;
3587 goto err;
3588 }
3589
3590 data->fn = action_trace;
3591 data->onmatch.synth_event = event;
3592 data->onmatch.var_ref_idx = var_ref_idx;
3593 out:
3594 return ret;
3595 err:
3596 mutex_lock(&synth_event_mutex);
3597 event->ref--;
3598 mutex_unlock(&synth_event_mutex);
3599
3600 goto out;
3601}
3602
3603static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3604{
3605 char *match_event, *match_event_system;
3606 char *synth_event_name, *params;
3607 struct action_data *data;
3608 int ret = -EINVAL;
3609
3610 data = kzalloc(sizeof(*data), GFP_KERNEL);
3611 if (!data)
3612 return ERR_PTR(-ENOMEM);
3613
3614 match_event = strsep(&str, ")");
3615 if (!match_event || !str)
3616 goto free;
3617
3618 match_event_system = strsep(&match_event, ".");
3619 if (!match_event)
3620 goto free;
3621
3622 if (IS_ERR(event_file(tr, match_event_system, match_event)))
3623 goto free;
3624
3625 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3626 if (!data->onmatch.match_event) {
3627 ret = -ENOMEM;
3628 goto free;
3629 }
3630
3631 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3632 if (!data->onmatch.match_event_system) {
3633 ret = -ENOMEM;
3634 goto free;
3635 }
3636
3637 strsep(&str, ".");
3638 if (!str)
3639 goto free;
3640
3641 synth_event_name = strsep(&str, "(");
3642 if (!synth_event_name || !str)
3643 goto free;
3644
3645 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3646 if (!data->onmatch.synth_event_name) {
3647 ret = -ENOMEM;
3648 goto free;
3649 }
3650
3651 params = strsep(&str, ")");
3652 if (!params || !str || (str && strlen(str)))
3653 goto free;
3654
3655 ret = parse_action_params(params, data);
3656 if (ret)
3657 goto free;
3658 out:
3659 return data;
3660 free:
3661 onmatch_destroy(data);
3662 data = ERR_PTR(ret);
3663 goto out;
3664}
3665
7ef224d1
TZ		 3666static int create_hitcount_val(struct hist_trigger_data *hist_data)
3667{
3668 hist_data->fields[HITCOUNT_IDX] =
30350d65 3669 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
7ef224d1
TZ		 3670 if (!hist_data->fields[HITCOUNT_IDX])
3671 return -ENOMEM;
3672
3673 hist_data->n_vals++;
30350d65 3674 hist_data->n_fields++;
7ef224d1
TZ		 3675
3676 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3677 return -EINVAL;
3678
3679 return 0;
3680}
3681
30350d65
TZ		 3682static int __create_val_field(struct hist_trigger_data *hist_data,
3683 unsigned int val_idx,
3684 struct trace_event_file *file,
3685 char *var_name, char *field_str,
3686 unsigned long flags)
f2606835 3687{
100719dc 3688 struct hist_field *hist_field;
f2606835
TZ		 3689 int ret = 0;
3690
100719dc
TZ		 3691 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3692 if (IS_ERR(hist_field)) {
3693 ret = PTR_ERR(hist_field);
f2606835
TZ		 3694 goto out;
3695 }
3696
100719dc
TZ		 3697 hist_data->fields[val_idx] = hist_field;
3698
f2606835 3699 ++hist_data->n_vals;
30350d65 3700 ++hist_data->n_fields;
f2606835 3701
30350d65 3702 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
f2606835
TZ		 3703 ret = -EINVAL;
3704 out:
3705 return ret;
3706}
3707
30350d65
TZ		 3708static int create_val_field(struct hist_trigger_data *hist_data,
3709 unsigned int val_idx,
3710 struct trace_event_file *file,
3711 char *field_str)
3712{
3713 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3714 return -EINVAL;
3715
3716 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3717}
3718
3719static int create_var_field(struct hist_trigger_data *hist_data,
3720 unsigned int val_idx,
3721 struct trace_event_file *file,
3722 char *var_name, char *expr_str)
3723{
3724 unsigned long flags = 0;
3725
3726 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3727 return -EINVAL;
3728 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3729 return -EINVAL;
3730 }
3731
3732 flags |= HIST_FIELD_FL_VAR;
3733 hist_data->n_vars++;
3734 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3735 return -EINVAL;
3736
3737 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3738}
3739
7ef224d1
TZ		 3740static int create_val_fields(struct hist_trigger_data *hist_data,
3741 struct trace_event_file *file)
3742{
f2606835 3743 char *fields_str, *field_str;
30350d65 3744 unsigned int i, j = 1;
7ef224d1
TZ		 3745 int ret;
3746
3747 ret = create_hitcount_val(hist_data);
f2606835
TZ		 3748 if (ret)
3749 goto out;
7ef224d1 3750
f2606835
TZ		 3751 fields_str = hist_data->attrs->vals_str;
3752 if (!fields_str)
3753 goto out;
3754
3755 strsep(&fields_str, "=");
3756 if (!fields_str)
3757 goto out;
3758
3759 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3760 j < TRACING_MAP_VALS_MAX; i++) {
3761 field_str = strsep(&fields_str, ",");
3762 if (!field_str)
3763 break;
30350d65 3764
f2606835
TZ		 3765 if (strcmp(field_str, "hitcount") == 0)
3766 continue;
30350d65 3767
f2606835
TZ		 3768 ret = create_val_field(hist_data, j++, file, field_str);
3769 if (ret)
3770 goto out;
3771 }
30350d65 3772
f2606835
TZ		 3773 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3774 ret = -EINVAL;
3775 out:
7ef224d1
TZ		 3776 return ret;
3777}
3778
3779static int create_key_field(struct hist_trigger_data *hist_data,
3780 unsigned int key_idx,
76a3b0c8 3781 unsigned int key_offset,
7ef224d1
TZ		 3782 struct trace_event_file *file,
3783 char *field_str)
3784{
30350d65 3785 struct hist_field *hist_field = NULL;
100719dc 3786
7ef224d1
TZ		 3787 unsigned long flags = 0;
3788 unsigned int key_size;
3789 int ret = 0;
3790
30350d65 3791 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
7ef224d1
TZ		 3792 return -EINVAL;
3793
3794 flags |= HIST_FIELD_FL_KEY;
3795
69a0200c
TZ		 3796 if (strcmp(field_str, "stacktrace") == 0) {
3797 flags |= HIST_FIELD_FL_STACKTRACE;
3798 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
30350d65 3799 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
69a0200c 3800 } else {
100719dc
TZ		 3801 hist_field = parse_expr(hist_data, file, field_str, flags,
3802 NULL, 0);
3803 if (IS_ERR(hist_field)) {
3804 ret = PTR_ERR(hist_field);
3805 goto out;
69a0200c
TZ		 3806 }
3807
067fe038
TZ		 3808 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
3809 destroy_hist_field(hist_field, 0);
3810 ret = -EINVAL;
3811 goto out;
3812 }
3813
100719dc 3814 key_size = hist_field->size;
7ef224d1
TZ		 3815 }
3816
100719dc 3817 hist_data->fields[key_idx] = hist_field;
7ef224d1
TZ		 3818
3819 key_size = ALIGN(key_size, sizeof(u64));
3820 hist_data->fields[key_idx]->size = key_size;
76a3b0c8 3821 hist_data->fields[key_idx]->offset = key_offset;
100719dc 3822
76a3b0c8 3823 hist_data->key_size += key_size;
100719dc 3824
7ef224d1
TZ		 3825 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3826 ret = -EINVAL;
3827 goto out;
3828 }
3829
3830 hist_data->n_keys++;
30350d65 3831 hist_data->n_fields++;
7ef224d1
TZ		 3832
3833 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3834 return -EINVAL;
3835
3836 ret = key_size;
3837 out:
3838 return ret;
3839}
3840
3841static int create_key_fields(struct hist_trigger_data *hist_data,
3842 struct trace_event_file *file)
3843{
76a3b0c8 3844 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
7ef224d1
TZ		 3845 char *fields_str, *field_str;
3846 int ret = -EINVAL;
3847
3848 fields_str = hist_data->attrs->keys_str;
3849 if (!fields_str)
3850 goto out;
3851
3852 strsep(&fields_str, "=");
3853 if (!fields_str)
3854 goto out;
3855
76a3b0c8 3856 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
7ef224d1
TZ		 3857 field_str = strsep(&fields_str, ",");
3858 if (!field_str)
3859 break;
76a3b0c8
TZ		 3860 ret = create_key_field(hist_data, i, key_offset,
3861 file, field_str);
7ef224d1
TZ		 3862 if (ret < 0)
3863 goto out;
76a3b0c8 3864 key_offset += ret;
7ef224d1
TZ		 3865 }
3866 if (fields_str) {
3867 ret = -EINVAL;
3868 goto out;
3869 }
3870 ret = 0;
3871 out:
3872 return ret;
3873}
3874
30350d65
TZ		 3875static int create_var_fields(struct hist_trigger_data *hist_data,
3876 struct trace_event_file *file)
3877{
3878 unsigned int i, j = hist_data->n_vals;
3879 int ret = 0;
3880
3881 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3882
3883 for (i = 0; i < n_vars; i++) {
3884 char *var_name = hist_data->attrs->var_defs.name[i];
3885 char *expr = hist_data->attrs->var_defs.expr[i];
3886
3887 ret = create_var_field(hist_data, j++, file, var_name, expr);
3888 if (ret)
3889 goto out;
3890 }
3891 out:
3892 return ret;
3893}
3894
3895static void free_var_defs(struct hist_trigger_data *hist_data)
3896{
3897 unsigned int i;
3898
3899 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3900 kfree(hist_data->attrs->var_defs.name[i]);
3901 kfree(hist_data->attrs->var_defs.expr[i]);
3902 }
3903
3904 hist_data->attrs->var_defs.n_vars = 0;
3905}
3906
3907static int parse_var_defs(struct hist_trigger_data *hist_data)
3908{
3909 char *s, *str, *var_name, *field_str;
3910 unsigned int i, j, n_vars = 0;
3911 int ret = 0;
3912
3913 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3914 str = hist_data->attrs->assignment_str[i];
3915 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3916 field_str = strsep(&str, ",");
3917 if (!field_str)
3918 break;
3919
3920 var_name = strsep(&field_str, "=");
3921 if (!var_name || !field_str) {
3922 ret = -EINVAL;
3923 goto free;
3924 }
3925
3926 if (n_vars == TRACING_MAP_VARS_MAX) {
3927 ret = -EINVAL;
3928 goto free;
3929 }
3930
3931 s = kstrdup(var_name, GFP_KERNEL);
3932 if (!s) {
3933 ret = -ENOMEM;
3934 goto free;
3935 }
3936 hist_data->attrs->var_defs.name[n_vars] = s;
3937
3938 s = kstrdup(field_str, GFP_KERNEL);
3939 if (!s) {
3940 kfree(hist_data->attrs->var_defs.name[n_vars]);
3941 ret = -ENOMEM;
3942 goto free;
3943 }
3944 hist_data->attrs->var_defs.expr[n_vars++] = s;
3945
3946 hist_data->attrs->var_defs.n_vars = n_vars;
3947 }
3948 }
3949
3950 return ret;
3951 free:
3952 free_var_defs(hist_data);
3953
3954 return ret;
3955}
3956
7ef224d1
TZ		 3957static int create_hist_fields(struct hist_trigger_data *hist_data,
3958 struct trace_event_file *file)
3959{
3960 int ret;
3961
30350d65
TZ		 3962 ret = parse_var_defs(hist_data);
3963 if (ret)
3964 goto out;
3965
7ef224d1
TZ		 3966 ret = create_val_fields(hist_data, file);
3967 if (ret)
3968 goto out;
3969
30350d65 3970 ret = create_var_fields(hist_data, file);
7ef224d1
TZ		 3971 if (ret)
3972 goto out;
3973
30350d65
TZ		 3974 ret = create_key_fields(hist_data, file);
3975 if (ret)
3976 goto out;
7ef224d1 3977 out:
30350d65
TZ		 3978 free_var_defs(hist_data);
3979
7ef224d1
TZ		 3980 return ret;
3981}
3982
e62347d2
TZ		 3983static int is_descending(const char *str)
3984{
3985 if (!str)
3986 return 0;
3987
3988 if (strcmp(str, "descending") == 0)
3989 return 1;
3990
3991 if (strcmp(str, "ascending") == 0)
3992 return 0;
3993
3994 return -EINVAL;
3995}
3996
7ef224d1
TZ		 3997static int create_sort_keys(struct hist_trigger_data *hist_data)
3998{
e62347d2 3999 char *fields_str = hist_data->attrs->sort_key_str;
e62347d2
TZ		 4000 struct tracing_map_sort_key *sort_key;
4001 int descending, ret = 0;
30350d65 4002 unsigned int i, j, k;
e62347d2
TZ		 4003
4004 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4005
4006 if (!fields_str)
4007 goto out;
4008
4009 strsep(&fields_str, "=");
4010 if (!fields_str) {
4011 ret = -EINVAL;
4012 goto out;
4013 }
4014
4015 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
85013256 4016 struct hist_field *hist_field;
e62347d2 4017 char *field_str, *field_name;
85013256 4018 const char *test_name;
e62347d2
TZ		 4019
4020 sort_key = &hist_data->sort_keys[i];
4021
4022 field_str = strsep(&fields_str, ",");
4023 if (!field_str) {
4024 if (i == 0)
4025 ret = -EINVAL;
4026 break;
4027 }
4028
4029 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4030 ret = -EINVAL;
4031 break;
4032 }
7ef224d1 4033
e62347d2
TZ		 4034 field_name = strsep(&field_str, ".");
4035 if (!field_name) {
4036 ret = -EINVAL;
4037 break;
4038 }
4039
4040 if (strcmp(field_name, "hitcount") == 0) {
4041 descending = is_descending(field_str);
4042 if (descending < 0) {
4043 ret = descending;
4044 break;
4045 }
4046 sort_key->descending = descending;
4047 continue;
4048 }
7ef224d1 4049
30350d65
TZ		 4050 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4051 unsigned int idx;
4052
85013256 4053 hist_field = hist_data->fields[j];
30350d65
TZ		 4054 if (hist_field->flags & HIST_FIELD_FL_VAR)
4055 continue;
4056
4057 idx = k++;
4058
85013256
TZ		 4059 test_name = hist_field_name(hist_field, 0);
4060
4061 if (strcmp(field_name, test_name) == 0) {
30350d65 4062 sort_key->field_idx = idx;
e62347d2
TZ		 4063 descending = is_descending(field_str);
4064 if (descending < 0) {
4065 ret = descending;
4066 goto out;
4067 }
4068 sort_key->descending = descending;
4069 break;
4070 }
4071 }
4072 if (j == hist_data->n_fields) {
4073 ret = -EINVAL;
4074 break;
4075 }
4076 }
30350d65 4077
e62347d2
TZ		 4078 hist_data->n_sort_keys = i;
4079 out:
7ef224d1
TZ		 4080 return ret;
4081}
4082
0212e2aa
TZ		 4083static void destroy_actions(struct hist_trigger_data *hist_data)
4084{
4085 unsigned int i;
4086
4087 for (i = 0; i < hist_data->n_actions; i++) {
4088 struct action_data *data = hist_data->actions[i];
4089
c282a386
TZ		 4090 if (data->fn == action_trace)
4091 onmatch_destroy(data);
50450603
TZ		 4092 else if (data->fn == onmax_save)
4093 onmax_destroy(data);
c282a386
TZ		 4094 else
4095 kfree(data);
0212e2aa
TZ		 4096 }
4097}
4098
4099static int parse_actions(struct hist_trigger_data *hist_data)
4100{
c282a386
TZ		 4101 struct trace_array *tr = hist_data->event_file->tr;
4102 struct action_data *data;
0212e2aa
TZ		 4103 unsigned int i;
4104 int ret = 0;
4105 char *str;
4106
4107 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4108 str = hist_data->attrs->action_str[i];
c282a386
TZ		 4109
4110 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4111 char *action_str = str + strlen("onmatch(");
4112
4113 data = onmatch_parse(tr, action_str);
4114 if (IS_ERR(data)) {
4115 ret = PTR_ERR(data);
4116 break;
4117 }
4118 data->fn = action_trace;
50450603
TZ		 4119 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4120 char *action_str = str + strlen("onmax(");
4121
4122 data = onmax_parse(action_str);
4123 if (IS_ERR(data)) {
4124 ret = PTR_ERR(data);
4125 break;
4126 }
4127 data->fn = onmax_save;
c282a386
TZ		 4128 } else {
4129 ret = -EINVAL;
4130 break;
4131 }
4132
4133 hist_data->actions[hist_data->n_actions++] = data;
0212e2aa
TZ		 4134 }
4135
4136 return ret;
4137}
4138
4139static int create_actions(struct hist_trigger_data *hist_data,
4140 struct trace_event_file *file)
4141{
4142 struct action_data *data;
4143 unsigned int i;
4144 int ret = 0;
4145
4146 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4147 data = hist_data->actions[i];
c282a386
TZ		 4148
4149 if (data->fn == action_trace) {
4150 ret = onmatch_create(hist_data, file, data);
4151 if (ret)
4152 return ret;
50450603
TZ		 4153 } else if (data->fn == onmax_save) {
4154 ret = onmax_create(hist_data, data);
4155 if (ret)
4156 return ret;
c282a386 4157 }
0212e2aa
TZ		 4158 }
4159
4160 return ret;
4161}
4162
50450603
TZ		 4163static void print_actions(struct seq_file *m,
4164 struct hist_trigger_data *hist_data,
4165 struct tracing_map_elt *elt)
4166{
4167 unsigned int i;
4168
4169 for (i = 0; i < hist_data->n_actions; i++) {
4170 struct action_data *data = hist_data->actions[i];
4171
4172 if (data->fn == onmax_save)
4173 onmax_print(m, hist_data, elt, data);
4174 }
4175}
4176
4177static void print_onmax_spec(struct seq_file *m,
4178 struct hist_trigger_data *hist_data,
4179 struct action_data *data)
4180{
4181 unsigned int i;
4182
4183 seq_puts(m, ":onmax(");
4184 seq_printf(m, "%s", data->onmax.var_str);
4185 seq_printf(m, ").%s(", data->onmax.fn_name);
4186
4187 for (i = 0; i < hist_data->n_max_vars; i++) {
4188 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4189 if (i < hist_data->n_max_vars - 1)
4190 seq_puts(m, ",");
4191 }
4192 seq_puts(m, ")");
4193}
4194
c282a386
TZ		 4195static void print_onmatch_spec(struct seq_file *m,
4196 struct hist_trigger_data *hist_data,
4197 struct action_data *data)
4198{
4199 unsigned int i;
4200
4201 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4202 data->onmatch.match_event);
4203
4204 seq_printf(m, "%s(", data->onmatch.synth_event->name);
4205
4206 for (i = 0; i < data->n_params; i++) {
4207 if (i)
4208 seq_puts(m, ",");
4209 seq_printf(m, "%s", data->params[i]);
4210 }
4211
4212 seq_puts(m, ")");
4213}
4214
4215static void print_actions_spec(struct seq_file *m,
4216 struct hist_trigger_data *hist_data)
4217{
4218 unsigned int i;
4219
4220 for (i = 0; i < hist_data->n_actions; i++) {
4221 struct action_data *data = hist_data->actions[i];
4222
4223 if (data->fn == action_trace)
4224 print_onmatch_spec(m, hist_data, data);
50450603
TZ		 4225 else if (data->fn == onmax_save)
4226 print_onmax_spec(m, hist_data, data);
c282a386
TZ		 4227 }
4228}
4229
02205a67
TZ		 4230static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4231{
4232 unsigned int i;
4233
4234 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4235 kfree(hist_data->field_var_hists[i]->cmd);
4236 kfree(hist_data->field_var_hists[i]);
4237 }
4238}
4239
7ef224d1
TZ		 4240static void destroy_hist_data(struct hist_trigger_data *hist_data)
4241{
0212e2aa
TZ		 4242 if (!hist_data)
4243 return;
4244
7ef224d1
TZ		 4245 destroy_hist_trigger_attrs(hist_data->attrs);
4246 destroy_hist_fields(hist_data);
4247 tracing_map_destroy(hist_data->map);
0212e2aa
TZ		 4248
4249 destroy_actions(hist_data);
02205a67
TZ		 4250 destroy_field_vars(hist_data);
4251 destroy_field_var_hists(hist_data);
c282a386 4252 destroy_synth_var_refs(hist_data);
0212e2aa 4253
7ef224d1
TZ		 4254 kfree(hist_data);
4255}
4256
4257static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4258{
4259 struct tracing_map *map = hist_data->map;
4260 struct ftrace_event_field *field;
4261 struct hist_field *hist_field;
d50c744e 4262 int i, idx;
7ef224d1
TZ		 4263
4264 for_each_hist_field(i, hist_data) {
4265 hist_field = hist_data->fields[i];
4266 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4267 tracing_map_cmp_fn_t cmp_fn;
4268
4269 field = hist_field->field;
4270
69a0200c
TZ		 4271 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4272 cmp_fn = tracing_map_cmp_none;
ad42febe
TZ		 4273 else if (!field)
4274 cmp_fn = tracing_map_cmp_num(hist_field->size,
4275 hist_field->is_signed);
69a0200c 4276 else if (is_string_field(field))
7ef224d1
TZ		 4277 cmp_fn = tracing_map_cmp_string;
4278 else
4279 cmp_fn = tracing_map_cmp_num(field->size,
4280 field->is_signed);
76a3b0c8
TZ		 4281 idx = tracing_map_add_key_field(map,
4282 hist_field->offset,
4283 cmp_fn);
30350d65 4284 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
7ef224d1
TZ		 4285 idx = tracing_map_add_sum_field(map);
4286
4287 if (idx < 0)
4288 return idx;
30350d65
TZ		 4289
4290 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4291 idx = tracing_map_add_var(map);
4292 if (idx < 0)
4293 return idx;
4294 hist_field->var.idx = idx;
4295 hist_field->var.hist_data = hist_data;
4296 }
7ef224d1
TZ		 4297 }
4298
4299 return 0;
4300}
4301
4302static struct hist_trigger_data *
4303create_hist_data(unsigned int map_bits,
4304 struct hist_trigger_attrs *attrs,
30350d65
TZ		 4305 struct trace_event_file *file,
4306 bool remove)
7ef224d1 4307{
6b4827ad 4308 const struct tracing_map_ops *map_ops = NULL;
7ef224d1
TZ		 4309 struct hist_trigger_data *hist_data;
4310 int ret = 0;
4311
4312 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4313 if (!hist_data)
4314 return ERR_PTR(-ENOMEM);
4315
4316 hist_data->attrs = attrs;
30350d65 4317 hist_data->remove = remove;
067fe038 4318 hist_data->event_file = file;
7ef224d1 4319
0212e2aa
TZ		 4320 ret = parse_actions(hist_data);
4321 if (ret)
4322 goto free;
4323
7ef224d1
TZ		 4324 ret = create_hist_fields(hist_data, file);
4325 if (ret)
4326 goto free;
4327
4328 ret = create_sort_keys(hist_data);
4329 if (ret)
4330 goto free;
4331
af6a29bc 4332 map_ops = &hist_trigger_elt_data_ops;
6b4827ad 4333
7ef224d1 4334 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
6b4827ad 4335 map_ops, hist_data);
7ef224d1
TZ		 4336 if (IS_ERR(hist_data->map)) {
4337 ret = PTR_ERR(hist_data->map);
4338 hist_data->map = NULL;
4339 goto free;
4340 }
4341
4342 ret = create_tracing_map_fields(hist_data);
4343 if (ret)
4344 goto free;
7ef224d1
TZ		 4345 out:
4346 return hist_data;
4347 free:
4348 hist_data->attrs = NULL;
4349
4350 destroy_hist_data(hist_data);
4351
4352 hist_data = ERR_PTR(ret);
4353
4354 goto out;
4355}
4356
4357static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
fbd302cb 4358 struct tracing_map_elt *elt, void *rec,
067fe038
TZ		 4359 struct ring_buffer_event *rbe,
4360 u64 *var_ref_vals)
7ef224d1 4361{
067fe038 4362 struct hist_elt_data *elt_data;
7ef224d1 4363 struct hist_field *hist_field;
30350d65 4364 unsigned int i, var_idx;
7ef224d1
TZ		 4365 u64 hist_val;
4366
067fe038
TZ		 4367 elt_data = elt->private_data;
4368 elt_data->var_ref_vals = var_ref_vals;
4369
7ef224d1
TZ		 4370 for_each_hist_val_field(i, hist_data) {
4371 hist_field = hist_data->fields[i];
df35d93b 4372 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4373 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4374 var_idx = hist_field->var.idx;
4375 tracing_map_set_var(elt, var_idx, hist_val);
4376 continue;
4377 }
7ef224d1
TZ		 4378 tracing_map_update_sum(elt, i, hist_val);
4379 }
30350d65
TZ		 4380
4381 for_each_hist_key_field(i, hist_data) {
4382 hist_field = hist_data->fields[i];
4383 if (hist_field->flags & HIST_FIELD_FL_VAR) {
df35d93b 4384 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
30350d65
TZ		 4385 var_idx = hist_field->var.idx;
4386 tracing_map_set_var(elt, var_idx, hist_val);
4387 }
4388 }
02205a67
TZ		 4389
4390 update_field_vars(hist_data, elt, rbe, rec);
7ef224d1
TZ		 4391}
4392
6a475cb1
TZ		 4393static inline void add_to_key(char *compound_key, void *key,
4394 struct hist_field *key_field, void *rec)
4395{
4396 size_t size = key_field->size;
4397
4398 if (key_field->flags & HIST_FIELD_FL_STRING) {
4399 struct ftrace_event_field *field;
4400
4401 field = key_field->field;
4402 if (field->filter_type == FILTER_DYN_STRING)
4403 size = *(u32 *)(rec + field->offset) >> 16;
4404 else if (field->filter_type == FILTER_PTR_STRING)
4405 size = strlen(key);
4406 else if (field->filter_type == FILTER_STATIC_STRING)
4407 size = field->size;
4408
4409 /* ensure NULL-termination */
4410 if (size > key_field->size - 1)
4411 size = key_field->size - 1;
4412 }
4413
4414 memcpy(compound_key + key_field->offset, key, size);
4415}
4416
0212e2aa
TZ		 4417static void
4418hist_trigger_actions(struct hist_trigger_data *hist_data,
4419 struct tracing_map_elt *elt, void *rec,
4420 struct ring_buffer_event *rbe, u64 *var_ref_vals)
4421{
4422 struct action_data *data;
4423 unsigned int i;
4424
4425 for (i = 0; i < hist_data->n_actions; i++) {
4426 data = hist_data->actions[i];
4427 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4428 }
4429}
4430
1ac4f51c 4431static void event_hist_trigger(struct event_trigger_data *data, void *rec,
fbd302cb 4432 struct ring_buffer_event *rbe)
7ef224d1
TZ		 4433{
4434 struct hist_trigger_data *hist_data = data->private_data;
6a475cb1 4435 bool use_compound_key = (hist_data->n_keys > 1);
69a0200c 4436 unsigned long entries[HIST_STACKTRACE_DEPTH];
067fe038 4437 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
76a3b0c8 4438 char compound_key[HIST_KEY_SIZE_MAX];
df35d93b 4439 struct tracing_map_elt *elt = NULL;
69a0200c 4440 struct stack_trace stacktrace;
7ef224d1 4441 struct hist_field *key_field;
7ef224d1
TZ		 4442 u64 field_contents;
4443 void *key = NULL;
4444 unsigned int i;
4445
6a475cb1 4446 memset(compound_key, 0, hist_data->key_size);
76a3b0c8 4447
7ef224d1
TZ		 4448 for_each_hist_key_field(i, hist_data) {
4449 key_field = hist_data->fields[i];
4450
69a0200c
TZ		 4451 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4452 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4453 stacktrace.entries = entries;
4454 stacktrace.nr_entries = 0;
4455 stacktrace.skip = HIST_STACKTRACE_SKIP;
76a3b0c8 4456
69a0200c
TZ		 4457 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4458 save_stack_trace(&stacktrace);
4459
4460 key = entries;
4461 } else {
df35d93b 4462 field_contents = key_field->fn(key_field, elt, rbe, rec);
6a475cb1 4463 if (key_field->flags & HIST_FIELD_FL_STRING) {
69a0200c 4464 key = (void *)(unsigned long)field_contents;
6a475cb1
TZ		 4465 use_compound_key = true;
4466 } else
69a0200c 4467 key = (void *)&field_contents;
76a3b0c8 4468 }
6a475cb1
TZ		 4469
4470 if (use_compound_key)
4471 add_to_key(compound_key, key, key_field, rec);
7ef224d1
TZ		 4472 }
4473
6a475cb1 4474 if (use_compound_key)
76a3b0c8
TZ		 4475 key = compound_key;
4476
067fe038
TZ		 4477 if (hist_data->n_var_refs &&
4478 !resolve_var_refs(hist_data, key, var_ref_vals, false))
4479 return;
4480
7ef224d1 4481 elt = tracing_map_insert(hist_data->map, key);
067fe038
TZ		 4482 if (!elt)
4483 return;
4484
4485 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
0212e2aa
TZ		 4486
4487 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4488 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
7ef224d1
TZ		 4489}
4490
69a0200c
TZ		 4491static void hist_trigger_stacktrace_print(struct seq_file *m,
4492 unsigned long *stacktrace_entries,
4493 unsigned int max_entries)
4494{
4495 char str[KSYM_SYMBOL_LEN];
4496 unsigned int spaces = 8;
4497 unsigned int i;
4498
4499 for (i = 0; i < max_entries; i++) {
4500 if (stacktrace_entries[i] == ULONG_MAX)
4501 return;
4502
4503 seq_printf(m, "%*c", 1 + spaces, ' ');
4504 sprint_symbol(str, stacktrace_entries[i]);
4505 seq_printf(m, "%s\n", str);
4506 }
4507}
4508
7ef224d1
TZ		 4509static void
4510hist_trigger_entry_print(struct seq_file *m,
4511 struct hist_trigger_data *hist_data, void *key,
4512 struct tracing_map_elt *elt)
4513{
4514 struct hist_field *key_field;
c6afad49 4515 char str[KSYM_SYMBOL_LEN];
69a0200c 4516 bool multiline = false;
85013256 4517 const char *field_name;
7ef224d1
TZ		 4518 unsigned int i;
4519 u64 uval;
4520
4521 seq_puts(m, "{ ");
4522
4523 for_each_hist_key_field(i, hist_data) {
4524 key_field = hist_data->fields[i];
4525
4526 if (i > hist_data->n_vals)
4527 seq_puts(m, ", ");
4528
85013256
TZ		 4529 field_name = hist_field_name(key_field, 0);
4530
0c4a6b46
TZ		 4531 if (key_field->flags & HIST_FIELD_FL_HEX) {
4532 uval = *(u64 *)(key + key_field->offset);
85013256 4533 seq_printf(m, "%s: %llx", field_name, uval);
c6afad49
TZ		 4534 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4535 uval = *(u64 *)(key + key_field->offset);
4536 sprint_symbol_no_offset(str, uval);
85013256
TZ		 4537 seq_printf(m, "%s: [%llx] %-45s", field_name,
4538 uval, str);
c6afad49
TZ		 4539 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4540 uval = *(u64 *)(key + key_field->offset);
4541 sprint_symbol(str, uval);
85013256
TZ		 4542 seq_printf(m, "%s: [%llx] %-55s", field_name,
4543 uval, str);
6b4827ad 4544 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
af6a29bc
TZ		 4545 struct hist_elt_data *elt_data = elt->private_data;
4546 char *comm;
4547
4548 if (WARN_ON_ONCE(!elt_data))
4549 return;
4550
4551 comm = elt_data->comm;
6b4827ad
TZ		 4552
4553 uval = *(u64 *)(key + key_field->offset);
85013256
TZ		 4554 seq_printf(m, "%s: %-16s[%10llu]", field_name,
4555 comm, uval);
31696198
TZ		 4556 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4557 const char *syscall_name;
4558
4559 uval = *(u64 *)(key + key_field->offset);
4560 syscall_name = get_syscall_name(uval);
4561 if (!syscall_name)
4562 syscall_name = "unknown_syscall";
4563
85013256
TZ		 4564 seq_printf(m, "%s: %-30s[%3llu]", field_name,
4565 syscall_name, uval);
69a0200c
TZ		 4566 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4567 seq_puts(m, "stacktrace:\n");
4568 hist_trigger_stacktrace_print(m,
4569 key + key_field->offset,
4570 HIST_STACKTRACE_DEPTH);
4571 multiline = true;
4b94f5b7 4572 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
85013256 4573 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4b94f5b7 4574 *(u64 *)(key + key_field->offset));
0c4a6b46 4575 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
85013256 4576 seq_printf(m, "%s: %-50s", field_name,
76a3b0c8 4577 (char *)(key + key_field->offset));
7ef224d1 4578 } else {
76a3b0c8 4579 uval = *(u64 *)(key + key_field->offset);
85013256 4580 seq_printf(m, "%s: %10llu", field_name, uval);
7ef224d1
TZ		 4581 }
4582 }
4583
69a0200c
TZ		 4584 if (!multiline)
4585 seq_puts(m, " ");
4586
4587 seq_puts(m, "}");
7ef224d1
TZ		 4588
4589 seq_printf(m, " hitcount: %10llu",
4590 tracing_map_read_sum(elt, HITCOUNT_IDX));
4591
f2606835 4592 for (i = 1; i < hist_data->n_vals; i++) {
85013256
TZ		 4593 field_name = hist_field_name(hist_data->fields[i], 0);
4594
100719dc
TZ		 4595 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4596 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
30350d65
TZ		 4597 continue;
4598
0c4a6b46 4599 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
85013256 4600 seq_printf(m, " %s: %10llx", field_name,
0c4a6b46
TZ		 4601 tracing_map_read_sum(elt, i));
4602 } else {
85013256 4603 seq_printf(m, " %s: %10llu", field_name,
0c4a6b46
TZ		 4604 tracing_map_read_sum(elt, i));
4605 }
f2606835
TZ		 4606 }
4607
50450603
TZ		 4608 print_actions(m, hist_data, elt);
4609
7ef224d1
TZ		 4610 seq_puts(m, "\n");
4611}
4612
4613static int print_entries(struct seq_file *m,
4614 struct hist_trigger_data *hist_data)
4615{
4616 struct tracing_map_sort_entry **sort_entries = NULL;
4617 struct tracing_map *map = hist_data->map;
d50c744e 4618 int i, n_entries;
7ef224d1
TZ		 4619
4620 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4621 hist_data->n_sort_keys,
4622 &sort_entries);
4623 if (n_entries < 0)
4624 return n_entries;
4625
4626 for (i = 0; i < n_entries; i++)
4627 hist_trigger_entry_print(m, hist_data,
4628 sort_entries[i]->key,
4629 sort_entries[i]->elt);
4630
4631 tracing_map_destroy_sort_entries(sort_entries, n_entries);
4632
4633 return n_entries;
4634}
4635
52a7f16d
TZ		 4636static void hist_trigger_show(struct seq_file *m,
4637 struct event_trigger_data *data, int n)
7ef224d1 4638{
7ef224d1 4639 struct hist_trigger_data *hist_data;
6e7a2398 4640 int n_entries;
7ef224d1 4641
52a7f16d
TZ		 4642 if (n > 0)
4643 seq_puts(m, "\n\n");
7ef224d1
TZ		 4644
4645 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4646 data->ops->print(m, data->ops, data);
52a7f16d 4647 seq_puts(m, "#\n\n");
7ef224d1
TZ		 4648
4649 hist_data = data->private_data;
4650 n_entries = print_entries(m, hist_data);
6e7a2398 4651 if (n_entries < 0)
7ef224d1 4652 n_entries = 0;
7ef224d1
TZ		 4653
4654 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
4655 (u64)atomic64_read(&hist_data->map->hits),
4656 n_entries, (u64)atomic64_read(&hist_data->map->drops));
52a7f16d
TZ		 4657}
4658
4659static int hist_show(struct seq_file *m, void *v)
4660{
4661 struct event_trigger_data *data;
4662 struct trace_event_file *event_file;
4663 int n = 0, ret = 0;
4664
4665 mutex_lock(&event_mutex);
4666
4667 event_file = event_file_data(m->private);
4668 if (unlikely(!event_file)) {
4669 ret = -ENODEV;
4670 goto out_unlock;
4671 }
4672
4673 list_for_each_entry_rcu(data, &event_file->triggers, list) {
4674 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4675 hist_trigger_show(m, data, n++);
4676 }
4677
7ef224d1
TZ		 4678 out_unlock:
4679 mutex_unlock(&event_mutex);
4680
4681 return ret;
4682}
4683
4684static int event_hist_open(struct inode *inode, struct file *file)
4685{
4686 return single_open(file, hist_show, file);
4687}
4688
4689const struct file_operations event_hist_fops = {
4690 .open = event_hist_open,
4691 .read = seq_read,
4692 .llseek = seq_lseek,
4693 .release = single_release,
4694};
4695
4696static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4697{
85013256
TZ		 4698 const char *field_name = hist_field_name(hist_field, 0);
4699
30350d65
TZ		 4700 if (hist_field->var.name)
4701 seq_printf(m, "%s=", hist_field->var.name);
4702
ad42febe
TZ		 4703 if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4704 seq_puts(m, "common_timestamp");
8b7622bf
TZ		 4705 else if (hist_field->flags & HIST_FIELD_FL_CPU)
4706 seq_puts(m, "cpu");
067fe038 4707 else if (field_name) {
7e8b88a3
TZ		 4708 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4709 hist_field->flags & HIST_FIELD_FL_ALIAS)
067fe038 4710 seq_putc(m, '$');
ad42febe 4711 seq_printf(m, "%s", field_name);
067fe038 4712 }
ad42febe 4713
0c4a6b46
TZ		 4714 if (hist_field->flags) {
4715 const char *flags_str = get_hist_field_flags(hist_field);
4716
4717 if (flags_str)
4718 seq_printf(m, ".%s", flags_str);
4719 }
7ef224d1
TZ		 4720}
4721
4722static int event_hist_trigger_print(struct seq_file *m,
4723 struct event_trigger_ops *ops,
4724 struct event_trigger_data *data)
4725{
4726 struct hist_trigger_data *hist_data = data->private_data;
30350d65
TZ		 4727 struct hist_field *field;
4728 bool have_var = false;
7ef224d1
TZ		 4729 unsigned int i;
4730
5463bfda
TZ		 4731 seq_puts(m, "hist:");
4732
4733 if (data->name)
4734 seq_printf(m, "%s:", data->name);
4735
4736 seq_puts(m, "keys=");
7ef224d1
TZ		 4737
4738 for_each_hist_key_field(i, hist_data) {
30350d65 4739 field = hist_data->fields[i];
7ef224d1
TZ		 4740
4741 if (i > hist_data->n_vals)
4742 seq_puts(m, ",");
4743
30350d65 4744 if (field->flags & HIST_FIELD_FL_STACKTRACE)
69a0200c
TZ		 4745 seq_puts(m, "stacktrace");
4746 else
30350d65 4747 hist_field_print(m, field);
7ef224d1
TZ		 4748 }
4749
4750 seq_puts(m, ":vals=");
f2606835
TZ		 4751
4752 for_each_hist_val_field(i, hist_data) {
30350d65
TZ		 4753 field = hist_data->fields[i];
4754 if (field->flags & HIST_FIELD_FL_VAR) {
4755 have_var = true;
4756 continue;
4757 }
4758
f2606835
TZ		 4759 if (i == HITCOUNT_IDX)
4760 seq_puts(m, "hitcount");
4761 else {
4762 seq_puts(m, ",");
30350d65
TZ		 4763 hist_field_print(m, field);
4764 }
4765 }
4766
4767 if (have_var) {
4768 unsigned int n = 0;
4769
4770 seq_puts(m, ":");
4771
4772 for_each_hist_val_field(i, hist_data) {
4773 field = hist_data->fields[i];
4774
4775 if (field->flags & HIST_FIELD_FL_VAR) {
4776 if (n++)
4777 seq_puts(m, ",");
4778 hist_field_print(m, field);
4779 }
f2606835
TZ		 4780 }
4781 }
7ef224d1
TZ		 4782
4783 seq_puts(m, ":sort=");
e62347d2
TZ		 4784
4785 for (i = 0; i < hist_data->n_sort_keys; i++) {
4786 struct tracing_map_sort_key *sort_key;
30350d65
TZ		 4787 unsigned int idx, first_key_idx;
4788
4789 /* skip VAR vals */
4790 first_key_idx = hist_data->n_vals - hist_data->n_vars;
e62347d2
TZ		 4791
4792 sort_key = &hist_data->sort_keys[i];
ad42febe
TZ		 4793 idx = sort_key->field_idx;
4794
1a361dfc 4795 if (WARN_ON(idx >= HIST_FIELDS_MAX))
ad42febe 4796 return -EINVAL;
e62347d2
TZ		 4797
4798 if (i > 0)
4799 seq_puts(m, ",");
4800
ad42febe 4801 if (idx == HITCOUNT_IDX)
e62347d2 4802 seq_puts(m, "hitcount");
30350d65
TZ		 4803 else {
4804 if (idx >= first_key_idx)
4805 idx += hist_data->n_vars;
e62347d2 4806 hist_field_print(m, hist_data->fields[idx]);
30350d65 4807 }
e62347d2
TZ		 4808
4809 if (sort_key->descending)
4810 seq_puts(m, ".descending");
4811 }
7ef224d1
TZ		 4812 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
4813
c282a386
TZ		 4814 print_actions_spec(m, hist_data);
4815
7ef224d1
TZ		 4816 if (data->filter_str)
4817 seq_printf(m, " if %s", data->filter_str);
4818
83e99914
TZ		 4819 if (data->paused)
4820 seq_puts(m, " [paused]");
4821 else
4822 seq_puts(m, " [active]");
7ef224d1
TZ		 4823
4824 seq_putc(m, '\n');
4825
4826 return 0;
4827}
4828
5463bfda
TZ		 4829static int event_hist_trigger_init(struct event_trigger_ops *ops,
4830 struct event_trigger_data *data)
4831{
4832 struct hist_trigger_data *hist_data = data->private_data;
4833
4834 if (!data->ref && hist_data->attrs->name)
4835 save_named_trigger(hist_data->attrs->name, data);
4836
4837 data->ref++;
4838
4839 return 0;
4840}
4841
02205a67
TZ		 4842static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
4843{
4844 struct trace_event_file *file;
4845 unsigned int i;
4846 char *cmd;
4847 int ret;
4848
4849 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4850 file = hist_data->field_var_hists[i]->hist_data->event_file;
4851 cmd = hist_data->field_var_hists[i]->cmd;
4852 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
4853 "!hist", "hist", cmd);
4854 }
4855}
4856
7ef224d1
TZ		 4857static void event_hist_trigger_free(struct event_trigger_ops *ops,
4858 struct event_trigger_data *data)
4859{
4860 struct hist_trigger_data *hist_data = data->private_data;
4861
4862 if (WARN_ON_ONCE(data->ref <= 0))
4863 return;
4864
4865 data->ref--;
4866 if (!data->ref) {
5463bfda
TZ		 4867 if (data->name)
4868 del_named_trigger(data);
067fe038 4869
7ef224d1 4870 trigger_data_free(data);
067fe038
TZ		 4871
4872 remove_hist_vars(hist_data);
4873
02205a67
TZ		 4874 unregister_field_var_hists(hist_data);
4875
7ef224d1
TZ		 4876 destroy_hist_data(hist_data);
4877 }
4878}
4879
4880static struct event_trigger_ops event_hist_trigger_ops = {
4881 .func = event_hist_trigger,
4882 .print = event_hist_trigger_print,
5463bfda 4883 .init = event_hist_trigger_init,
7ef224d1
TZ		 4884 .free = event_hist_trigger_free,
4885};
4886
5463bfda
TZ		 4887static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
4888 struct event_trigger_data *data)
4889{
4890 data->ref++;
4891
4892 save_named_trigger(data->named_data->name, data);
4893
4894 event_hist_trigger_init(ops, data->named_data);
4895
4896 return 0;
4897}
4898
4899static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
4900 struct event_trigger_data *data)
4901{
4902 if (WARN_ON_ONCE(data->ref <= 0))
4903 return;
4904
4905 event_hist_trigger_free(ops, data->named_data);
4906
4907 data->ref--;
4908 if (!data->ref) {
4909 del_named_trigger(data);
4910 trigger_data_free(data);
4911 }
4912}
4913
4914static struct event_trigger_ops event_hist_trigger_named_ops = {
4915 .func = event_hist_trigger,
4916 .print = event_hist_trigger_print,
4917 .init = event_hist_trigger_named_init,
4918 .free = event_hist_trigger_named_free,
4919};
4920
7ef224d1
TZ		 4921static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
4922 char *param)
4923{
4924 return &event_hist_trigger_ops;
4925}
4926
e86ae9ba
TZ		 4927static void hist_clear(struct event_trigger_data *data)
4928{
4929 struct hist_trigger_data *hist_data = data->private_data;
e86ae9ba 4930
5463bfda
TZ		 4931 if (data->name)
4932 pause_named_trigger(data);
e86ae9ba
TZ		 4933
4934 synchronize_sched();
4935
4936 tracing_map_clear(hist_data->map);
4937
5463bfda
TZ		 4938 if (data->name)
4939 unpause_named_trigger(data);
4940}
4941
4942static bool compatible_field(struct ftrace_event_field *field,
4943 struct ftrace_event_field *test_field)
4944{
4945 if (field == test_field)
4946 return true;
4947 if (field == NULL || test_field == NULL)
4948 return false;
4949 if (strcmp(field->name, test_field->name) != 0)
4950 return false;
4951 if (strcmp(field->type, test_field->type) != 0)
4952 return false;
4953 if (field->size != test_field->size)
4954 return false;
4955 if (field->is_signed != test_field->is_signed)
4956 return false;
4957
4958 return true;
e86ae9ba
TZ		 4959}
4960
52a7f16d 4961static bool hist_trigger_match(struct event_trigger_data *data,
5463bfda
TZ		 4962 struct event_trigger_data *data_test,
4963 struct event_trigger_data *named_data,
4964 bool ignore_filter)
52a7f16d
TZ		 4965{
4966 struct tracing_map_sort_key *sort_key, *sort_key_test;
4967 struct hist_trigger_data *hist_data, *hist_data_test;
4968 struct hist_field *key_field, *key_field_test;
4969 unsigned int i;
4970
5463bfda
TZ		 4971 if (named_data && (named_data != data_test) &&
4972 (named_data != data_test->named_data))
4973 return false;
4974
4975 if (!named_data && is_named_trigger(data_test))
4976 return false;
4977
52a7f16d
TZ		 4978 hist_data = data->private_data;
4979 hist_data_test = data_test->private_data;
4980
4981 if (hist_data->n_vals != hist_data_test->n_vals ||
4982 hist_data->n_fields != hist_data_test->n_fields ||
4983 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
4984 return false;
4985
5463bfda
TZ		 4986 if (!ignore_filter) {
4987 if ((data->filter_str && !data_test->filter_str) ||
4988 (!data->filter_str && data_test->filter_str))
4989 return false;
4990 }
52a7f16d
TZ		 4991
4992 for_each_hist_field(i, hist_data) {
4993 key_field = hist_data->fields[i];
4994 key_field_test = hist_data_test->fields[i];
4995
4996 if (key_field->flags != key_field_test->flags)
4997 return false;
5463bfda 4998 if (!compatible_field(key_field->field, key_field_test->field))
52a7f16d
TZ		 4999 return false;
5000 if (key_field->offset != key_field_test->offset)
5001 return false;
ad42febe
TZ		 5002 if (key_field->size != key_field_test->size)
5003 return false;
5004 if (key_field->is_signed != key_field_test->is_signed)
5005 return false;
1a361dfc
TZ		 5006 if (!!key_field->var.name != !!key_field_test->var.name)
5007 return false;
5008 if (key_field->var.name &&
5009 strcmp(key_field->var.name, key_field_test->var.name) != 0)
5010 return false;
52a7f16d
TZ		 5011 }
5012
5013 for (i = 0; i < hist_data->n_sort_keys; i++) {
5014 sort_key = &hist_data->sort_keys[i];
5015 sort_key_test = &hist_data_test->sort_keys[i];
5016
5017 if (sort_key->field_idx != sort_key_test->field_idx ||
5018 sort_key->descending != sort_key_test->descending)
5019 return false;
5020 }
5021
5463bfda 5022 if (!ignore_filter && data->filter_str &&
52a7f16d
TZ		 5023 (strcmp(data->filter_str, data_test->filter_str) != 0))
5024 return false;
5025
5026 return true;
5027}
5028
7ef224d1
TZ		 5029static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5030 struct event_trigger_data *data,
5031 struct trace_event_file *file)
5032{
83e99914 5033 struct hist_trigger_data *hist_data = data->private_data;
5463bfda 5034 struct event_trigger_data *test, *named_data = NULL;
7ef224d1
TZ		 5035 int ret = 0;
5036
5463bfda
TZ		 5037 if (hist_data->attrs->name) {
5038 named_data = find_named_trigger(hist_data->attrs->name);
5039 if (named_data) {
5040 if (!hist_trigger_match(data, named_data, named_data,
5041 true)) {
5042 ret = -EINVAL;
5043 goto out;
5044 }
5045 }
5046 }
5047
5048 if (hist_data->attrs->name && !named_data)
5049 goto new;
5050
7ef224d1
TZ		 5051 list_for_each_entry_rcu(test, &file->triggers, list) {
5052 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 5053 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d 5054 continue;
83e99914
TZ		 5055 if (hist_data->attrs->pause)
5056 test->paused = true;
5057 else if (hist_data->attrs->cont)
5058 test->paused = false;
e86ae9ba
TZ		 5059 else if (hist_data->attrs->clear)
5060 hist_clear(test);
83e99914
TZ		 5061 else
5062 ret = -EEXIST;
7ef224d1
TZ		 5063 goto out;
5064 }
5065 }
5463bfda 5066 new:
e86ae9ba 5067 if (hist_data->attrs->cont || hist_data->attrs->clear) {
83e99914
TZ		 5068 ret = -ENOENT;
5069 goto out;
5070 }
5071
7522c03a
TZ		 5072 if (hist_data->attrs->pause)
5073 data->paused = true;
5074
5463bfda
TZ		 5075 if (named_data) {
5076 destroy_hist_data(data->private_data);
5077 data->private_data = named_data->private_data;
5078 set_named_trigger_data(data, named_data);
5079 data->ops = &event_hist_trigger_named_ops;
5080 }
5081
7ef224d1
TZ		 5082 if (data->ops->init) {
5083 ret = data->ops->init(data->ops, data);
5084 if (ret < 0)
5085 goto out;
5086 }
5087
7ef224d1
TZ		 5088 ret++;
5089
ad42febe
TZ		 5090 if (hist_data->enable_timestamps)
5091 tracing_set_time_stamp_abs(file->tr, true);
067fe038
TZ		 5092 out:
5093 return ret;
5094}
5095
5096static int hist_trigger_enable(struct event_trigger_data *data,
5097 struct trace_event_file *file)
5098{
5099 int ret = 0;
5100
5101 list_add_tail_rcu(&data->list, &file->triggers);
5102
5103 update_cond_flag(file);
ad42febe 5104
7ef224d1
TZ		 5105 if (trace_event_trigger_enable_disable(file, 1) < 0) {
5106 list_del_rcu(&data->list);
5107 update_cond_flag(file);
5108 ret--;
5109 }
067fe038 5110
7ef224d1
TZ		 5111 return ret;
5112}
5113
4b147936
TZ		 5114static bool have_hist_trigger_match(struct event_trigger_data *data,
5115 struct trace_event_file *file)
5116{
5117 struct hist_trigger_data *hist_data = data->private_data;
5118 struct event_trigger_data *test, *named_data = NULL;
5119 bool match = false;
5120
5121 if (hist_data->attrs->name)
5122 named_data = find_named_trigger(hist_data->attrs->name);
5123
5124 list_for_each_entry_rcu(test, &file->triggers, list) {
5125 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5126 if (hist_trigger_match(data, test, named_data, false)) {
5127 match = true;
5128 break;
5129 }
5130 }
5131 }
5132
5133 return match;
5134}
5135
067fe038
TZ		 5136static bool hist_trigger_check_refs(struct event_trigger_data *data,
5137 struct trace_event_file *file)
5138{
5139 struct hist_trigger_data *hist_data = data->private_data;
5140 struct event_trigger_data *test, *named_data = NULL;
5141
5142 if (hist_data->attrs->name)
5143 named_data = find_named_trigger(hist_data->attrs->name);
5144
5145 list_for_each_entry_rcu(test, &file->triggers, list) {
5146 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5147 if (!hist_trigger_match(data, test, named_data, false))
5148 continue;
5149 hist_data = test->private_data;
5150 if (check_var_refs(hist_data))
5151 return true;
5152 break;
5153 }
5154 }
5155
5156 return false;
5157}
5158
52a7f16d
TZ		 5159static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5160 struct event_trigger_data *data,
5161 struct trace_event_file *file)
5162{
5463bfda
TZ		 5163 struct hist_trigger_data *hist_data = data->private_data;
5164 struct event_trigger_data *test, *named_data = NULL;
52a7f16d
TZ		 5165 bool unregistered = false;
5166
5463bfda
TZ		 5167 if (hist_data->attrs->name)
5168 named_data = find_named_trigger(hist_data->attrs->name);
5169
52a7f16d
TZ		 5170 list_for_each_entry_rcu(test, &file->triggers, list) {
5171 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5463bfda 5172 if (!hist_trigger_match(data, test, named_data, false))
52a7f16d
TZ		 5173 continue;
5174 unregistered = true;
5175 list_del_rcu(&test->list);
5176 trace_event_trigger_enable_disable(file, 0);
5177 update_cond_flag(file);
5178 break;
5179 }
5180 }
5181
5182 if (unregistered && test->ops->free)
5183 test->ops->free(test->ops, test);
ad42febe
TZ		 5184
5185 if (hist_data->enable_timestamps) {
30350d65 5186 if (!hist_data->remove || unregistered)
ad42febe
TZ		 5187 tracing_set_time_stamp_abs(file->tr, false);
5188 }
52a7f16d
TZ		 5189}
5190
067fe038
TZ		 5191static bool hist_file_check_refs(struct trace_event_file *file)
5192{
5193 struct hist_trigger_data *hist_data;
5194 struct event_trigger_data *test;
5195
5196 list_for_each_entry_rcu(test, &file->triggers, list) {
5197 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5198 hist_data = test->private_data;
5199 if (check_var_refs(hist_data))
5200 return true;
5201 }
5202 }
5203
5204 return false;
5205}
5206
52a7f16d
TZ		 5207static void hist_unreg_all(struct trace_event_file *file)
5208{
47c18569 5209 struct event_trigger_data *test, *n;
ad42febe 5210 struct hist_trigger_data *hist_data;
4b147936
TZ		 5211 struct synth_event *se;
5212 const char *se_name;
52a7f16d 5213
067fe038
TZ		 5214 if (hist_file_check_refs(file))
5215 return;
5216
47c18569 5217 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d 5218 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
ad42febe 5219 hist_data = test->private_data;
52a7f16d
TZ		 5220 list_del_rcu(&test->list);
5221 trace_event_trigger_enable_disable(file, 0);
4b147936
TZ		 5222
5223 mutex_lock(&synth_event_mutex);
5224 se_name = trace_event_name(file->event_call);
5225 se = find_synth_event(se_name);
5226 if (se)
5227 se->ref--;
5228 mutex_unlock(&synth_event_mutex);
5229
52a7f16d 5230 update_cond_flag(file);
ad42febe
TZ		 5231 if (hist_data->enable_timestamps)
5232 tracing_set_time_stamp_abs(file->tr, false);
52a7f16d
TZ		 5233 if (test->ops->free)
5234 test->ops->free(test->ops, test);
5235 }
5236 }
5237}
5238
7ef224d1
TZ		 5239static int event_hist_trigger_func(struct event_command *cmd_ops,
5240 struct trace_event_file *file,
5241 char *glob, char *cmd, char *param)
5242{
5243 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5244 struct event_trigger_data *trigger_data;
5245 struct hist_trigger_attrs *attrs;
5246 struct event_trigger_ops *trigger_ops;
5247 struct hist_trigger_data *hist_data;
4b147936
TZ		 5248 struct synth_event *se;
5249 const char *se_name;
30350d65 5250 bool remove = false;
ec5ce098 5251 char *trigger, *p;
7ef224d1
TZ		 5252 int ret = 0;
5253
5254 if (!param)
5255 return -EINVAL;
5256
30350d65
TZ		 5257 if (glob[0] == '!')
5258 remove = true;
5259
ec5ce098
TZ		 5260 /*
5261 * separate the trigger from the filter (k:v [if filter])
5262 * allowing for whitespace in the trigger
5263 */
5264 p = trigger = param;
5265 do {
5266 p = strstr(p, "if");
5267 if (!p)
5268 break;
5269 if (p == param)
5270 return -EINVAL;
5271 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5272 p++;
5273 continue;
5274 }
5275 if (p >= param + strlen(param) - strlen("if") - 1)
5276 return -EINVAL;
5277 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5278 p++;
5279 continue;
5280 }
5281 break;
5282 } while (p);
5283
5284 if (!p)
5285 param = NULL;
5286 else {
5287 *(p - 1) = '\0';
5288 param = strstrip(p);
5289 trigger = strstrip(trigger);
5290 }
7ef224d1
TZ		 5291
5292 attrs = parse_hist_trigger_attrs(trigger);
5293 if (IS_ERR(attrs))
5294 return PTR_ERR(attrs);
5295
5296 if (attrs->map_bits)
5297 hist_trigger_bits = attrs->map_bits;
5298
30350d65 5299 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
7ef224d1
TZ		 5300 if (IS_ERR(hist_data)) {
5301 destroy_hist_trigger_attrs(attrs);
5302 return PTR_ERR(hist_data);
5303 }
5304
5305 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5306
7ef224d1 5307 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
4b147936
TZ		 5308 if (!trigger_data) {
5309 ret = -ENOMEM;
7ef224d1 5310 goto out_free;
4b147936 5311 }
7ef224d1
TZ		 5312
5313 trigger_data->count = -1;
5314 trigger_data->ops = trigger_ops;
5315 trigger_data->cmd_ops = cmd_ops;
5316
5317 INIT_LIST_HEAD(&trigger_data->list);
5318 RCU_INIT_POINTER(trigger_data->filter, NULL);
5319
5320 trigger_data->private_data = hist_data;
5321
52a7f16d
TZ		 5322 /* if param is non-empty, it's supposed to be a filter */
5323 if (param && cmd_ops->set_filter) {
5324 ret = cmd_ops->set_filter(param, trigger_data, file);
5325 if (ret < 0)
5326 goto out_free;
5327 }
5328
30350d65 5329 if (remove) {
4b147936
TZ		 5330 if (!have_hist_trigger_match(trigger_data, file))
5331 goto out_free;
5332
067fe038
TZ		 5333 if (hist_trigger_check_refs(trigger_data, file)) {
5334 ret = -EBUSY;
5335 goto out_free;
5336 }
5337
7ef224d1 5338 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
4b147936
TZ		 5339
5340 mutex_lock(&synth_event_mutex);
5341 se_name = trace_event_name(file->event_call);
5342 se = find_synth_event(se_name);
5343 if (se)
5344 se->ref--;
5345 mutex_unlock(&synth_event_mutex);
5346
7ef224d1
TZ		 5347 ret = 0;
5348 goto out_free;
5349 }
5350
7ef224d1
TZ		 5351 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5352 /*
5353 * The above returns on success the # of triggers registered,
5354 * but if it didn't register any it returns zero. Consider no
5355 * triggers registered a failure too.
5356 */
5357 if (!ret) {
e86ae9ba 5358 if (!(attrs->pause || attrs->cont || attrs->clear))
83e99914 5359 ret = -ENOENT;
7ef224d1
TZ		 5360 goto out_free;
5361 } else if (ret < 0)
5362 goto out_free;
067fe038
TZ		 5363
5364 if (get_named_trigger_data(trigger_data))
5365 goto enable;
5366
5367 if (has_hist_vars(hist_data))
5368 save_hist_vars(hist_data);
5369
0212e2aa
TZ		 5370 ret = create_actions(hist_data, file);
5371 if (ret)
5372 goto out_unreg;
5373
067fe038
TZ		 5374 ret = tracing_map_init(hist_data->map);
5375 if (ret)
5376 goto out_unreg;
5377enable:
5378 ret = hist_trigger_enable(trigger_data, file);
5379 if (ret)
5380 goto out_unreg;
5381
4b147936
TZ		 5382 mutex_lock(&synth_event_mutex);
5383 se_name = trace_event_name(file->event_call);
5384 se = find_synth_event(se_name);
5385 if (se)
5386 se->ref++;
5387 mutex_unlock(&synth_event_mutex);
5388
7ef224d1
TZ		 5389 /* Just return zero, not the number of registered triggers */
5390 ret = 0;
5391 out:
5392 return ret;
067fe038
TZ		 5393 out_unreg:
5394 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
7ef224d1
TZ		 5395 out_free:
5396 if (cmd_ops->set_filter)
5397 cmd_ops->set_filter(NULL, trigger_data, NULL);
5398
067fe038
TZ		 5399 remove_hist_vars(hist_data);
5400
7ef224d1
TZ		 5401 kfree(trigger_data);
5402
5403 destroy_hist_data(hist_data);
5404 goto out;
5405}
5406
5407static struct event_command trigger_hist_cmd = {
5408 .name = "hist",
5409 .trigger_type = ETT_EVENT_HIST,
5410 .flags = EVENT_CMD_FL_NEEDS_REC,
5411 .func = event_hist_trigger_func,
5412 .reg = hist_register_trigger,
52a7f16d
TZ		 5413 .unreg = hist_unregister_trigger,
5414 .unreg_all = hist_unreg_all,
7ef224d1
TZ		 5415 .get_trigger_ops = event_hist_get_trigger_ops,
5416 .set_filter = set_trigger_filter,
5417};
5418
5419__init int register_trigger_hist_cmd(void)
5420{
5421 int ret;
5422
5423 ret = register_event_command(&trigger_hist_cmd);
5424 WARN_ON(ret < 0);
5425
5426 return ret;
5427}
d0bad49b
TZ		 5428
5429static void
1ac4f51c
TZ		 5430hist_enable_trigger(struct event_trigger_data *data, void *rec,
5431 struct ring_buffer_event *event)
d0bad49b
TZ		 5432{
5433 struct enable_trigger_data *enable_data = data->private_data;
5434 struct event_trigger_data *test;
5435
5436 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5437 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5438 if (enable_data->enable)
5439 test->paused = false;
5440 else
5441 test->paused = true;
d0bad49b
TZ		 5442 }
5443 }
5444}
5445
5446static void
1ac4f51c
TZ		 5447hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5448 struct ring_buffer_event *event)
d0bad49b
TZ		 5449{
5450 if (!data->count)
5451 return;
5452
5453 if (data->count != -1)
5454 (data->count)--;
5455
1ac4f51c 5456 hist_enable_trigger(data, rec, event);
d0bad49b
TZ		 5457}
5458
5459static struct event_trigger_ops hist_enable_trigger_ops = {
5460 .func = hist_enable_trigger,
5461 .print = event_enable_trigger_print,
5462 .init = event_trigger_init,
5463 .free = event_enable_trigger_free,
5464};
5465
5466static struct event_trigger_ops hist_enable_count_trigger_ops = {
5467 .func = hist_enable_count_trigger,
5468 .print = event_enable_trigger_print,
5469 .init = event_trigger_init,
5470 .free = event_enable_trigger_free,
5471};
5472
5473static struct event_trigger_ops hist_disable_trigger_ops = {
5474 .func = hist_enable_trigger,
5475 .print = event_enable_trigger_print,
5476 .init = event_trigger_init,
5477 .free = event_enable_trigger_free,
5478};
5479
5480static struct event_trigger_ops hist_disable_count_trigger_ops = {
5481 .func = hist_enable_count_trigger,
5482 .print = event_enable_trigger_print,
5483 .init = event_trigger_init,
5484 .free = event_enable_trigger_free,
5485};
5486
5487static struct event_trigger_ops *
5488hist_enable_get_trigger_ops(char *cmd, char *param)
5489{
5490 struct event_trigger_ops *ops;
5491 bool enable;
5492
5493 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5494
5495 if (enable)
5496 ops = param ? &hist_enable_count_trigger_ops :
5497 &hist_enable_trigger_ops;
5498 else
5499 ops = param ? &hist_disable_count_trigger_ops :
5500 &hist_disable_trigger_ops;
5501
5502 return ops;
5503}
5504
52a7f16d
TZ		 5505static void hist_enable_unreg_all(struct trace_event_file *file)
5506{
47c18569 5507 struct event_trigger_data *test, *n;
52a7f16d 5508
47c18569 5509 list_for_each_entry_safe(test, n, &file->triggers, list) {
52a7f16d
TZ		 5510 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5511 list_del_rcu(&test->list);
5512 update_cond_flag(file);
5513 trace_event_trigger_enable_disable(file, 0);
5514 if (test->ops->free)
5515 test->ops->free(test->ops, test);
5516 }
5517 }
5518}
5519
d0bad49b
TZ		 5520static struct event_command trigger_hist_enable_cmd = {
5521 .name = ENABLE_HIST_STR,
5522 .trigger_type = ETT_HIST_ENABLE,
5523 .func = event_enable_trigger_func,
5524 .reg = event_enable_register_trigger,
5525 .unreg = event_enable_unregister_trigger,
52a7f16d 5526 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5527 .get_trigger_ops = hist_enable_get_trigger_ops,
5528 .set_filter = set_trigger_filter,
5529};
5530
5531static struct event_command trigger_hist_disable_cmd = {
5532 .name = DISABLE_HIST_STR,
5533 .trigger_type = ETT_HIST_ENABLE,
5534 .func = event_enable_trigger_func,
5535 .reg = event_enable_register_trigger,
5536 .unreg = event_enable_unregister_trigger,
52a7f16d 5537 .unreg_all = hist_enable_unreg_all,
d0bad49b
TZ		 5538 .get_trigger_ops = hist_enable_get_trigger_ops,
5539 .set_filter = set_trigger_filter,
5540};
5541
5542static __init void unregister_trigger_hist_enable_disable_cmds(void)
5543{
5544 unregister_event_command(&trigger_hist_enable_cmd);
5545 unregister_event_command(&trigger_hist_disable_cmd);
5546}
5547
5548__init int register_trigger_hist_enable_disable_cmds(void)
5549{
5550 int ret;
5551
5552 ret = register_event_command(&trigger_hist_enable_cmd);
5553 if (WARN_ON(ret < 0))
5554 return ret;
5555 ret = register_event_command(&trigger_hist_disable_cmd);
5556 if (WARN_ON(ret < 0))
5557 unregister_trigger_hist_enable_disable_cmds();
5558
5559 return ret;
5560}
4b147936
TZ		 5561
5562static __init int trace_events_hist_init(void)
5563{
5564 struct dentry *entry = NULL;
5565 struct dentry *d_tracer;
5566 int err = 0;
5567
5568 d_tracer = tracing_init_dentry();
5569 if (IS_ERR(d_tracer)) {
5570 err = PTR_ERR(d_tracer);
5571 goto err;
5572 }
5573
5574 entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5575 NULL, &synth_events_fops);
5576 if (!entry) {
5577 err = -ENODEV;
5578 goto err;
5579 }
5580
5581 return err;
5582 err:
5583 pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5584
5585 return err;
5586}
5587
5588fs_initcall(trace_events_hist_init);
Linux 6.x block layer and io_uring tree(s)